chunk file data to chunks of size >= chunk_size chunk_separator is the separator between chunks
| 30 | // chunk file data to chunks of size >= chunk_size |
| 31 | // chunk_separator is the separator between chunks |
| 32 | static std::vector<chunk> chunk_file(const std::string & filename, int chunk_size, const std::string & chunk_separator) { |
| 33 | std::vector<chunk> chunks; |
| 34 | std::ifstream f(filename.c_str()); |
| 35 | |
| 36 | if (!f.is_open()) { |
| 37 | LOG_ERR("could not open file %s\n", filename.c_str()); |
| 38 | return chunks; |
| 39 | } |
| 40 | |
| 41 | chunk current_chunk; |
| 42 | char buffer[1024]; |
| 43 | int64_t filepos = 0; |
| 44 | std::string current; |
| 45 | while (f.read(buffer, 1024)) { |
| 46 | current += std::string(buffer, f.gcount()); |
| 47 | size_t pos; |
| 48 | while ((pos = current.find(chunk_separator)) != std::string::npos) { |
| 49 | current_chunk.textdata += current.substr(0, pos + chunk_separator.size()); |
| 50 | if ((int) current_chunk.textdata.size() > chunk_size) { |
| 51 | // save chunk |
| 52 | current_chunk.filepos = filepos; |
| 53 | current_chunk.filename = filename; |
| 54 | chunks.push_back(current_chunk); |
| 55 | // update filepos |
| 56 | filepos += (int) current_chunk.textdata.size(); |
| 57 | // reset current_chunk |
| 58 | current_chunk = chunk(); |
| 59 | } |
| 60 | current = current.substr(pos + chunk_separator.size()); |
| 61 | } |
| 62 | |
| 63 | } |
| 64 | // add leftover data to last chunk |
| 65 | if (current_chunk.textdata.size() > 0) { |
| 66 | if (chunks.empty()) { |
| 67 | current_chunk.filepos = filepos; |
| 68 | current_chunk.filename = filename; |
| 69 | chunks.push_back(current_chunk); |
| 70 | } else { |
| 71 | chunks.back().textdata += current_chunk.textdata; |
| 72 | } |
| 73 | } |
| 74 | f.close(); |
| 75 | return chunks; |
| 76 | } |
| 77 | |
| 78 | static void batch_add_seq(llama_batch & batch, const std::vector<int32_t> & tokens, llama_seq_id seq_id) { |
| 79 | size_t n_tokens = tokens.size(); |