| 15 | } |
| 16 | |
| 17 | struct chunk { |
| 18 | // filename |
| 19 | std::string filename; |
| 20 | // original file position |
| 21 | size_t filepos; |
| 22 | // original text data |
| 23 | std::string textdata; |
| 24 | // tokenized text data |
| 25 | std::vector<llama_token> tokens; |
| 26 | // embedding |
| 27 | std::vector<float> embedding; |
| 28 | }; |
| 29 | |
| 30 | // chunk file data to chunks of size >= chunk_size |
| 31 | // chunk_separator is the separator between chunks |