| 819 | } |
| 820 | |
| 821 | size_t tokenize_file( |
| 822 | struct llama_context * lctx, |
| 823 | const char * filename, |
| 824 | const std::string & sample_start, |
| 825 | bool include_sample_start, |
| 826 | bool overlapping_samples, |
| 827 | unsigned context_length, |
| 828 | std::vector<llama_token> & out_tokens, |
| 829 | std::vector<size_t> & out_samples_begin, |
| 830 | std::vector<size_t> & out_samples_size) { |
| 831 | struct llama_file f(filename, "rb"); |
| 832 | |
| 833 | if (f.size == 0) { |
| 834 | out_tokens.clear(); |
| 835 | out_samples_begin.clear(); |
| 836 | out_samples_size.clear(); |
| 837 | printf("%s: warning: empty or not existing training data file '%s'\n", |
| 838 | __func__, filename); |
| 839 | return out_tokens.size(); |
| 840 | } |
| 841 | |
| 842 | // account for possible leading whitespace that will be added by tokenizer |
| 843 | // e.g. '\t' will be tokenized by llama spm tokenizer to [29871, 12] |
| 844 | const int n_max_tokens_overhead = 1; |
| 845 | |
| 846 | std::vector<char> buf; |
| 847 | buf.resize(f.size); |
| 848 | |
| 849 | f.read_raw(buf.data(), f.size); |
| 850 | |
| 851 | std::vector<int> utf8_units; |
| 852 | std::vector<int> utf8_nunits; |
| 853 | utf8_units.resize(buf.size()); |
| 854 | utf8_nunits.resize(buf.size()); |
| 855 | mark_utf8_units(buf.data(), utf8_units.data(), utf8_nunits.data(), buf.size()); |
| 856 | |
| 857 | if (sample_start.size() == 0) { |
| 858 | // tokenize all data at once |
| 859 | out_tokens.resize(buf.size() + n_max_tokens_overhead); |
| 860 | |
| 861 | int n_tokens = llama_tokenize( |
| 862 | llama_get_model(lctx), |
| 863 | buf.data(), |
| 864 | (int) buf.size(), |
| 865 | out_tokens.data(), |
| 866 | (int) out_tokens.size(), |
| 867 | false, false); |
| 868 | if (n_tokens < 0) { |
| 869 | out_tokens.resize(-n_tokens); |
| 870 | n_tokens = llama_tokenize( |
| 871 | llama_get_model(lctx), |
| 872 | buf.data(), |
| 873 | (int) buf.size(), |
| 874 | out_tokens.data(), |
| 875 | (int) out_tokens.size(), |
| 876 | false, false); |
| 877 | } |
| 878 | if (n_tokens >= 0) { |
no test coverage detected