| 147 | |
| 148 | |
| 149 | void Tokenize(const MistralTokenizer& tokenizer, const std::string& prompt) { |
| 150 | auto start_time = std::chrono::high_resolution_clock::now(); |
| 151 | std::vector<int> tokens = tokenizer.encode(prompt); |
| 152 | auto end_time = std::chrono::high_resolution_clock::now(); |
| 153 | auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time); |
| 154 | printf("Tokenization took %lld μs\n", duration.count()); |
| 155 | // for (size_t i = 0; i < tokens.size(); i++) { |
| 156 | // printf("%d\n", tokens[i]); |
| 157 | // } |
| 158 | // return; |
| 159 | |
| 160 | std::vector<int> times; |
| 161 | for (int i = 0; i < 1000; i++) { |
| 162 | auto start_time = std::chrono::high_resolution_clock::now(); |
| 163 | std::vector<int> _tokens = tokenizer.encode(prompt); |
| 164 | auto end_time = std::chrono::high_resolution_clock::now(); |
| 165 | auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time); |
| 166 | times.push_back(duration.count()); |
| 167 | } |
| 168 | |
| 169 | printf("Average tokenization time: %lld μs\n", std::accumulate(times.begin(), times.end(), 0LL) / times.size()); |
| 170 | auto min_time = *std::min_element(times.begin(), times.end()); |
| 171 | auto max_time = *std::max_element(times.begin(), times.end()); |
| 172 | printf("Min tokenization time: %lld μs\n", min_time); |
| 173 | printf("Max tokenization time: %lld μs\n", max_time); |
| 174 | |
| 175 | // auto start_time = std::chrono::high_resolution_clock::now(); |
| 176 | // std::vector<int> tokens = tokenizer.encode(prompt); |
| 177 | // auto end_time = std::chrono::high_resolution_clock::now(); |
| 178 | // auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time); |
| 179 | // printf("Tokenization took %lld μs\n", duration.count()); |
| 180 | } |
| 181 | |
| 182 | int main() { |
| 183 | // printf("Hello, World!\n"); |