| 138 | |
| 139 | |
| 140 | void Tokenize(const Llama4Tokenizer& tokenizer, const std::string& prompt) { |
| 141 | // Perform 5 warmup runs |
| 142 | for (int i = 0; i < 5; i++) { |
| 143 | tokenizer.encode(prompt); |
| 144 | } |
| 145 | |
| 146 | auto start_time = std::chrono::high_resolution_clock::now(); |
| 147 | std::vector<int> tokens = tokenizer.encode(prompt); |
| 148 | auto end_time = std::chrono::high_resolution_clock::now(); |
| 149 | auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time); |
| 150 | printf("Tokenization took %lld μs\n", duration.count()); |
| 151 | printf("Token count: %zu\n", tokens.size()); |
| 152 | // for (size_t i = 0; i < tokens.size(); i++) { |
| 153 | // printf("%d\n", tokens[i]); |
| 154 | // } |
| 155 | |
| 156 | // decode the tokens. |
| 157 | std::vector<unsigned char> decoded_bytes = tokenizer.bpe->decode_bytes(tokens); |
| 158 | // printf("Decoded bytes: %s\n", decoded_bytes.data()); |
| 159 | |
| 160 | std::vector<int> times; |
| 161 | for (int i = 0; i < 1000000; i++) { |
| 162 | auto start_time = std::chrono::high_resolution_clock::now(); |
| 163 | std::vector<int> _tokens = tokenizer.encode(prompt); |
| 164 | auto end_time = std::chrono::high_resolution_clock::now(); |
| 165 | auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time); |
| 166 | times.push_back(duration.count()); |
| 167 | } |
| 168 | |
| 169 | printf("Average tokenization time: %lld μs\n", std::accumulate(times.begin(), times.end(), 0LL) / times.size()); |
| 170 | auto min_time = *std::min_element(times.begin(), times.end()); |
| 171 | auto max_time = *std::max_element(times.begin(), times.end()); |
| 172 | printf("Min tokenization time: %lld μs\n", min_time); |
| 173 | printf("Max tokenization time: %lld μs\n", max_time); |
| 174 | |
| 175 | // // auto start_time = std::chrono::high_resolution_clock::now(); |
| 176 | // // std::vector<int> tokens = tokenizer.encode(prompt); |
| 177 | // // auto end_time = std::chrono::high_resolution_clock::now(); |
| 178 | // // auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time); |
| 179 | // // printf("Tokenization took %lld μs\n", duration.count()); |
| 180 | } |
| 181 | |
| 182 | int main() { |
| 183 | // build the tokenizer. |