| 328 | } |
| 329 | |
| 330 | static std::map<std::string, std::vector<gpt_vocab::id>> extract_tests_from_file(const std::string & fpath_test){ |
| 331 | if (fpath_test.empty()){ |
| 332 | fprintf(stderr, "%s : No test file found.\n", __func__); |
| 333 | return std::map<std::string, std::vector<gpt_vocab::id>>(); |
| 334 | } |
| 335 | |
| 336 | std::map<std::string, std::vector<gpt_vocab::id>> tests; |
| 337 | |
| 338 | auto fin = std::ifstream(fpath_test, std::ios_base::in); |
| 339 | const char * delimeter = " => "; |
| 340 | const char del_tok = ','; |
| 341 | std::string line; |
| 342 | while (std::getline(fin, line)) { |
| 343 | size_t delimiterPos = line.find(delimeter); |
| 344 | if (delimiterPos != std::string::npos) { |
| 345 | std::string text = line.substr(0, delimiterPos); |
| 346 | std::string s_tokens = line.substr(delimiterPos + std::strlen(delimeter)); |
| 347 | tests[text] = parse_tokens_from_string(s_tokens, del_tok); |
| 348 | } |
| 349 | } |
| 350 | return tests; |
| 351 | } |
| 352 | |
| 353 | void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test){ |
| 354 | std::map<std::string, std::vector<gpt_vocab::id>> tests = extract_tests_from_file(fpath_test); |