| 363 | typename bpe_tok |
| 364 | > |
| 365 | void bpe_tokenizer_test( |
| 366 | ) |
| 367 | /*! |
| 368 | requires |
| 369 | - bpe_tok is an implementation of bpe_tokenizer.h |
| 370 | ensures |
| 371 | - runs tests on bpe_tok for compliance with the specs |
| 372 | !*/ |
| 373 | { |
| 374 | print_spinner(); |
| 375 | |
| 376 | bpe_tok test; |
| 377 | |
| 378 | std::string training_text = R"( |
| 379 | Byte Pair Encoding (BPE) is a subword tokenization algorithm widely used in Natural Language Processing (NLP). |
| 380 | It iteratively merges the most frequent pairs of bytes or characters to form a vocabulary of subword units. |
| 381 | This approach is particularly useful for handling out-of-vocabulary words and reducing the size of the vocabulary |
| 382 | while maintaining the ability to represent any text. BPE was introduced in the paper "Neural Machine Translation |
| 383 | of Rare Words with Subword Units" by Sennrich et al. in 2016. The algorithm is simple yet effective and has been |
| 384 | adopted in many state-of-the-art NLP models, including GPT and BERT. |
| 385 | )"; |
| 386 | |
| 387 | test.train(training_text, 300); |
| 388 | |
| 389 | std::ostringstream out_stream; |
| 390 | serialize(test, out_stream); |
| 391 | |
| 392 | bpe_tok loaded_test; |
| 393 | std::istringstream in_stream(out_stream.str()); |
| 394 | deserialize(loaded_test, in_stream); |
| 395 | |
| 396 | std::vector<std::string> test_strings = { |
| 397 | "This is a test of the tokenisation process...\nimplemented in the Dlib library!", // English |
| 398 | "Ceci est un test du processus de\ntokenisation implémenté dans\nla bibliothèque Dlib!", // French |
| 399 | "Dette er en test af tokeniseringsprocessen implementeret i Dlib-biblioteket!", // Danish |
| 400 | "这是对Dlib库中实现的标记化过程的测试!" // Chinese |
| 401 | }; |
| 402 | |
| 403 | for (const auto& text : test_strings) { |
| 404 | std::vector<int> encoded = loaded_test.encode(text); |
| 405 | std::string decoded = postprocess_decoded_text(loaded_test.decode(encoded)); |
| 406 | |
| 407 | DLIB_TEST_MSG(text == decoded, "decoded: " << decoded); |
| 408 | } |
| 409 | } |
| 410 | |
| 411 | class tokenizer_tester : public tester |
| 412 | { |
nothing calls this directly
no test coverage detected