()
| 21 | |
| 22 | |
| 23 | def test_llama_cpp_tokenization(): |
| 24 | llama = llama_cpp.Llama(model_path=MODEL, vocab_only=True, verbose=False) |
| 25 | |
| 26 | assert llama |
| 27 | assert llama._ctx.ctx is not None |
| 28 | |
| 29 | text = b"Hello World" |
| 30 | |
| 31 | tokens = llama.tokenize(text) |
| 32 | assert tokens[0] == llama.token_bos() |
| 33 | assert tokens == [1, 15043, 2787] |
| 34 | detokenized = llama.detokenize(tokens) |
| 35 | assert detokenized == text |
| 36 | |
| 37 | tokens = llama.tokenize(text, add_bos=False) |
| 38 | assert tokens[0] != llama.token_bos() |
| 39 | assert tokens == [15043, 2787] |
| 40 | |
| 41 | detokenized = llama.detokenize(tokens) |
| 42 | assert detokenized != text |
| 43 | |
| 44 | text = b"Hello World</s>" |
| 45 | tokens = llama.tokenize(text) |
| 46 | assert tokens[-1] != llama.token_eos() |
| 47 | assert tokens == [1, 15043, 2787, 829, 29879, 29958] |
| 48 | |
| 49 | tokens = llama.tokenize(text, special=True) |
| 50 | assert tokens[-1] == llama.token_eos() |
| 51 | assert tokens == [1, 15043, 2787, 2] |
| 52 | |
| 53 | text = b"" |
| 54 | tokens = llama.tokenize(text, add_bos=True, special=True) |
| 55 | assert tokens[-1] != llama.token_eos() |
| 56 | assert tokens == [llama.token_bos()] |
| 57 | assert text == llama.detokenize(tokens) |
| 58 | |
| 59 | |
| 60 | @pytest.fixture |
nothing calls this directly
no test coverage detected
searching dependent graphs…