(model_path, stop_words)
| 37 | ]) |
| 38 | @pytest.mark.parametrize('stop_words', ['.', ' ', '?', '']) |
| 39 | def test_tokenizer_with_stop_words(model_path, stop_words): |
| 40 | tokenizer = HuggingFaceTokenizer(model_path, trust_remote_code=True) |
| 41 | indexes = tokenizer.indexes_containing_token(stop_words) |
| 42 | assert indexes is not None |
| 43 | |
| 44 | |
| 45 | def test_glm4_special_token(): |
nothing calls this directly
no test coverage detected