(tokenizer)
| 52 | |
| 53 | |
| 54 | def test_encode_long_text(tokenizer): |
| 55 | text = "OpenMythos is a recurrent depth transformer. " * 100 |
| 56 | ids = tokenizer.encode(text) |
| 57 | print(f"Long text ({len(text)} chars) → {len(ids)} tokens") |
| 58 | assert len(ids) > 100 |
| 59 | |
| 60 | |
| 61 | def test_custom_model_id(): |