(t *testing.T)
| 497 | } |
| 498 | |
| 499 | func TestNGramTokenizerQueryTokens(t *testing.T) { |
| 500 | tokenizer := NGramTokenizer{lang: "en"} |
| 501 | |
| 502 | queryTokens, err := BuildNGramQueryTokens("quick brown fox", tokenizer) |
| 503 | require.NoError(t, err) |
| 504 | require.Greater(t, len(queryTokens), 0, "QueryTokens should return tokens for trigram input") |
| 505 | |
| 506 | id := tokenizer.Identifier() |
| 507 | require.Contains(t, queryTokens, encodeToken("quick brown fox", id)) |
| 508 | |
| 509 | queryTokens2, err := BuildNGramQueryTokens("hello", tokenizer) |
| 510 | require.NoError(t, err) |
| 511 | require.Greater(t, len(queryTokens2), 0, "QueryTokens should return tokens for single word") |
| 512 | require.Contains(t, queryTokens2, encodeToken("hello", id)) |
| 513 | |
| 514 | queryTokens3, err := BuildNGramQueryTokens("", tokenizer) |
| 515 | require.NoError(t, err) |
| 516 | require.Equal(t, 0, len(queryTokens3), "QueryTokens should return empty for empty string") |
| 517 | } |
| 518 | |
| 519 | func TestNGramTokenizerQueryTokensVariousInputs(t *testing.T) { |
| 520 | tokenizer := NGramTokenizer{lang: "en"} |
nothing calls this directly
no test coverage detected