TokenizerNames returns the tokenizer names for given predicate
(ctx context.Context, pred string)
| 413 | |
| 414 | // TokenizerNames returns the tokenizer names for given predicate |
| 415 | func (s *state) TokenizerNames(ctx context.Context, pred string) []string { |
| 416 | var names []string |
| 417 | tokenizers := s.Tokenizer(ctx, pred) |
| 418 | for _, t := range tokenizers { |
| 419 | names = append(names, t.Name()) |
| 420 | } |
| 421 | return names |
| 422 | } |
| 423 | |
| 424 | // HasTokenizer is a convenience func that checks if a given tokenizer is found in pred. |
| 425 | // Returns true if found, else false. |
no test coverage detected