Tokenize text into a list of token strings suitable for BM25-like algorithms indexing/retrieval. This method returns the actual sub-word tokens as strings (e.g., ["▁Hello", "▁world"]), preserving token boundaries. These tokens can be directly used as terms in BM25. Args:
(self, content: str)
| 43 | return self.tokenizer.decode(token_ids, skip_special_tokens=True) |
| 44 | |
| 45 | def segment(self, content: str) -> List[str]: |
| 46 | """Tokenize text into a list of token strings suitable for BM25-like algorithms indexing/retrieval. |
| 47 | |
| 48 | This method returns the actual sub-word tokens as strings (e.g., ["▁Hello", "▁world"]), |
| 49 | preserving token boundaries. These tokens can be directly used as terms in BM25. |
| 50 | |
| 51 | Args: |
| 52 | content: Input text string. |
| 53 | |
| 54 | Returns: |
| 55 | List of token strings (not IDs), ready for BM25-style processing. |
| 56 | """ |
| 57 | if not content.strip(): |
| 58 | return [] |
| 59 | |
| 60 | token_ids = self.encode(content) |
| 61 | # Decode each token ID individually to get its string representation |
| 62 | token_strings = [ |
| 63 | self.tokenizer.decode([tid], skip_special_tokens=True) |
| 64 | for tid in token_ids |
| 65 | ] |
| 66 | return token_strings |
| 67 | |
| 68 | def count_tokens(self, |
| 69 | contents: Union[str, List[str]]) -> Union[int, List[int]]: |
no test coverage detected