(text: str)
| 229 | class SimpleTokenizer: |
| 230 | @staticmethod |
| 231 | def tokenize(text: str) -> List[str]: |
| 232 | # Normalize unicode and clean text |
| 233 | text = unicodedata.normalize("NFD", text) |
| 234 | text = re.sub(r"[^\w\s]", " ", text.lower()) |
| 235 | # Split and filter |
| 236 | tokens = [ |
| 237 | token |
| 238 | for token in text.split() |
| 239 | if len(token) > 1 and token not in STOPWORDS |
| 240 | ] |
| 241 | return tokens |
| 242 | |
| 243 | |
| 244 | def compute_bm25_idf(total_docs: int, docs_with_term: int) -> float: |
no outgoing calls
no test coverage detected