Extract meaningful lowercase tokens.
(text: str)
| 272 | |
| 273 | |
| 274 | def tokenize(text: str) -> set[str]: |
| 275 | """Extract meaningful lowercase tokens.""" |
| 276 | if not text: |
| 277 | return set() |
| 278 | text = text.lower() |
| 279 | text = re.sub(r"```.*?```", "", text, flags=re.DOTALL) |
| 280 | text = re.sub(r"https?://\S+", "", text) |
| 281 | words = re.findall(r"[a-z][a-z0-9_.]+", text) |
| 282 | return {w for w in words if len(w) > 2 and w not in STOPWORDS} |
| 283 | |
| 284 | |
| 285 | def extract_signatures(text: str) -> set[str]: |