(text: Any)
| 130 | |
| 131 | |
| 132 | def tokenize(text: Any) -> List[str]: |
| 133 | tokens: List[str] = [] |
| 134 | for raw_token in TOKEN_RE.findall(str(text or "").casefold()): |
| 135 | token = raw_token.replace("_", "-") |
| 136 | if len(token) <= 1: |
| 137 | continue |
| 138 | tokens.append(token) |
| 139 | if "-" in token: |
| 140 | tokens.extend(part for part in token.split("-") if len(part) > 1) |
| 141 | return tokens |
| 142 | |
| 143 | |
| 144 | def paper_text(row: Dict[str, Any]) -> str: |
no outgoing calls
no test coverage detected