(text: Any)
| 135 | |
| 136 | |
| 137 | def tokenize(text: Any) -> List[str]: |
| 138 | tokens: List[str] = [] |
| 139 | for raw_token in TOKEN_RE.findall(str(text or "").casefold()): |
| 140 | token = raw_token.replace("_", "-") |
| 141 | if len(token) <= 1: |
| 142 | continue |
| 143 | tokens.append(token) |
| 144 | if "-" in token: |
| 145 | tokens.extend(part for part in token.split("-") if len(part) > 1) |
| 146 | return tokens |
| 147 | |
| 148 | |
| 149 | def paper_text(row: Dict[str, Any]) -> str: |
no outgoing calls
no test coverage detected