Get cleaning function by type.
(clean_type: str)
| 132 | |
| 133 | |
| 134 | def get_clean_fn(clean_type: str): |
| 135 | """Get cleaning function by type.""" |
| 136 | if clean_type == 'lower': |
| 137 | return _clean_lower |
| 138 | elif clean_type == 'whitespace': |
| 139 | return _clean_whitespace |
| 140 | else: |
| 141 | return _clean_lower # Default to lower |
| 142 | |
| 143 | |
| 144 | class SimpleTokenizer(object): |