Split a string into individual characters, optionally removing punctuation and stop-words in the process.
(line, lowercase=True, filter_punctuation=True, **kwargs)
| 149 | |
| 150 | |
| 151 | def tokenize_chars(line, lowercase=True, filter_punctuation=True, **kwargs): |
| 152 | """ |
| 153 | Split a string into individual characters, optionally removing punctuation |
| 154 | and stop-words in the process. |
| 155 | """ |
| 156 | line = line.lower() if lowercase else line |
| 157 | line = strip_punctuation(line) if filter_punctuation else line |
| 158 | chars = list(re.sub(" {2,}", " ", line).strip()) |
| 159 | return chars |
| 160 | |
| 161 | |
| 162 | def remove_stop_words(words): |
nothing calls this directly
no test coverage detected