lambda fn for removing stopwrods
(text, tokenizer, stopword_list, is_lower_case=False)
| 138 | |
| 139 | |
| 140 | def remove_stopwords(text, tokenizer, stopword_list, is_lower_case=False): |
| 141 | ''' lambda fn for removing stopwrods ''' |
| 142 | tokens = tokenizer.tokenize(text) |
| 143 | tokens = [token.strip() for token in tokens] |
| 144 | if is_lower_case: |
| 145 | filtered_tokens = [ |
| 146 | token for token in tokens if token not in stopword_list |
| 147 | ] |
| 148 | else: |
| 149 | filtered_tokens = [ |
| 150 | token for token in tokens if token.lower() not in stopword_list |
| 151 | ] |
| 152 | filtered_text = ' '.join(filtered_tokens) |
| 153 | return filtered_text |
| 154 | |
| 155 | |
| 156 | def tokenize(x): |