| 54 | # so let's create a function that does all this pre-processing for us |
| 55 | |
| 56 | def my_tokenizer(s): |
| 57 | s = s.lower() # downcase |
| 58 | tokens = nltk.tokenize.word_tokenize(s) # split string into words (tokens) |
| 59 | tokens = [t for t in tokens if len(t) > 2] # remove short words, they're probably not useful |
| 60 | tokens = [wordnet_lemmatizer.lemmatize(t) for t in tokens] # put words into base form |
| 61 | tokens = [t for t in tokens if t not in stopwords] # remove stopwords |
| 62 | return tokens |
| 63 | |
| 64 | |
| 65 | # create a word-to-index map so that we can create our word-frequency vectors later |