| 29 | 'brief', 'vol', 'fundamental', 'guide', 'essential', 'printed', |
| 30 | 'third', 'second', 'fourth', }) |
| 31 | def my_tokenizer(s): |
| 32 | s = s.lower() # downcase |
| 33 | tokens = nltk.tokenize.word_tokenize(s) # split string into words (tokens) |
| 34 | tokens = [t for t in tokens if len(t) > 2] # remove short words, they're probably not useful |
| 35 | tokens = [wordnet_lemmatizer.lemmatize(t) for t in tokens] # put words into base form |
| 36 | tokens = [t for t in tokens if t not in stopwords] # remove stopwords |
| 37 | tokens = [t for t in tokens if not any(c.isdigit() for c in t)] # remove any digits, i.e. "3rd edition" |
| 38 | return tokens |
| 39 | |
| 40 | |
| 41 | # create a word-to-index map so that we can create our word-frequency vectors later |