(text)
| 23 | |
| 24 | |
| 25 | def preprocess(text): |
| 26 | try: |
| 27 | nltk.data.find("tokenizers/punkt") |
| 28 | except LookupError: |
| 29 | nltk.download("punkt") |
| 30 | tokens = word_tokenize(text) |
| 31 | # split into lower-case word tokens, in numpy array with shape of (seq, 1) |
| 32 | words = np.asarray([w.lower() for w in tokens]).reshape(-1, 1) |
| 33 | # split words into chars, in numpy array with shape of (seq, 1, 1, 16) |
| 34 | chars = [[c for c in t][:16] for t in tokens] |
| 35 | chars = [cs + [""] * (16 - len(cs)) for cs in chars] |
| 36 | chars = np.asarray(chars).reshape(-1, 1, 1, 16) |
| 37 | return words, chars |
| 38 | |
| 39 | |
| 40 | def get_map_func(filepath): |
no test coverage detected