()
| 66 | |
| 67 | |
| 68 | def get_robert_frost(): |
| 69 | word2idx = {'START': 0, 'END': 1} |
| 70 | current_idx = 2 |
| 71 | sentences = [] |
| 72 | for line in open('../hmm_class/robert_frost.txt'): |
| 73 | line = line.strip() |
| 74 | if line: |
| 75 | tokens = remove_punctuation(line.lower()).split() |
| 76 | sentence = [] |
| 77 | for t in tokens: |
| 78 | if t not in word2idx: |
| 79 | word2idx[t] = current_idx |
| 80 | current_idx += 1 |
| 81 | idx = word2idx[t] |
| 82 | sentence.append(idx) |
| 83 | sentences.append(sentence) |
| 84 | return sentences, word2idx |
| 85 | |
| 86 | def my_tokenizer(s): |
| 87 | s = remove_punctuation(s) |
no outgoing calls
no test coverage detected