| 5 | |
| 6 | |
| 7 | class Dictionary(object): |
| 8 | def __init__(self): |
| 9 | self.word2idx = {} |
| 10 | self.idx2word = [] |
| 11 | self.counter = Counter() |
| 12 | self.total = 0 |
| 13 | |
| 14 | def add_word(self, word): |
| 15 | if word not in self.word2idx: |
| 16 | self.idx2word.append(word) |
| 17 | self.word2idx[word] = len(self.idx2word) - 1 |
| 18 | token_id = self.word2idx[word] |
| 19 | self.counter[token_id] += 1 |
| 20 | self.total += 1 |
| 21 | return self.word2idx[word] |
| 22 | |
| 23 | def __len__(self): |
| 24 | return len(self.idx2word) |
| 25 | |
| 26 | |
| 27 | class Corpus(object): |