Collect bigram counts for the tokens in vocab
(self, vocab)
| 262 | return self |
| 263 | |
| 264 | def _get_counts(self, vocab): |
| 265 | """Collect bigram counts for the tokens in vocab""" |
| 266 | pair_counts = defaultdict(int) |
| 267 | for word, count in vocab.items(): |
| 268 | pairs = ngrams(word.split(" "), 2) |
| 269 | for p in pairs: |
| 270 | pair_counts[p] += count |
| 271 | return pair_counts |
| 272 | |
| 273 | def _merge(self, bigram, vocab): |
| 274 | """Replace `bigram` with a single token and update vocab accordingly""" |