Loads a vocabulary file into a dictionary.
(vocab_file)
| 95 | |
| 96 | |
| 97 | def load_vocab(vocab_file): |
| 98 | """Loads a vocabulary file into a dictionary.""" |
| 99 | vocab = collections.OrderedDict() |
| 100 | with open(vocab_file, "r", encoding="utf-8") as reader: |
| 101 | tokens = reader.readlines() |
| 102 | for index, token in enumerate(tokens): |
| 103 | token = token.rstrip("\n") |
| 104 | vocab[token] = index |
| 105 | return vocab |
| 106 | |
| 107 | |
| 108 | def whitespace_tokenize(text): |