(self, dict_file: str)
| 92 | self._add_symbol("vocab_pad_token{}".format(i), 0) |
| 93 | |
| 94 | def _load_dict(self, dict_file: str): |
| 95 | with open(dict_file, "r") as f: |
| 96 | for line in f: |
| 97 | line = line.strip() |
| 98 | if line == "" or line.startswith("#"): |
| 99 | continue |
| 100 | sym, count = line.split() |
| 101 | self._add_symbol(sym, int(count)) |
| 102 | |
| 103 | def _add_symbol(self, sym: str, count: int): |
| 104 | self._idx[sym] = self._num_symbols |