(
self,
vocab_file,
merges_file,
errors="replace",
unk_token="<|endoftext|>",
bos_token="<|endoftext|>",
eos_token="<eom>",
pad_token=None,
add_prefix_space=False,
add_bos_token=False,
**kwargs,
)
| 142 | model_input_names = ["input_ids", "attention_mask"] |
| 143 | |
| 144 | def __init__( |
| 145 | self, |
| 146 | vocab_file, |
| 147 | merges_file, |
| 148 | errors="replace", |
| 149 | unk_token="<|endoftext|>", |
| 150 | bos_token="<|endoftext|>", |
| 151 | eos_token="<eom>", |
| 152 | pad_token=None, |
| 153 | add_prefix_space=False, |
| 154 | add_bos_token=False, |
| 155 | **kwargs, |
| 156 | ): |
| 157 | bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token |
| 158 | eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token |
| 159 | unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token |
| 160 | pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token |
| 161 | super().__init__( |
| 162 | errors=errors, |
| 163 | unk_token=unk_token, |
| 164 | bos_token=bos_token, |
| 165 | eos_token=eos_token, |
| 166 | pad_token=pad_token, |
| 167 | add_prefix_space=add_prefix_space, |
| 168 | add_bos_token=add_bos_token, |
| 169 | **kwargs, |
| 170 | ) |
| 171 | self.add_bos_token = add_bos_token |
| 172 | |
| 173 | with open(vocab_file, encoding="utf-8") as vocab_handle: |
| 174 | self.encoder = json.load(vocab_handle) |
| 175 | self.decoder = {v: k for k, v in self.encoder.items()} |
| 176 | self.errors = errors # how to handle errors in decoding |
| 177 | self.byte_encoder = bytes_to_unicode() |
| 178 | self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} |
| 179 | with open(merges_file, encoding="utf-8") as merges_handle: |
| 180 | bpe_merges = merges_handle.read().split("\n")[1:-1] |
| 181 | bpe_merges = [tuple(merge.split()) for merge in bpe_merges] |
| 182 | self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges)))) |
| 183 | self.cache = {} |
| 184 | self.add_prefix_space = add_prefix_space |
| 185 | |
| 186 | # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions |
| 187 | self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""") |
| 188 | |
| 189 | @property |
| 190 | def vocab_size(self): |
nothing calls this directly
no test coverage detected