(
self,
vocab_file,
unk_token="<unk>",
bos_token="<_start>",
eos_token="<_end>",
pad_token="<_pad>",
sp_model_kwargs: Optional[Dict[str, Any]] = None,
add_bos_token=True,
add_eos_token=False,
clean_up_tokenization_spaces=False,
**kwargs,
)
| 23 | model_input_names = ["input_ids", "attention_mask"] |
| 24 | |
| 25 | def __init__( |
| 26 | self, |
| 27 | vocab_file, |
| 28 | unk_token="<unk>", |
| 29 | bos_token="<_start>", |
| 30 | eos_token="<_end>", |
| 31 | pad_token="<_pad>", |
| 32 | sp_model_kwargs: Optional[Dict[str, Any]] = None, |
| 33 | add_bos_token=True, |
| 34 | add_eos_token=False, |
| 35 | clean_up_tokenization_spaces=False, |
| 36 | **kwargs, |
| 37 | ): |
| 38 | self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs |
| 39 | bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token |
| 40 | eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token |
| 41 | unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token |
| 42 | pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token |
| 43 | self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) |
| 44 | self.sp_model.Load(vocab_file) |
| 45 | super().__init__( |
| 46 | bos_token=bos_token, |
| 47 | eos_token=eos_token, |
| 48 | unk_token=unk_token, |
| 49 | pad_token=pad_token, |
| 50 | add_bos_token=add_bos_token, |
| 51 | add_eos_token=add_eos_token, |
| 52 | sp_model_kwargs=self.sp_model_kwargs, |
| 53 | clean_up_tokenization_spaces=clean_up_tokenization_spaces, |
| 54 | **kwargs, |
| 55 | ) |
| 56 | self.vocab_file = vocab_file |
| 57 | self.add_bos_token = add_bos_token |
| 58 | self.add_eos_token = add_eos_token |
| 59 | |
| 60 | |
| 61 | def __getstate__(self): |
nothing calls this directly
no outgoing calls
no test coverage detected