(self, name, seq_len=None, clean=None, **kwargs)
| 37 | class HuggingfaceTokenizer: |
| 38 | |
| 39 | def __init__(self, name, seq_len=None, clean=None, **kwargs): |
| 40 | assert clean in (None, 'whitespace', 'lower', 'canonicalize') |
| 41 | self.name = name |
| 42 | self.seq_len = seq_len |
| 43 | self.clean = clean |
| 44 | |
| 45 | # init tokenizer |
| 46 | self.tokenizer = AutoTokenizer.from_pretrained(name, **kwargs) |
| 47 | self.vocab_size = self.tokenizer.vocab_size |
| 48 | |
| 49 | def __call__(self, sequence, **kwargs): |
| 50 | return_mask = kwargs.pop('return_mask', False) |
nothing calls this directly
no outgoing calls
no test coverage detected