(self, data)
| 35 | 'bert-base-uncased') |
| 36 | |
| 37 | def __call__(self, data): |
| 38 | text = data['label'] |
| 39 | char_text, char_len = self.encode(text) |
| 40 | if char_text is None: |
| 41 | return None |
| 42 | data['length'] = np.array(char_len) |
| 43 | data['char_label'] = np.array(char_text) |
| 44 | if self.only_char: |
| 45 | return data |
| 46 | bpe_text = self.bpe_encode(text) |
| 47 | if bpe_text is None: |
| 48 | return None |
| 49 | wp_text = self.wp_encode(text) |
| 50 | data['bpe_label'] = np.array(bpe_text) |
| 51 | data['wp_label'] = wp_text |
| 52 | return data |
| 53 | |
| 54 | def add_special_char(self, dict_character): |
| 55 | dict_character = self.list_token + dict_character |
nothing calls this directly
no test coverage detected