(self)
| 25 | super().tearDown() |
| 26 | |
| 27 | def test_tokenize(self): |
| 28 | cfg = dict(type='Tokenize', tokenizer_name='bert-base-cased') |
| 29 | preprocessor = build_preprocessor(cfg, Fields.nlp) |
| 30 | input = { |
| 31 | InputFields.text: |
| 32 | 'Do not meddle in the affairs of wizards, ' |
| 33 | 'for they are subtle and quick to anger.' |
| 34 | } |
| 35 | output = preprocessor(input) |
| 36 | self.assertTrue(InputFields.text in output) |
| 37 | self.assertEqual(output['input_ids'], [ |
| 38 | 101, 2091, 1136, 1143, 13002, 1107, 1103, 5707, 1104, 16678, 1116, |
| 39 | 117, 1111, 1152, 1132, 11515, 1105, 3613, 1106, 4470, 119, 102 |
| 40 | ]) |
| 41 | self.assertEqual( |
| 42 | output['token_type_ids'], |
| 43 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) |
| 44 | self.assertEqual( |
| 45 | output['attention_mask'], |
| 46 | [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]) |
| 47 | |
| 48 | def test_save_pretrained(self): |
| 49 | preprocessor = Preprocessor.from_pretrained( |
nothing calls this directly
no test coverage detected