Preprocessor.
| 49 | |
| 50 | |
| 51 | class Preprocessor(object): |
| 52 | """Preprocessor.""" |
| 53 | |
| 54 | def __init__(self, max_seq_length, tokenizer): |
| 55 | self._tokenizer = tokenizer |
| 56 | self._max_seq_length = max_seq_length |
| 57 | tf.logging.info("Max sequence length {}".format(self._max_seq_length)) |
| 58 | |
| 59 | def create_example(self, key, text): |
| 60 | """Create example.""" |
| 61 | tokens = ["[CLS]"] |
| 62 | tokens.extend(self._tokenizer.tokenize(text)) |
| 63 | if len(tokens) > self._max_seq_length - 1: |
| 64 | tokens = tokens[:self._max_seq_length - 1] |
| 65 | tokens.append("[SEP]") |
| 66 | inputs_ids = self._tokenizer.convert_tokens_to_ids(tokens) |
| 67 | |
| 68 | example = tf.train.Example() |
| 69 | add_int64_feature("input_ids", inputs_ids, example) |
| 70 | add_int64_feature("key", [int(key)], example) |
| 71 | return example.SerializeToString() |
| 72 | |
| 73 | def example_from_json_line(self, key, text): |
| 74 | if not isinstance(key, six.text_type): |
| 75 | key = int(key.decode("utf-8")) |
| 76 | if not isinstance(text, six.text_type): |
| 77 | text = text.decode("utf-8") |
| 78 | return self.create_example(key, text) |
| 79 | |
| 80 | |
| 81 | def create_block_info(input_path, preprocessor): |
no outgoing calls
no test coverage detected
searching dependent graphs…