MCPcopy Create free account
hub / github.com/google-research/language / Preprocessor

Class Preprocessor

language/multivec/utils/data_processor.py:51–78  ·  view source on GitHub ↗

Preprocessor.

Source from the content-addressed store, hash-verified

49
50
51class Preprocessor(object):
52 """Preprocessor."""
53
54 def __init__(self, max_seq_length, tokenizer):
55 self._tokenizer = tokenizer
56 self._max_seq_length = max_seq_length
57 tf.logging.info("Max sequence length {}".format(self._max_seq_length))
58
59 def create_example(self, key, text):
60 """Create example."""
61 tokens = ["[CLS]"]
62 tokens.extend(self._tokenizer.tokenize(text))
63 if len(tokens) > self._max_seq_length - 1:
64 tokens = tokens[:self._max_seq_length - 1]
65 tokens.append("[SEP]")
66 inputs_ids = self._tokenizer.convert_tokens_to_ids(tokens)
67
68 example = tf.train.Example()
69 add_int64_feature("input_ids", inputs_ids, example)
70 add_int64_feature("key", [int(key)], example)
71 return example.SerializeToString()
72
73 def example_from_json_line(self, key, text):
74 if not isinstance(key, six.text_type):
75 key = int(key.decode("utf-8"))
76 if not isinstance(text, six.text_type):
77 text = text.decode("utf-8")
78 return self.create_example(key, text)
79
80
81def create_block_info(input_path, preprocessor):

Callers 1

mainFunction · 0.70

Calls

no outgoing calls

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…