MCPcopy Create free account
hub / github.com/google-research/language / main

Function main

language/multivec/utils/data_processor.py:107–127  ·  view source on GitHub ↗
(_)

Source from the content-addressed store, hash-verified

105
106
107def main(_):
108 pool = multiprocessing.Pool(FLAGS.num_threads)
109 tf.logging.info("Using hub module %s", FLAGS.bert_hub_module_path)
110 tokenizer = get_tokenizer(FLAGS.bert_hub_module_path)
111 preprocessor = Preprocessor(FLAGS.max_seq_length, tokenizer)
112 mapper = functools.partial(create_block_info, preprocessor=preprocessor)
113 block_count = 0
114 input_paths = tf.io.gfile.glob(FLAGS.input_pattern)
115 tf.logging.info("Processing %d input files.", len(input_paths))
116 output_dir = os.path.dirname(FLAGS.output_path)
117 if not tf.io.gfile.exists(output_dir):
118 tf.io.gfile.makedirs(output_dir)
119
120 with tf.python_io.TFRecordWriter(FLAGS.output_path) as examples_writer:
121 for examples in pool.imap_unordered(mapper, input_paths):
122 for example in examples:
123 examples_writer.write(example)
124 block_count += 1
125 if block_count % 10000 == 0:
126 tf.logging.info("Wrote %d blocks.", block_count)
127 tf.logging.info("Wrote %d blocks in total.", block_count)
128
129
130if __name__ == "__main__":

Callers

nothing calls this directly

Calls 2

get_tokenizerFunction · 0.70
PreprocessorClass · 0.70

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…