(_)
| 105 | |
| 106 | |
| 107 | def main(_): |
| 108 | pool = multiprocessing.Pool(FLAGS.num_threads) |
| 109 | tf.logging.info("Using hub module %s", FLAGS.bert_hub_module_path) |
| 110 | tokenizer = get_tokenizer(FLAGS.bert_hub_module_path) |
| 111 | preprocessor = Preprocessor(FLAGS.max_seq_length, tokenizer) |
| 112 | mapper = functools.partial(create_block_info, preprocessor=preprocessor) |
| 113 | block_count = 0 |
| 114 | input_paths = tf.io.gfile.glob(FLAGS.input_pattern) |
| 115 | tf.logging.info("Processing %d input files.", len(input_paths)) |
| 116 | output_dir = os.path.dirname(FLAGS.output_path) |
| 117 | if not tf.io.gfile.exists(output_dir): |
| 118 | tf.io.gfile.makedirs(output_dir) |
| 119 | |
| 120 | with tf.python_io.TFRecordWriter(FLAGS.output_path) as examples_writer: |
| 121 | for examples in pool.imap_unordered(mapper, input_paths): |
| 122 | for example in examples: |
| 123 | examples_writer.write(example) |
| 124 | block_count += 1 |
| 125 | if block_count % 10000 == 0: |
| 126 | tf.logging.info("Wrote %d blocks.", block_count) |
| 127 | tf.logging.info("Wrote %d blocks in total.", block_count) |
| 128 | |
| 129 | |
| 130 | if __name__ == "__main__": |
nothing calls this directly
no test coverage detected
searching dependent graphs…