(_)
| 110 | |
| 111 | |
| 112 | def main(_): |
| 113 | encoded_path = os.path.join(FLAGS.retriever_module_path, "encoded", |
| 114 | "encoded.ckpt") |
| 115 | tf.logging.info("Embeddings will be written to %s", encoded_path) |
| 116 | |
| 117 | if FLAGS.use_tpu and FLAGS.tpu_name: |
| 118 | tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( |
| 119 | FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) |
| 120 | else: |
| 121 | tpu_cluster_resolver = None |
| 122 | run_config = tf_estimator.tpu.RunConfig( |
| 123 | cluster=tpu_cluster_resolver, |
| 124 | master=FLAGS.master, |
| 125 | tpu_config=tf_estimator.tpu.TPUConfig(iterations_per_loop=1000)) |
| 126 | estimator = tf_estimator.tpu.TPUEstimator( |
| 127 | use_tpu=FLAGS.use_tpu, |
| 128 | model_fn=model_fn, |
| 129 | config=run_config, |
| 130 | train_batch_size=FLAGS.batch_size, |
| 131 | eval_batch_size=FLAGS.batch_size, |
| 132 | predict_batch_size=FLAGS.batch_size) |
| 133 | |
| 134 | start_time = time.time() |
| 135 | all_block_emb = None |
| 136 | i = 0 |
| 137 | for outputs in estimator.predict(input_fn=input_fn): |
| 138 | if i == 0: |
| 139 | all_block_emb = np.zeros( |
| 140 | shape=(FLAGS.num_blocks, outputs["block_emb"].shape[-1]), |
| 141 | dtype=np.float32) |
| 142 | if i >= FLAGS.num_blocks: |
| 143 | break |
| 144 | all_block_emb[i, :] = outputs["block_emb"] |
| 145 | i += 1 |
| 146 | if i % 1000 == 0: |
| 147 | elapse_time = time.time() - start_time |
| 148 | examples_per_second = i / elapse_time |
| 149 | remaining_minutes = ((FLAGS.num_blocks - i) / examples_per_second) / 60 |
| 150 | tf.logging.info( |
| 151 | "[%d] examples/sec: %.2f, " |
| 152 | "elapsed minutes: %.2f, " |
| 153 | "remaining minutes: %.2f", i, examples_per_second, elapse_time / 60, |
| 154 | remaining_minutes) |
| 155 | tf.logging.info("Expected %d rows, found %d rows", FLAGS.num_blocks, i) |
| 156 | tf.logging.info("Saving block embedding to %s...", encoded_path) |
| 157 | scann_utils.write_array_to_checkpoint("block_emb", all_block_emb, |
| 158 | encoded_path) |
| 159 | tf.logging.info("Done saving block embeddings.") |
| 160 | |
| 161 | |
| 162 | if __name__ == "__main__": |
nothing calls this directly
no test coverage detected
searching dependent graphs…