MCPcopy Create free account
hub / github.com/google-research/language / main

Function main

language/orqa/predict/encode_blocks.py:112–159  ·  view source on GitHub ↗
(_)

Source from the content-addressed store, hash-verified

110
111
112def main(_):
113 encoded_path = os.path.join(FLAGS.retriever_module_path, "encoded",
114 "encoded.ckpt")
115 tf.logging.info("Embeddings will be written to %s", encoded_path)
116
117 if FLAGS.use_tpu and FLAGS.tpu_name:
118 tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(
119 FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
120 else:
121 tpu_cluster_resolver = None
122 run_config = tf_estimator.tpu.RunConfig(
123 cluster=tpu_cluster_resolver,
124 master=FLAGS.master,
125 tpu_config=tf_estimator.tpu.TPUConfig(iterations_per_loop=1000))
126 estimator = tf_estimator.tpu.TPUEstimator(
127 use_tpu=FLAGS.use_tpu,
128 model_fn=model_fn,
129 config=run_config,
130 train_batch_size=FLAGS.batch_size,
131 eval_batch_size=FLAGS.batch_size,
132 predict_batch_size=FLAGS.batch_size)
133
134 start_time = time.time()
135 all_block_emb = None
136 i = 0
137 for outputs in estimator.predict(input_fn=input_fn):
138 if i == 0:
139 all_block_emb = np.zeros(
140 shape=(FLAGS.num_blocks, outputs["block_emb"].shape[-1]),
141 dtype=np.float32)
142 if i >= FLAGS.num_blocks:
143 break
144 all_block_emb[i, :] = outputs["block_emb"]
145 i += 1
146 if i % 1000 == 0:
147 elapse_time = time.time() - start_time
148 examples_per_second = i / elapse_time
149 remaining_minutes = ((FLAGS.num_blocks - i) / examples_per_second) / 60
150 tf.logging.info(
151 "[%d] examples/sec: %.2f, "
152 "elapsed minutes: %.2f, "
153 "remaining minutes: %.2f", i, examples_per_second, elapse_time / 60,
154 remaining_minutes)
155 tf.logging.info("Expected %d rows, found %d rows", FLAGS.num_blocks, i)
156 tf.logging.info("Saving block embedding to %s...", encoded_path)
157 scann_utils.write_array_to_checkpoint("block_emb", all_block_emb,
158 encoded_path)
159 tf.logging.info("Done saving block embeddings.")
160
161
162if __name__ == "__main__":

Callers

nothing calls this directly

Calls 1

predictMethod · 0.45

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…