MCPcopy Create free account
hub / github.com/google-research/language / main

Function main

language/multivec/predict/encode_blocks.py:153–215  ·  view source on GitHub ↗
(_)

Source from the content-addressed store, hash-verified

151
152
153def main(_):
154 if FLAGS.encode_query:
155 encoding_path = FLAGS.output_path + "/queries_" + FLAGS.suffix + "_encodings.h5py"
156 passage_id_path = FLAGS.output_path + "/queries_" + FLAGS.suffix + "_ids.h5py"
157 else:
158 encoding_path = FLAGS.output_path + "/passage_encodings.h5py"
159 passage_id_path = FLAGS.output_path + "/passage_ids.h5py"
160 if not tf.io.gfile.exists(FLAGS.output_path):
161 tf.io.gfile.makedirs(FLAGS.output_path)
162 tf.logging.info("Embeddings will be written to %s", encoding_path)
163 tf.logging.info("Passage ID will be written to %s", passage_id_path)
164
165 if FLAGS.use_tpu and FLAGS.tpu_name:
166 tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(
167 FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
168 else:
169 tpu_cluster_resolver = None
170 run_config = tf_estimator.tpu.RunConfig(
171 cluster=tpu_cluster_resolver,
172 master=FLAGS.master,
173 tpu_config=tf_estimator.tpu.TPUConfig(iterations_per_loop=1000))
174 if FLAGS.encode_query:
175 model_fn = model_fn_query
176 else:
177 model_fn = model_fn_passage
178 estimator = tf_estimator.tpu.TPUEstimator(
179 use_tpu=FLAGS.use_tpu,
180 model_fn=model_fn,
181 config=run_config,
182 train_batch_size=FLAGS.batch_size,
183 eval_batch_size=FLAGS.batch_size,
184 predict_batch_size=FLAGS.batch_size)
185
186 start_time = time.time()
187 all_block_emb = None
188 i = 0
189 num_vec = FLAGS.num_blocks * FLAGS.num_vec_per_block
190 for outputs in estimator.predict(input_fn=input_fn):
191 if i == 0:
192 all_block_emb = np.zeros(
193 shape=(num_vec, outputs["block_emb"].shape[-1]), dtype=np.float32)
194 all_paragraph_ids = np.zeros(shape=(num_vec), dtype=np.int32)
195 if i >= num_vec:
196 break
197 for k in range(FLAGS.num_vec_per_block):
198 all_block_emb[i, :] = outputs["block_emb"][k, :]
199 all_paragraph_ids[i] = outputs["key"][0]
200 i += 1
201 if i % 1000 == 0:
202 elapse_time = time.time() - start_time
203 examples_per_second = i / elapse_time
204 remaining_minutes = ((num_vec - i) / examples_per_second) / 60
205 tf.logging.info(
206 "[%d] examples/sec: %.2f, "
207 "elapsed minutes: %.2f, "
208 "remaining minutes: %.2f", i, examples_per_second, elapse_time / 60,
209 remaining_minutes)
210 tf.logging.info("Expected %d rows, found %d rows", FLAGS.num_blocks, i)

Callers

nothing calls this directly

Calls 2

save_to_h5pyFunction · 0.85
predictMethod · 0.45

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…