(_)
| 151 | |
| 152 | |
| 153 | def main(_): |
| 154 | if FLAGS.encode_query: |
| 155 | encoding_path = FLAGS.output_path + "/queries_" + FLAGS.suffix + "_encodings.h5py" |
| 156 | passage_id_path = FLAGS.output_path + "/queries_" + FLAGS.suffix + "_ids.h5py" |
| 157 | else: |
| 158 | encoding_path = FLAGS.output_path + "/passage_encodings.h5py" |
| 159 | passage_id_path = FLAGS.output_path + "/passage_ids.h5py" |
| 160 | if not tf.io.gfile.exists(FLAGS.output_path): |
| 161 | tf.io.gfile.makedirs(FLAGS.output_path) |
| 162 | tf.logging.info("Embeddings will be written to %s", encoding_path) |
| 163 | tf.logging.info("Passage ID will be written to %s", passage_id_path) |
| 164 | |
| 165 | if FLAGS.use_tpu and FLAGS.tpu_name: |
| 166 | tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( |
| 167 | FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) |
| 168 | else: |
| 169 | tpu_cluster_resolver = None |
| 170 | run_config = tf_estimator.tpu.RunConfig( |
| 171 | cluster=tpu_cluster_resolver, |
| 172 | master=FLAGS.master, |
| 173 | tpu_config=tf_estimator.tpu.TPUConfig(iterations_per_loop=1000)) |
| 174 | if FLAGS.encode_query: |
| 175 | model_fn = model_fn_query |
| 176 | else: |
| 177 | model_fn = model_fn_passage |
| 178 | estimator = tf_estimator.tpu.TPUEstimator( |
| 179 | use_tpu=FLAGS.use_tpu, |
| 180 | model_fn=model_fn, |
| 181 | config=run_config, |
| 182 | train_batch_size=FLAGS.batch_size, |
| 183 | eval_batch_size=FLAGS.batch_size, |
| 184 | predict_batch_size=FLAGS.batch_size) |
| 185 | |
| 186 | start_time = time.time() |
| 187 | all_block_emb = None |
| 188 | i = 0 |
| 189 | num_vec = FLAGS.num_blocks * FLAGS.num_vec_per_block |
| 190 | for outputs in estimator.predict(input_fn=input_fn): |
| 191 | if i == 0: |
| 192 | all_block_emb = np.zeros( |
| 193 | shape=(num_vec, outputs["block_emb"].shape[-1]), dtype=np.float32) |
| 194 | all_paragraph_ids = np.zeros(shape=(num_vec), dtype=np.int32) |
| 195 | if i >= num_vec: |
| 196 | break |
| 197 | for k in range(FLAGS.num_vec_per_block): |
| 198 | all_block_emb[i, :] = outputs["block_emb"][k, :] |
| 199 | all_paragraph_ids[i] = outputs["key"][0] |
| 200 | i += 1 |
| 201 | if i % 1000 == 0: |
| 202 | elapse_time = time.time() - start_time |
| 203 | examples_per_second = i / elapse_time |
| 204 | remaining_minutes = ((num_vec - i) / examples_per_second) / 60 |
| 205 | tf.logging.info( |
| 206 | "[%d] examples/sec: %.2f, " |
| 207 | "elapsed minutes: %.2f, " |
| 208 | "remaining minutes: %.2f", i, examples_per_second, elapse_time / 60, |
| 209 | remaining_minutes) |
| 210 | tf.logging.info("Expected %d rows, found %d rows", FLAGS.num_blocks, i) |
nothing calls this directly
no test coverage detected
searching dependent graphs…