Make retrieval examples.
(serialized_example)
| 70 | |
| 71 | |
| 72 | def parse_examples(serialized_example): |
| 73 | """Make retrieval examples.""" |
| 74 | feature_spec = dict( |
| 75 | title_ids=tf.FixedLenSequenceFeature([], tf.int64, True), |
| 76 | token_ids=tf.FixedLenSequenceFeature([], tf.int64, True)) |
| 77 | features = tf.parse_single_example(serialized_example, feature_spec) |
| 78 | features = {k: tf.cast(v, tf.int32) for k, v in features.items()} |
| 79 | tokenizer = bert_utils.get_tokenizer(FLAGS.retriever_module_path) |
| 80 | cls_id, sep_id = tokenizer.convert_tokens_to_ids(["[CLS]", "[SEP]"]) |
| 81 | block_ids, block_mask, block_segment_ids = bert_utils.pad_or_truncate_pair( |
| 82 | token_ids_a=features["title_ids"], |
| 83 | token_ids_b=features["token_ids"], |
| 84 | sequence_length=FLAGS.block_seq_len, |
| 85 | cls_id=cls_id, |
| 86 | sep_id=sep_id) |
| 87 | return dict( |
| 88 | block_ids=block_ids, |
| 89 | block_mask=block_mask, |
| 90 | block_segment_ids=block_segment_ids) |
| 91 | |
| 92 | |
| 93 | def input_fn(params): |
nothing calls this directly
no test coverage detected
searching dependent graphs…