Make retrieval examples.
(serialized_example)
| 109 | |
| 110 | |
| 111 | def parse_examples(serialized_example): |
| 112 | """Make retrieval examples.""" |
| 113 | feature_spec = dict( |
| 114 | input_ids=tf.FixedLenSequenceFeature([], tf.int64, True), |
| 115 | key=tf.FixedLenSequenceFeature([], tf.int64, True)) |
| 116 | features = tf.parse_single_example(serialized_example, feature_spec) |
| 117 | features = {k: tf.cast(v, tf.int32) for k, v in features.items()} |
| 118 | block_ids, block_mask, block_segment_ids = pad_or_truncate_pair( |
| 119 | token_ids=features["input_ids"], sequence_length=FLAGS.block_seq_len) |
| 120 | key = tf.ensure_shape(features["key"], [1]) |
| 121 | return dict( |
| 122 | block_ids=block_ids, |
| 123 | block_mask=block_mask, |
| 124 | block_segment_ids=block_segment_ids, |
| 125 | key=key) |
| 126 | |
| 127 | |
| 128 | def input_fn(params): |
nothing calls this directly
no test coverage detected
searching dependent graphs…