MCPcopy Create free account
hub / github.com/google-research/language / parse_examples

Function parse_examples

language/orqa/predict/encode_blocks.py:72–90  ·  view source on GitHub ↗

Make retrieval examples.

(serialized_example)

Source from the content-addressed store, hash-verified

70
71
72def parse_examples(serialized_example):
73 """Make retrieval examples."""
74 feature_spec = dict(
75 title_ids=tf.FixedLenSequenceFeature([], tf.int64, True),
76 token_ids=tf.FixedLenSequenceFeature([], tf.int64, True))
77 features = tf.parse_single_example(serialized_example, feature_spec)
78 features = {k: tf.cast(v, tf.int32) for k, v in features.items()}
79 tokenizer = bert_utils.get_tokenizer(FLAGS.retriever_module_path)
80 cls_id, sep_id = tokenizer.convert_tokens_to_ids(["[CLS]", "[SEP]"])
81 block_ids, block_mask, block_segment_ids = bert_utils.pad_or_truncate_pair(
82 token_ids_a=features["title_ids"],
83 token_ids_b=features["token_ids"],
84 sequence_length=FLAGS.block_seq_len,
85 cls_id=cls_id,
86 sep_id=sep_id)
87 return dict(
88 block_ids=block_ids,
89 block_mask=block_mask,
90 block_segment_ids=block_segment_ids)
91
92
93def input_fn(params):

Callers

nothing calls this directly

Calls 2

get_tokenizerMethod · 0.45
convert_tokens_to_idsMethod · 0.45

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…