MCPcopy Create free account
hub / github.com/google-research/language / retrieve

Function retrieve

language/orqa/models/orqa_model.py:88–149  ·  view source on GitHub ↗

Do retrieval.

(features, retriever_beam_size, mode, params)

Source from the content-addressed store, hash-verified

86
87
88def retrieve(features, retriever_beam_size, mode, params):
89 """Do retrieval."""
90 tokenizer, vocab_lookup_table = bert_utils.get_tf_tokenizer(
91 params["retriever_module_path"])
92
93 question_token_ids = tokenizer.tokenize(
94 tf.expand_dims(features["question"], 0))
95 question_token_ids = tf.cast(
96 question_token_ids.merge_dims(1, 2).to_tensor(), tf.int32)
97 cls_token_id = vocab_lookup_table.lookup(tf.constant("[CLS]"))
98 sep_token_id = vocab_lookup_table.lookup(tf.constant("[SEP]"))
99 question_token_ids = tf.concat(
100 [[[tf.cast(cls_token_id, tf.int32)]], question_token_ids,
101 [[tf.cast(sep_token_id, tf.int32)]]], -1)
102
103 retriever_module = hub.Module(
104 params["retriever_module_path"],
105 tags={"train"} if mode == tf_estimator.ModeKeys.TRAIN else {},
106 trainable=True)
107
108 # [1, projection_size]
109 question_emb = retriever_module(
110 inputs=dict(
111 input_ids=question_token_ids,
112 input_mask=tf.ones_like(question_token_ids),
113 segment_ids=tf.zeros_like(question_token_ids)),
114 signature="projected")
115
116 block_emb, searcher = scann_utils.load_scann_searcher(
117 var_name="block_emb",
118 checkpoint_path=os.path.join(params["retriever_module_path"], "encoded",
119 "encoded.ckpt"),
120 num_neighbors=retriever_beam_size)
121
122 # [1, retriever_beam_size]
123 retrieved_block_ids, _ = searcher.search_batched(question_emb)
124
125 # [1, retriever_beam_size, projection_size]
126 retrieved_block_emb = tf.gather(block_emb, retrieved_block_ids)
127
128 # [retriever_beam_size]
129 retrieved_block_ids = tf.squeeze(retrieved_block_ids)
130
131 # [retriever_beam_size, projection_size]
132 retrieved_block_emb = tf.squeeze(retrieved_block_emb)
133
134 # [1, retriever_beam_size]
135 retrieved_logits = tf.matmul(
136 question_emb, retrieved_block_emb, transpose_b=True)
137
138 # [retriever_beam_size]
139 retrieved_logits = tf.squeeze(retrieved_logits, 0)
140
141 blocks_dataset = tf.data.TFRecordDataset(
142 params["block_records_path"], buffer_size=512 * 1024 * 1024)
143 blocks_dataset = blocks_dataset.batch(
144 params["num_block_records"], drop_remainder=True)
145 blocks = tf.get_local_variable(

Callers 1

model_fnFunction · 0.85

Calls 3

to_tensorMethod · 0.80
constantMethod · 0.80
tokenizeMethod · 0.45

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…