MCPcopy Create free account
hub / github.com/google-research/language / input_fn

Method input_fn

language/realm/retrieval.py:351–397  ·  view source on GitHub ↗

Constructs the dataset fed to Estimator.

(params)

Source from the content-addressed store, hash-verified

349 }
350
351 def input_fn(params):
352 """Constructs the dataset fed to Estimator."""
353 # We cannot access self._featurizer via closure, because this function is
354 # passed to another device. Hence, we need to reconstruct the featurizer
355 # from its hyerparameters (passed through `params`).
356
357 tokenizer = featurization.Tokenizer(
358 vocab_path=params['vocab_path'],
359 do_lower_case=params['do_lower_case'])
360
361 featurizer = featurization.Featurizer(
362 query_seq_len=params['query_seq_len'],
363 candidate_seq_len=params['candidate_seq_len'],
364 num_candidates=params['num_candidates'],
365 max_masks=params['max_masks'],
366 tokenizer=tokenizer,
367 separate_candidate_segments=params['separate_candidate_segments'])
368
369 dataset = get_documents_dataset()
370
371 def featurize(doc_dict):
372 return featurizer.featurize_document_tf(doc_dict['title_token_ids'],
373 doc_dict['body_token_ids'])
374
375 dataset = dataset.map(
376 featurize, num_parallel_calls=tf.data.experimental.AUTOTUNE)
377
378 # Add a document index variable.
379 dataset = dataset.enumerate()
380
381 def _enumerate_to_dict(result_idx, tensor_dict):
382 return dict(tensor_dict, result_idx=result_idx)
383
384 dataset = dataset.map(
385 _enumerate_to_dict, num_parallel_calls=tf.data.experimental.AUTOTUNE)
386
387 # Pad the end of the dataset with one full extra batch.
388 # This ensures that we don't drop the remainder.
389 if total_docs % batch_size != 0:
390 # Pad using the first value of the dataset, repeated batch_size times.
391 pad_vals = dataset.take(1).repeat(batch_size)
392 dataset = dataset.concatenate(pad_vals)
393
394 # Batch the dataset.
395 dataset = dataset.batch(batch_size, drop_remainder=True)
396 dataset = dataset.prefetch(2) # Prefetch for efficiency.
397 return dataset
398
399 def model_fn(features, labels, mode, params):
400 """Constructs the model used by Estimator."""

Callers

nothing calls this directly

Calls 1

get_documents_datasetFunction · 0.85

Tested by

no test coverage detected