Constructs the dataset fed to Estimator.
(params)
| 349 | } |
| 350 | |
| 351 | def input_fn(params): |
| 352 | """Constructs the dataset fed to Estimator.""" |
| 353 | # We cannot access self._featurizer via closure, because this function is |
| 354 | # passed to another device. Hence, we need to reconstruct the featurizer |
| 355 | # from its hyerparameters (passed through `params`). |
| 356 | |
| 357 | tokenizer = featurization.Tokenizer( |
| 358 | vocab_path=params['vocab_path'], |
| 359 | do_lower_case=params['do_lower_case']) |
| 360 | |
| 361 | featurizer = featurization.Featurizer( |
| 362 | query_seq_len=params['query_seq_len'], |
| 363 | candidate_seq_len=params['candidate_seq_len'], |
| 364 | num_candidates=params['num_candidates'], |
| 365 | max_masks=params['max_masks'], |
| 366 | tokenizer=tokenizer, |
| 367 | separate_candidate_segments=params['separate_candidate_segments']) |
| 368 | |
| 369 | dataset = get_documents_dataset() |
| 370 | |
| 371 | def featurize(doc_dict): |
| 372 | return featurizer.featurize_document_tf(doc_dict['title_token_ids'], |
| 373 | doc_dict['body_token_ids']) |
| 374 | |
| 375 | dataset = dataset.map( |
| 376 | featurize, num_parallel_calls=tf.data.experimental.AUTOTUNE) |
| 377 | |
| 378 | # Add a document index variable. |
| 379 | dataset = dataset.enumerate() |
| 380 | |
| 381 | def _enumerate_to_dict(result_idx, tensor_dict): |
| 382 | return dict(tensor_dict, result_idx=result_idx) |
| 383 | |
| 384 | dataset = dataset.map( |
| 385 | _enumerate_to_dict, num_parallel_calls=tf.data.experimental.AUTOTUNE) |
| 386 | |
| 387 | # Pad the end of the dataset with one full extra batch. |
| 388 | # This ensures that we don't drop the remainder. |
| 389 | if total_docs % batch_size != 0: |
| 390 | # Pad using the first value of the dataset, repeated batch_size times. |
| 391 | pad_vals = dataset.take(1).repeat(batch_size) |
| 392 | dataset = dataset.concatenate(pad_vals) |
| 393 | |
| 394 | # Batch the dataset. |
| 395 | dataset = dataset.batch(batch_size, drop_remainder=True) |
| 396 | dataset = dataset.prefetch(2) # Prefetch for efficiency. |
| 397 | return dataset |
| 398 | |
| 399 | def model_fn(features, labels, mode, params): |
| 400 | """Constructs the model used by Estimator.""" |
nothing calls this directly
no test coverage detected