Embeds a Dataset of documents using Estimator. Args: get_documents_dataset: a function that returns a TF Dataset, where each element is a dict with attributes described below. total_docs: total number of documents in the Dataset. batch_size: number of documents to embe
(self, get_documents_dataset, total_docs, batch_size)
| 309 | self._log_interval = 10 # When embedding, log every 10 seconds. |
| 310 | |
| 311 | def embed(self, get_documents_dataset, total_docs, batch_size): |
| 312 | """Embeds a Dataset of documents using Estimator. |
| 313 | |
| 314 | Args: |
| 315 | get_documents_dataset: a function that returns a TF Dataset, where each |
| 316 | element is a dict with attributes described below. |
| 317 | total_docs: total number of documents in the Dataset. |
| 318 | batch_size: number of documents to embed in each batch. Each element in |
| 319 | the Dataset returned by get_documents_dataset should be a dict with the |
| 320 | attributes described below. |
| 321 | |
| 322 | get_documents_dataset should return a Dataset over dicts, each containing at |
| 323 | least the following attributes: |
| 324 | - title_token_ids: a 1-D int Tensor. |
| 325 | - body_token_ids: a 1-D int Tensor. |
| 326 | |
| 327 | Yields: |
| 328 | a [embed_dim] Numpy array, one for each document. |
| 329 | """ |
| 330 | if total_docs < 1: |
| 331 | raise ValueError('Must embed at least 1 document.') |
| 332 | |
| 333 | # These hyperparams are passed to Estimator. |
| 334 | params = { |
| 335 | 'vocab_path': |
| 336 | self._featurizer.tokenizer.vocab_path, |
| 337 | 'do_lower_case': |
| 338 | self._featurizer.tokenizer.do_lower_case, |
| 339 | 'query_seq_len': |
| 340 | self._featurizer.query_seq_len, |
| 341 | 'candidate_seq_len': |
| 342 | self._featurizer.candidate_seq_len, |
| 343 | 'num_candidates': |
| 344 | self._featurizer.num_candidates, |
| 345 | 'max_masks': |
| 346 | self._featurizer.max_masks, |
| 347 | 'separate_candidate_segments': |
| 348 | self._featurizer.separate_candidate_segments, |
| 349 | } |
| 350 | |
| 351 | def input_fn(params): |
| 352 | """Constructs the dataset fed to Estimator.""" |
| 353 | # We cannot access self._featurizer via closure, because this function is |
| 354 | # passed to another device. Hence, we need to reconstruct the featurizer |
| 355 | # from its hyerparameters (passed through `params`). |
| 356 | |
| 357 | tokenizer = featurization.Tokenizer( |
| 358 | vocab_path=params['vocab_path'], |
| 359 | do_lower_case=params['do_lower_case']) |
| 360 | |
| 361 | featurizer = featurization.Featurizer( |
| 362 | query_seq_len=params['query_seq_len'], |
| 363 | candidate_seq_len=params['candidate_seq_len'], |
| 364 | num_candidates=params['num_candidates'], |
| 365 | max_masks=params['max_masks'], |
| 366 | tokenizer=tokenizer, |
| 367 | separate_candidate_segments=params['separate_candidate_segments']) |
| 368 |
no test coverage detected