Bert Text encoder.
| 421 | |
| 422 | |
| 423 | class BertTextEncoder(FeverTextEncoder): |
| 424 | """Bert Text encoder.""" |
| 425 | |
| 426 | def __init__( |
| 427 | self, *, |
| 428 | tokenizer, max_seq_length, |
| 429 | include_title, include_sentence_id): |
| 430 | super().__init__() |
| 431 | self._tokenizer = tokenizer |
| 432 | self._max_seq_length = max_seq_length |
| 433 | self._include_title = include_title |
| 434 | self._include_sentence_id = include_sentence_id |
| 435 | self._cls_id = self._tokenizer.vocab['[CLS]'] |
| 436 | self._sep_id = self._tokenizer.vocab['[SEP]'] |
| 437 | self._pad_id = self._tokenizer.vocab['[PAD]'] |
| 438 | |
| 439 | def encode_example(self, *, claim, evidence, claim_label, evidence_label, |
| 440 | metadata, wikipedia_url, sentence_id): |
| 441 | claim_tokens, claim_word_ids, claim_mask, claim_segment_ids = self.encode( |
| 442 | claim) |
| 443 | evidence = create_evidence_input( |
| 444 | wikipedia_url if self._include_title else None, |
| 445 | sentence_id if self._include_sentence_id else None, |
| 446 | evidence |
| 447 | ) |
| 448 | evidence_tokens, evidence_word_ids, evidence_mask, evidence_segment_ids = self.encode( |
| 449 | evidence) |
| 450 | return { |
| 451 | 'claim_text_tokens': claim_tokens, |
| 452 | 'claim_text_word_ids': claim_word_ids, |
| 453 | 'claim_text_mask': claim_mask, |
| 454 | 'claim_text_segment_ids': claim_segment_ids, |
| 455 | 'evidence_text_tokens': evidence_tokens, |
| 456 | 'evidence_text_word_ids': evidence_word_ids, |
| 457 | 'evidence_text_mask': evidence_mask, |
| 458 | 'evidence_text_segment_ids': evidence_segment_ids, |
| 459 | 'claim_label': claim_label, |
| 460 | 'evidence_label': evidence_label, |
| 461 | 'metadata': metadata |
| 462 | } |
| 463 | |
| 464 | def build_encoder_fn(self): |
| 465 | |
| 466 | def encode(claim, evidence, claim_label, evidence_label, metadata, |
| 467 | wikipedia_url, sentence_id): |
| 468 | """Encode the example using the input TF example. |
| 469 | |
| 470 | Args: |
| 471 | claim: Claim text |
| 472 | evidence: Evidence text |
| 473 | claim_label: Claim verification label |
| 474 | evidence_label: Label if is relevant evidence/claim |
| 475 | metadata: Metadata about example |
| 476 | wikipedia_url: wikipedia_url of example, or None |
| 477 | sentence_id: sentence_id of the example, or None |
| 478 | |
| 479 | Returns: |
| 480 | Input fields of ClassicEmbedder model |
no outgoing calls
no test coverage detected
searching dependent graphs…