Method encode

language/serene/preprocessing.py:466–496

Encode the example using the input TF example. Args: claim: Claim text evidence: Evidence text claim_label: Claim verification label evidence_label: Label if is relevant evidence/claim metadata: Metadata about example wikipedia_url: wikipedia_ur

(claim, evidence, claim_label, evidence_label, metadata,
               wikipedia_url, sentence_id)

  def build_encoder_fn(self):

    def encode(claim, evidence, claim_label, evidence_label, metadata,
               wikipedia_url, sentence_id):
      """Encode the example using the input TF example.

      Args:
        claim: Claim text
        evidence: Evidence text
        claim_label: Claim verification label
        evidence_label: Label if is relevant evidence/claim
        metadata: Metadata about example
        wikipedia_url: wikipedia_url of example, or None
        sentence_id: sentence_id of the example, or None

      Returns:
        Input fields of ClassicEmbedder model
      """
      claim_tokens, claim_word_ids, claim_mask, claim_segment_ids = self.encode(
          claim.numpy())
      evidence = create_evidence_input(
          wikipedia_url.numpy().decode('utf8')
          if self._include_title else None,
          sentence_id.numpy().decode('utf8')
          if self._include_sentence_id else None,
          evidence.numpy().decode('utf8')
      )

      evidence_tokens, evidence_word_ids, evidence_mask, evidence_segment_ids = self.encode(
          evidence)
      return (claim_tokens, claim_word_ids, claim_mask, claim_segment_ids,
              evidence_tokens, evidence_word_ids, evidence_mask,
              evidence_segment_ids, claim_label, evidence_label, metadata)

    def encode_map_fn(example):
      """Read example from tfds and constructed named tensor inputs."""