Class BertInputs

language/conpono/reconstruct/preprocess.py:431–474

A namedtuple that groups together all BERT inputs.

class BertInputs(collections.namedtuple("BertInputs", BERT_INPUTS_LIST)):
  """A namedtuple that groups together all BERT inputs."""

  def pretty_print(self, tokenizer, show_padding=True):
    """Pretty-prints BERT inputs for human inspection.

    NOTE: this method assumes that every attribute of BertInputs is a Numpy
    array, not a Tensor. Also, it assumes that the attributes are NOT batched:
    the first dimension of each array should not be a batch dimension.

    Args:
      tokenizer: an instance of language.bert.FullTokenizer.
      show_padding: boolean indicating whether to print pad tokens.

    Returns:
      a pretty-printed unicode string.
    """
    # Convert IDs back to tokens.
    tokens = tokenizer.convert_ids_to_tokens(self.input_ids)
    target_tokens = tokenizer.convert_ids_to_tokens(self.masked_lm_ids)

    # Annotate masked tokens with prediction targets.
    for target, pos, weight in safe_zip(target_tokens, self.masked_lm_positions,
                                        self.masked_lm_weights):
      if weight != 0:
        if weight != 1:
          raise ValueError("Weight must be either 0 or 1.")
        tokens[pos] += u"({})".format(target)

    # Sequences to display.
    seqs = [["     Tokens:"] + tokens,
            ["Segment IDs:"] + list(self.segment_ids),
            [" Input mask:"] + list(self.input_mask)]

    if not show_padding:
      max_seq_length = sum(self.input_mask)
      for seq in seqs:
        # +1 here, because we added an extra "Label" in front of each sequence.
        del seq[max_seq_length + 1:]

    # Format seqs into strings, so that tokens line up vertically.
    aligned_seqs = whitespace_align_seqs(seqs)

    return u"\n".join(u"  ".join(seq) for seq in aligned_seqs)


def bert_preprocess(text_segments,