Class TokenTextEncoder

NLP/EMNLP2019-MAL/src/preprocess/text_encoder.py:743–926

Encoder based on a user-supplied vocabulary (file or list).

class TokenTextEncoder(TextEncoder):
    """Encoder based on a user-supplied vocabulary (file or list)."""

    def __init__(self,
                 vocab_filename,
                 reverse=False,
                 vocab_list=None,
                 replace_oov="UNK",
                 num_reserved_ids=NUM_RESERVED_TOKENS):
        """Initialize from a file or list, one token per line.

        Handling of reserved tokens works as follows:
        - When initializing from a list, we add reserved tokens to the vocab.
        - When initializing from a file, we do not add reserved tokens to the vocab.
        - When saving vocab files, we save reserved tokens to the file.

        Args:
          vocab_filename: If not None, the full filename to read vocab from. If this
             is not None, then vocab_list should be None.
          reverse: Boolean indicating if tokens should be reversed during encoding
             and decoding.
          vocab_list: If not None, a list of elements of the vocabulary. If this is
             not None, then vocab_filename should be None.
          replace_oov: If not None, every out-of-vocabulary token seen when
             encoding will be replaced by this string (which must be in vocab).
          num_reserved_ids: Number of IDs to save for reserved tokens like <EOS>.
        """
        super(TokenTextEncoder, self).__init__(num_reserved_ids=num_reserved_ids)
        self._reverse = reverse
        self._replace_oov = replace_oov
        if vocab_filename:
            self._init_vocab_from_file(vocab_filename)
        else:
            assert vocab_list is not None
            self._init_vocab_from_list(vocab_list)

    @classmethod
    def build_from_corpus(cls, filenames, vocab_size):
        """

        :param filenames:
        :param vocab_size:
        :return:
        """

        def create_dictionary(names, lim=0):
            """
            :param name:
            :param lim:
            :return:
            """
            global_counter = collections.Counter()
            for name in names:
                fd = open(name)
                for line in fd:
                    words = line.strip().split()
                    words = filter(lambda x: x != "-1", words)
                    global_counter.update(words)