Method add

hanlp/common/vocab.py:75–93

Tries to add a token into a vocab and returns its id. If it has already been there, its id will be returned and the vocab won't be updated. If the vocab is locked, an assertion failure will occur. Args: token: A new or existing token. Returns: Its a

(self, token: str)

            return False

    def add(self, token: str) -> int:
        """ Tries to add a token into a vocab and returns its id. If it has already been there, its id will be returned
        and the vocab won't be updated. If the vocab is locked, an assertion failure will occur.

        Args:
            token: A new or existing token.

        Returns:
            Its associated id.

        """
        assert self.mutable, 'It is not allowed to call add on an immutable Vocab'
        assert isinstance(token, str), f'Token type must be str but got {type(token)} from {token}'
        assert token is not None, 'Token must not be None'
        idx = self.token_to_idx.get(token, None)
        if idx is None:
            idx = len(self.token_to_idx)
            self.token_to_idx[token] = idx
        return idx

    def update(self, tokens: Iterable[str]) -> None:
        """Update the vocab with these tokens by adding them to vocab one by one.