Method encode

src/lmflow/models/hf_decoder_model.py:210–242

Perform encoding process of the tokenizer. Parameters ------------ inputs : str or list. The text sequence. args : Optional. Positional arguments. kwargs : Optional. Keyword arguments. Returns --

(self, input: Union[str, list[str]], *args, **kwargs)

        return tokenized_datasets

    def encode(self, input: Union[str, list[str]], *args, **kwargs) -> Union[list[int], list[list[int]]]:
        """
        Perform encoding process of the tokenizer.

        Parameters
        ------------
        inputs : str or list.
            The text sequence.

        args : Optional.
            Positional arguments.

        kwargs : Optional.
            Keyword arguments.

        Returns
        ------------
        outputs :
            if string input,return the tokenized inputs.
            "Hello,world!"-> [101, 7592, 1010, 2088, 102]
            if batch input,return {input_ids,attention_mask,token_type_ids}
            ["Hello,world!","Hello!"] ->
            {
                'input_ids': tensor([[  101,  7592,  1010,  2088,   102],...),
                'attention_mask': tensor([[1, 1, 1, 1, 1],[0,0,1,1,1]])
            }
        """
        if isinstance(input, list):
            return self.tokenizer(text=input, **kwargs)  # batch encode,will automatically do left padding
        elif isinstance(input, str):
            return self.tokenizer.encode(text=input, **kwargs)
        else:
            raise NotImplementedError(f'type "{type(input)}" cannot be encoded')

    def decode(self, input, **kwargs) -> Union[str, list[str]]:
        """