Method encode

lit_gpt/tokenizer.py:102–146

(
        self,
        string: str,
        device: Optional[torch.device] = None,
        bos: bool = False,
        eos: bool = False,
        max_length=-1,
    )

        return id_

    def encode(
        self,
        string: str,
        device: Optional[torch.device] = None,
        bos: bool = False,
        eos: bool = False,
        max_length=-1,
    ) -> torch.Tensor:
        if self.backend == HUGGINGFACE:
            # add_special_tokens=False as we want to manually handle it later.
            tokens = self.processor.encode(string, add_special_tokens=False).ids
        elif self.backend == SENTENCEPIECE:
            raise RuntimeError
            tokens = self.processor.encode(string)
        else:
            raise RuntimeError

        if bos:
            bos_id = self.bos_id
            if bos_id is None:
                raise NotImplementedError(
                    'Attempted to add bos, but this tokenizer does not defined a bos token.'
                )
            tokens = [bos_id] + tokens

        if max_length > 0:
            if eos:
                if self.eos_id is None:
                    raise NotImplementedError(
                        'Attempted to add eos, but this tokenizer does not defined an eos token'
                    )
                if len(tokens) >= max_length:
                    tokens = tokens[:max_length]
                    tokens[-1] = self.eos_id
                else:
                    tokens = tokens + [self.eos_id]
        else:
            if eos:
                if self.eos_id is None:
                    raise NotImplementedError(
                        'Attempted to add eos, but this tokenizer does not defined an eos token'
                    )
                tokens = tokens + [self.eos_id]

        return torch.tensor(tokens, dtype=torch.int, device=device)

    def decode(
        self,