Method encode

conftest.py:30–66

(
        self,
        string: str,
        device: Optional[torch.device] = None,
        bos: bool = False,
        eos: bool = False,
        max_length=-1,
    )

    eos_id = 2

    def encode(
        self,
        string: str,
        device: Optional[torch.device] = None,
        bos: bool = False,
        eos: bool = False,
        max_length=-1,
    ) -> torch.Tensor:
        tokens = [ord(c) for c in string]
        if bos:
            bos_id = self.bos_id
            if bos_id is None:
                raise NotImplementedError(
                    'Attempted to add bos, but this tokenizer does not defined a bos token.'
                )
            tokens = [bos_id] + tokens

        if max_length > 0:
            if eos:
                if self.eos_id is None:
                    raise NotImplementedError(
                        'Attempted to add eos, but this tokenizer does not defined an eos token'
                    )
                if len(tokens) >= max_length:
                    tokens = tokens[:max_length]
                    tokens[-1] = self.eos_id
                else:
                    tokens = tokens + [self.eos_id]
        else:
            if eos:
                if self.eos_id is None:
                    raise NotImplementedError(
                        'Attempted to add eos, but this tokenizer does not defined an eos token'
                    )
                tokens = tokens + [self.eos_id]

        return torch.tensor(tokens)

    def decode(self, tokens: torch.Tensor) -> str:
        decoded = ''