Method encode

scripts/verify_tokenizer.py:120–137

(self, text, context_length=32)

        return result

    def encode(self, text, context_length=32):
        # Clean + lowercase
        text = re.sub(r"\s+", " ", text).strip().lower()

        bpe_tokens = []
        for token in re.findall(self.pat, text):
            encoded = "".join(self.byte_encoder[b] for b in token.encode("utf-8"))
            bpe_tokens.extend(
                self.encoder[bpe_token] for bpe_token in self.bpe(encoded).split(" ")
            )

        # Build result: [SOT, tokens..., EOT, 0, 0, ...]
        result = [self.sot_token] + bpe_tokens + [self.eot_token]
        if len(result) > context_length:
            result = result[:context_length]
            result[-1] = self.eot_token
        result += [0] * (context_length - len(result))
        return result


TEST_STRINGS = [