(self, text)
| 552 | return word |
| 553 | |
| 554 | def encode(self, text): |
| 555 | bpe_tokens = [] |
| 556 | text = whitespace_clean(text.strip()).lower() |
| 557 | for token in re.findall(self.pat, text): |
| 558 | token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) |
| 559 | bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' ')) |
| 560 | # Truncation, keeping two slots for start and end tokens. |
| 561 | if len(bpe_tokens) > 75: |
| 562 | bpe_tokens = bpe_tokens[:75] |
| 563 | return [49406] + bpe_tokens + [49407] * (77 - len(bpe_tokens) - 1) |
| 564 | |
| 565 | class StableDiffusion: |
| 566 | def __init__(self): |
nothing calls this directly
no test coverage detected