| 50 | |
| 51 | class LTXVGemmaTokenizer: |
| 52 | def __init__(self, tokenizer_path: str, max_length: int = 1024): |
| 53 | self.tokenizer = AutoTokenizer.from_pretrained( |
| 54 | tokenizer_path, local_files_only=True, model_max_length=max_length |
| 55 | ) |
| 56 | # Gemma expects left padding for chat-style prompts; for plain text it doesn't matter much. |
| 57 | self.tokenizer.padding_side = "left" |
| 58 | if self.tokenizer.pad_token is None: |
| 59 | self.tokenizer.pad_token = self.tokenizer.eos_token |
| 60 | |
| 61 | self.max_length = max_length |
| 62 | |
| 63 | def tokenize_with_weights(self, text: str, return_word_ids: bool = False): |
| 64 | text = text.strip() |