| 304 | class _MinimalTokenizer: |
| 305 | """Fallback tokenizer for Parakeet-TDT when HF tokenizer load fails.""" |
| 306 | def __init__(self, name_or_path, config_obj=None): |
| 307 | self.name_or_path = name_or_path |
| 308 | self.model_max_length = 131072 |
| 309 | self.pad_token_id = 0 |
| 310 | self.eos_token_id = 0 |
| 311 | |
| 312 | try: |
| 313 | pad_id = config_obj.get('pad_token_id', 0) if isinstance(config_obj, dict) else 0 |
| 314 | self.pad_token_id = int(pad_id) if pad_id is not None else 0 |
| 315 | except Exception: |
| 316 | self.pad_token_id = 0 |
| 317 | |
| 318 | try: |
| 319 | decoding = config_obj.get('decoding', {}) if isinstance(config_obj, dict) else {} |
| 320 | blank_id = decoding.get('blank_id', None) if isinstance(decoding, dict) else None |
| 321 | if blank_id is not None: |
| 322 | self.eos_token_id = int(blank_id) |
| 323 | else: |
| 324 | self.eos_token_id = self.pad_token_id |
| 325 | except Exception: |
| 326 | self.eos_token_id = self.pad_token_id |
| 327 | |
| 328 | def _load_raw_hf_state_dict(repo_id, cast_to_bf16=True): |
| 329 | from safetensors.torch import load_file as load_safetensors_file |