(cfg, model_path)
| 50 | |
| 51 | |
| 52 | def _sync_ruler_tokenizer_model(cfg, model_path): |
| 53 | tokenizer = os.environ.get('TOKENIZER_MODEL', model_path) |
| 54 | if not getattr(cfg, 'datasets', None): |
| 55 | return |
| 56 | for dataset in cfg.datasets: |
| 57 | if isinstance(dataset, dict) and 'tokenizer_model' in dataset: |
| 58 | dataset['tokenizer_model'] = tokenizer |
| 59 | |
| 60 | |
| 61 | def _is_fp8_case(case_name: str) -> bool: |