Function build_processor

moss_tts_delay/finetuning/sft.py:260–283

(
    model_path: str,
    codec_path: str,
    need_audio_tokenizer: bool,
    audio_tokenizer_device: Optional[str],
    default_audio_tokenizer_device: str,
)

def build_processor(
    model_path: str,
    codec_path: str,
    need_audio_tokenizer: bool,
    audio_tokenizer_device: Optional[str],
    default_audio_tokenizer_device: str,
):
    config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
    tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
    processor = MossTTSDelayProcessor(
        tokenizer=tokenizer,
        audio_tokenizer=None,
        model_config=config,
    )

    if need_audio_tokenizer:
        processor = MossTTSDelayProcessor.from_pretrained(
            model_path,
            codec_path=codec_path,
        )
        device = audio_tokenizer_device or default_audio_tokenizer_device
        processor.audio_tokenizer = processor.audio_tokenizer.to(device)

    return processor


@contextmanager