Function process_inputs

generation_utils.py:272–314

(
    tokenizer,
    spt,
    prompt,
    text,
    device,
    silence_duration,
    audio_data=None,
    max_channels=8,
    pad_token=1024,
)

def process_inputs(
    tokenizer,
    spt,
    prompt,
    text,
    device,
    silence_duration,
    audio_data=None,
    max_channels=8,
    pad_token=1024,
):
    seq = f"<|begin_of_style|>{prompt}<|end_of_style|>\n<|begin_of_text|>{text}<|end_of_text|>\n<|begin_of_speech|>"
    inputs1 = np.array(tokenizer.encode(seq))
    input_ids = np.full((inputs1.shape[0], max_channels), pad_token)
    input_ids[:, 0] = inputs1

    if audio_data is not None:
        try:
            # audio_data should now be a processed audio tensor
            wav = audio_data

            # Add fixed 5-second silence at the end of audio (using 16k sample rate)
            silence_samples = int(silence_duration * 16000)
            silence = torch.zeros(wav.shape[0], silence_samples)
            wav = torch.cat([wav, silence], dim=1)

            with torch.no_grad():
                # Use SPT encoding
                encode_result = spt.encode([wav.squeeze().to(device)])
                audio_token = (
                    encode_result["codes_list"][0].permute(1, 0).cpu().numpy()
                )  # Adjust dimension order

            # similar to DAC encoding adjustment
            audio_token[:, 0] = (
                audio_token[:, 0] + 151665
            )  # Keep this line if offset is needed, otherwise delete
            input_ids = np.concatenate([input_ids, audio_token])
        except Exception as e:
            print(f"Error processing audio data: {e}")
            raise

    return input_ids


def shifting_inputs(input_ids, tokenizer, pad_token=1024, max_channels=8):