Build the full multi-channel input_ids for generation. Returns: input_ids: (S, 33) int64
(
tokenizer: Tokenizer,
text: str,
reference_codes: np.ndarray | None = None,
instruction: str | None = None,
tokens: int | None = None,
quality: str | None = None,
language: str | None = None,
sound_event: str | None = None,
ambient_sound: str | None = None,
)
| 68 | |
| 69 | |
| 70 | def build_generation_prompt( |
| 71 | tokenizer: Tokenizer, |
| 72 | text: str, |
| 73 | reference_codes: np.ndarray | None = None, |
| 74 | instruction: str | None = None, |
| 75 | tokens: int | None = None, |
| 76 | quality: str | None = None, |
| 77 | language: str | None = None, |
| 78 | sound_event: str | None = None, |
| 79 | ambient_sound: str | None = None, |
| 80 | ) -> np.ndarray: |
| 81 | """Build the full multi-channel input_ids for generation. |
| 82 | |
| 83 | Returns: |
| 84 | input_ids: (S, 33) int64 |
| 85 | """ |
| 86 | audio_start_tok = _get_special_token_str(tokenizer, AUDIO_START_TOKEN_ID) |
| 87 | audio_end_tok = _get_special_token_str(tokenizer, AUDIO_END_TOKEN_ID) |
| 88 | gen_slot_tok = _get_special_token_str(tokenizer, AUDIO_ASSISTANT_GEN_SLOT_TOKEN_ID) |
| 89 | delay_slot_tok = _get_special_token_str(tokenizer, AUDIO_ASSISTANT_DELAY_SLOT_TOKEN_ID) |
| 90 | user_slot_tok = _get_special_token_str(tokenizer, AUDIO_USER_SLOT_TOKEN_ID) |
| 91 | |
| 92 | has_ref = reference_codes is not None and reference_codes.shape[0] > 0 |
| 93 | |
| 94 | if has_ref: |
| 95 | ref_str = f"[S1]:\n{AUDIO_PLACEHOLDER}" |
| 96 | else: |
| 97 | ref_str = "None" |
| 98 | |
| 99 | user_content = ( |
| 100 | f"<user_inst>\n" |
| 101 | f"- Reference(s):\n{ref_str}\n" |
| 102 | f"- Instruction:\n{instruction}\n" |
| 103 | f"- Tokens:\n{tokens}\n" |
| 104 | f"- Quality:\n{quality}\n" |
| 105 | f"- Sound Event:\n{sound_event}\n" |
| 106 | f"- Ambient Sound:\n{ambient_sound}\n" |
| 107 | f"- Language:\n{language}\n" |
| 108 | f"- Text:\n{text}\n" |
| 109 | f"</user_inst>" |
| 110 | ) |
| 111 | |
| 112 | ref_lengths = [reference_codes.shape[0]] if has_ref else [] |
| 113 | user_content = _replace_audio_placeholders( |
| 114 | user_content, ref_lengths, n_vq=N_VQ, |
| 115 | gen_slot_token=user_slot_tok, delay_slot_token=user_slot_tok, |
| 116 | audio_start_token=audio_start_tok, audio_end_token=audio_end_tok, |
| 117 | ) |
| 118 | |
| 119 | im_start = _get_special_token_str(tokenizer, IM_START_TOKEN_ID) |
| 120 | im_end = _get_special_token_str(tokenizer, IM_END_TOKEN_ID) |
| 121 | |
| 122 | full_text = f"{im_start}user\n{user_content}{im_end}\n{im_start}assistant\n" |
| 123 | |
| 124 | ref_audio_list = [reference_codes] if has_ref else [] |
| 125 | unified_codes = _get_unified_codes( |
| 126 | tokenizer, full_text, ref_audio_list, |
| 127 | is_user=True, truncation=False, |
no test coverage detected