for single image multi-turn conversation conversation: [{'role': 'user', 'content': 'Describe this image'}, {'role': 'assistant', 'content': 'This is a cat.'}]
(conversation, tokenizer, llm_type=None, new_schema=False, max_length=2048)
| 140 | |
| 141 | |
| 142 | def conversation_to_ids(conversation, tokenizer, llm_type=None, new_schema=False, max_length=2048): |
| 143 | """ |
| 144 | for single image multi-turn conversation |
| 145 | conversation: [{'role': 'user', 'content': 'Describe this image'}, |
| 146 | {'role': 'assistant', 'content': 'This is a cat.'}] |
| 147 | """ |
| 148 | if llm_type == "llama3": |
| 149 | input_ids, context, raw_msg = conversation_to_ids_llama3( |
| 150 | conversation, tokenizer |
| 151 | ) |
| 152 | elif llm_type == "qwen": |
| 153 | input_ids, context, raw_msg = conversation_to_ids_qwen2( |
| 154 | conversation, tokenizer |
| 155 | ) |
| 156 | else: |
| 157 | input_ids, context, raw_msg = conversation_to_ids_minicpm( |
| 158 | conversation, tokenizer |
| 159 | ) |
| 160 | |
| 161 | ids = torch.from_numpy(np.hstack(input_ids, dtype=np.int32)) |
| 162 | context = torch.from_numpy(np.hstack(context, dtype=np.int8)) |
| 163 | if input_ids.shape[-1] > max_length: |
| 164 | ids =ids[:max_length] |
| 165 | context = context[:max_length] |
| 166 | logger.warning(f"The input length ({input_ids.shape[-1]}) exceeds the model's maximum length ({max_length}), so it has been truncated") |
| 167 | |
| 168 | if torch.all(context): |
| 169 | logger.error("No tokens available to compute loss.") |
| 170 | raise Exception("No tokens available to compute loss.") |
| 171 | |
| 172 | # build target |
| 173 | target = torch.full_like(ids, -100, dtype=torch.int32) |
| 174 | |
| 175 | for i in range(1, len(ids)): |
| 176 | if context[i] == 0: |
| 177 | target[i - 1] = ids[i] |
| 178 | |
| 179 | # build image bound |
| 180 | if new_schema: |
| 181 | start_cond = (ids == tokenizer.im_start_id) | (ids == tokenizer.slice_start_id) |
| 182 | end_cond = (ids == tokenizer.im_end_id) | (ids == tokenizer.slice_end_id) |
| 183 | image_start_tokens = torch.where(start_cond)[0] |
| 184 | image_start_tokens += 1 |
| 185 | image_end_tokens = torch.where(end_cond)[0] |
| 186 | else: |
| 187 | image_start_tokens = torch.where(ids == tokenizer.im_start_id)[0] |
| 188 | image_start_tokens += 1 |
| 189 | image_end_tokens = torch.where(ids == tokenizer.im_end_id)[0] |
| 190 | if len(image_start_tokens) != len(image_end_tokens): |
| 191 | logger.error("image start token != image end tokens") |
| 192 | raise Exception("image start token != image end tokens") |
| 193 | |
| 194 | if len(image_start_tokens) > 0: |
| 195 | image_bound = torch.hstack( |
| 196 | [image_start_tokens.unsqueeze(-1), image_end_tokens.unsqueeze(-1)] |
| 197 | ) |
| 198 | else: |
| 199 | image_bound = [] |
no test coverage detected