single(multi) image(s) preprocess, the image(s) will be placed at the top of the conversation
(
images_dict,
conversations,
tokenizer,
transform,
query_nums=64,
slice_config=None,
llm_type=None,
patch_size=14,
batch_vision=False,
max_length=2048,
)
| 322 | |
| 323 | |
| 324 | def preprocess( |
| 325 | images_dict, |
| 326 | conversations, |
| 327 | tokenizer, |
| 328 | transform, |
| 329 | query_nums=64, |
| 330 | slice_config=None, |
| 331 | llm_type=None, |
| 332 | patch_size=14, |
| 333 | batch_vision=False, |
| 334 | max_length=2048, |
| 335 | ): |
| 336 | """ |
| 337 | single(multi) image(s) preprocess, the image(s) will be placed at the top of the conversation |
| 338 | """ |
| 339 | conversations = copy.deepcopy(conversations) |
| 340 | assert len(conversations) > 1, "conversations length must large than 2" |
| 341 | |
| 342 | if slice_config is not None: |
| 343 | assert isinstance(slice_config, Dict) |
| 344 | assert "patch_size" in slice_config |
| 345 | assert "max_slice_nums" in slice_config |
| 346 | assert "scale_resolution" in slice_config |
| 347 | default_image_placeholder = ( |
| 348 | tokenizer.im_start + tokenizer.unk_token * query_nums + tokenizer.im_end |
| 349 | ) |
| 350 | new_schema = False |
| 351 | use_image_id = False |
| 352 | if llm_type=='qwen': |
| 353 | new_schema = True |
| 354 | use_image_id = True |
| 355 | image_placeholder_dict = {} |
| 356 | images = [] |
| 357 | image_id_cnt = 0 |
| 358 | for img_name, image in images_dict.items(): |
| 359 | if slice_config: |
| 360 | source_image, patches, best_grid = slice_image( |
| 361 | image, |
| 362 | slice_config["max_slice_nums"], |
| 363 | slice_config["scale_resolution"], |
| 364 | slice_config["patch_size"], |
| 365 | ) |
| 366 | images.append(source_image) |
| 367 | image_placeholder = default_image_placeholder |
| 368 | if len(patches) > 0: |
| 369 | for i in range(len(patches)): |
| 370 | for j in range(len(patches[0])): |
| 371 | images.append(patches[i][j]) |
| 372 | if use_image_id: |
| 373 | image_placeholder = f'{tokenizer.im_id_start}{image_id_cnt}{tokenizer.im_id_end}' + image_placeholder |
| 374 | image_id_cnt += 1 |
| 375 | image_placeholder += get_grid_placeholder( |
| 376 | tokenizer, best_grid, query_nums, new_schema = new_schema) |
| 377 | image_placeholder_dict[img_name] = image_placeholder |
| 378 | else: |
| 379 | images.append(image) |
| 380 | if use_image_id: |
| 381 | image_placeholder = f'{tokenizer.im_id_start}{image_id_cnt}{tokenizer.im_id_end}' + image_placeholder |
no test coverage detected