(self, i)
| 66 | return img |
| 67 | |
| 68 | def __getitem__(self, i) -> Dict[str, torch.Tensor]: |
| 69 | try: |
| 70 | if isinstance(self.raw_data[i]["image"], str): |
| 71 | # resize the image |
| 72 | images_dict = { "<image>" : self.__resize__(Image.open(self.raw_data[i]["image"]).convert("RGB"))} |
| 73 | elif isinstance(self.raw_data[i]["image"], Dict): |
| 74 | ### for multi-images input, the template for every image is <image_xx>, such as <image_00>, <image_01> |
| 75 | images_dict = {img_name : self.__resize__(Image.open(img_path).convert("RGB")) for img_name, img_path in self.raw_data[i]["image"].items()} |
| 76 | |
| 77 | ret = preprocess( |
| 78 | images_dict, |
| 79 | self.raw_data[i]["conversations"], |
| 80 | self.tokenizer, |
| 81 | self.transform, |
| 82 | query_nums=self.query_nums, |
| 83 | slice_config=self.slice_config, |
| 84 | llm_type=self.llm_type, |
| 85 | patch_size=self.patch_size, |
| 86 | batch_vision=self.batch_vision, |
| 87 | max_length=self.max_length |
| 88 | ) |
| 89 | ret = dict( |
| 90 | input_ids=ret["input_ids"], |
| 91 | position_ids=ret["position_ids"], |
| 92 | labels=ret["target"], |
| 93 | attention_mask=torch.ones_like(ret["input_ids"], dtype=torch.bool), |
| 94 | pixel_values=ret["pixel_values"], |
| 95 | tgt_sizes=ret["tgt_sizes"], |
| 96 | image_bound=ret["image_bound"], |
| 97 | ) |
| 98 | except Exception as e: |
| 99 | logger.error(f"data fetch error: {e}") |
| 100 | return self.__getitem__(random.randint(0, len(self))) |
| 101 | return ret |
| 102 | |
| 103 | |
| 104 | def data_collator(examples, padding_value=0, max_length=2048): |
nothing calls this directly
no test coverage detected