Data for visual input that has been converted into tokens
| 13 | |
| 14 | @dataclasses.dataclass |
| 15 | class TokenizedVisionData: |
| 16 | """Data for visual input that has been converted into tokens""" |
| 17 | |
| 18 | tokens: np.ndarray |
| 19 | """Token ids to use in the LLM""" |
| 20 | |
| 21 | images: np.ndarray |
| 22 | """Images in [n_images, n_patches, patch_size] format""" |
| 23 | |
| 24 | image_masks: Optional[np.ndarray] = None |
| 25 | """Images in [n_images, n_patches] format""" |
| 26 | |
| 27 | token_pooling: Optional[np.ndarray] = None |
| 28 | """[n_visual_tokens, pooling_dim] array of of how to pool ViT patches for patch_id tokens""" |
| 29 | |
| 30 | low_res_token_pooling: Optional[np.ndarray] = None |
| 31 | """[n_visual_tokens, pooling_dim] array of of how to pool ViT patches for low_res tokens""" |
| 32 | |
| 33 | position_ids: Optional[np.ndarray] = None |
| 34 | """Position id for each token if not sequential""" |
| 35 | |
| 36 | cum_token_pooling_bounds: Optional[Dict[str, np.ndarray]] = None |
| 37 | "cumulative token pooling boundaries used when training with context parallelism on images and videos" |
| 38 | |
| 39 | cum_image_bounds: Optional[int] = None |
| 40 | """boundaries of crops in an input""" |
| 41 | |
| 42 | other_data: Optional[Dict[str, np.ndarray]] = None |
| 43 | """Any extra data that needs to be passed to the model""" |
| 44 | |
| 45 | token_mapping: Optional[np.ndarray] = None |
| 46 | """Map patch coordinates -> patch ids""" |
| 47 | |
| 48 | |
| 49 | @dataclasses.dataclass |
no outgoing calls
no test coverage detected