| 33 | logger = logging.get_logger(__name__) |
| 34 | |
| 35 | class SpatialVLAProcessor(ProcessorMixin): |
| 36 | attributes = ["image_processor", "tokenizer"] |
| 37 | valid_kwargs = ["chat_template"] |
| 38 | image_processor_class = "SiglipImageProcessor" |
| 39 | tokenizer_class = ("GemmaTokenizer", "GemmaTokenizerFast") |
| 40 | |
| 41 | def __init__( |
| 42 | self, |
| 43 | image_processor=None, |
| 44 | tokenizer=None, |
| 45 | chat_template=None, |
| 46 | statistics: Optional[dict] = None, |
| 47 | bin_policy=None, |
| 48 | intrinsic_config=None, |
| 49 | action_config=None, |
| 50 | num_obs_steps=1, |
| 51 | obs_delta=1, |
| 52 | action_chunk_size=1, |
| 53 | min_sigma=0.0, |
| 54 | **kwargs, |
| 55 | ): |
| 56 | if image_processor is None: |
| 57 | raise ValueError("You need to specify an `image_processor`.") |
| 58 | if tokenizer is None: |
| 59 | raise ValueError("You need to specify a `tokenizer`.") |
| 60 | if not hasattr(image_processor, "image_seq_length"): |
| 61 | raise ValueError("Image processor is missing an `image_seq_length` attribute.") |
| 62 | |
| 63 | self.image_seq_length = image_processor.image_seq_length |
| 64 | |
| 65 | if not hasattr(tokenizer, "image_token"): |
| 66 | image_token = AddedToken(IMAGE_TOKEN, normalized=False, special=True) |
| 67 | tokens_to_add = {"additional_special_tokens": [image_token]} |
| 68 | tokenizer.add_special_tokens(tokens_to_add) |
| 69 | self.image_token_id = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN) |
| 70 | else: |
| 71 | self.image_token_id = tokenizer.image_token_id |
| 72 | |
| 73 | tokenizer.add_tokens(EXTRA_TOKENS) |
| 74 | tokenizer.add_bos_token = False |
| 75 | tokenizer.add_eos_token = False |
| 76 | |
| 77 | super().__init__(image_processor, tokenizer, chat_template=chat_template) |
| 78 | |
| 79 | # action tokenizer |
| 80 | self.statistics = statistics if statistics else {} |
| 81 | self.bin_policy = bin_policy |
| 82 | self.min_sigma = min_sigma |
| 83 | self.intrinsic_config = intrinsic_config |
| 84 | self.action_config = action_config |
| 85 | self.num_obs_steps = num_obs_steps |
| 86 | self.obs_delta = obs_delta |
| 87 | self.action_chunk_size = action_chunk_size |
| 88 | self.dataset_intrinsics = {} |
| 89 | height, width = image_processor.size["height"], image_processor.size["width"] |
| 90 | |
| 91 | # scale intrinsic matrix |
| 92 | for k, v in intrinsic_config.items(): |