preprocess the data Args: model_dir (str): model path mode: preprocessor mode (model mode)
(self,
model_dir: str,
mode=ModeKeys.INFERENCE,
*args,
**kwargs)
| 173 | class CLIPPreprocessor(Preprocessor): |
| 174 | |
| 175 | def __init__(self, |
| 176 | model_dir: str, |
| 177 | mode=ModeKeys.INFERENCE, |
| 178 | *args, |
| 179 | **kwargs): |
| 180 | """preprocess the data |
| 181 | |
| 182 | Args: |
| 183 | model_dir (str): model path |
| 184 | mode: preprocessor mode (model mode) |
| 185 | """ |
| 186 | super().__init__(*args, **kwargs) |
| 187 | model_dir = model_dir if osp.exists(model_dir) else snapshot_download( |
| 188 | model_dir, user_agent={Invoke.KEY: Invoke.PREPROCESSOR}) |
| 189 | self.mode = mode |
| 190 | # text tokenizer |
| 191 | from modelscope.models.multi_modal.clip.bert_tokenizer import FullTokenizer |
| 192 | if 'tokenizer' in kwargs and isinstance(kwargs['tokenizer'], |
| 193 | FullTokenizer): |
| 194 | self.tokenizer = kwargs['tokenizer'] |
| 195 | else: |
| 196 | vocab_file = f'{model_dir}/{ModelFile.VOCAB_FILE}' |
| 197 | self.tokenizer = FullTokenizer(vocab_file=vocab_file) |
| 198 | # image preprocessor |
| 199 | if 'resolution' in kwargs and isinstance(kwargs['resolution'], int): |
| 200 | self.image_resolution = kwargs['resolution'] |
| 201 | else: |
| 202 | self.image_resolution = json.load( |
| 203 | open( |
| 204 | '{}/vision_model_config.json'.format(model_dir), |
| 205 | encoding='utf-8'))['image_resolution'] |
| 206 | self.img_preprocess = self._build_image_transform() |
| 207 | # key mapping |
| 208 | # specify the input keys, compatible with training and inference whose key names may be different |
| 209 | self.input_keys = {'img': 'img', 'text': 'text'} |
| 210 | |
| 211 | def _build_image_transform(self): |
| 212 |
nothing calls this directly
no test coverage detected