| 221 | self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) |
| 222 | |
| 223 | def chat(self, input): |
| 224 | image = None |
| 225 | if "image" in input and len(input["image"]) > 10: # legacy API |
| 226 | try: |
| 227 | image = Image.open(io.BytesIO(base64.b64decode(input['image']))).convert('RGB') |
| 228 | except Exception as e: |
| 229 | return "Image decode error" |
| 230 | |
| 231 | msgs = json.loads(input["question"]) |
| 232 | |
| 233 | for msg in msgs: |
| 234 | contents = msg.pop('content') # support str or List[Dict] |
| 235 | if isinstance(contents, str): |
| 236 | contents = [contents] |
| 237 | |
| 238 | new_cnts = [] |
| 239 | for c in contents: |
| 240 | if isinstance(c, dict): |
| 241 | if c['type'] == 'text': |
| 242 | c = c['pairs'] |
| 243 | elif c['type'] == 'image': |
| 244 | c = Image.open(io.BytesIO(base64.b64decode(c["pairs"]))).convert('RGB') |
| 245 | else: |
| 246 | raise ValueError("content type only support text and image.") |
| 247 | new_cnts.append(c) |
| 248 | msg['content'] = new_cnts |
| 249 | print(f'msgs: {str(msgs)}') |
| 250 | |
| 251 | answer = self.model.chat( |
| 252 | image=image, |
| 253 | msgs=msgs, |
| 254 | tokenizer=self.tokenizer, |
| 255 | ) |
| 256 | return answer |
| 257 | |
| 258 | |
| 259 | class MiniCPMVChat: |