| 1463 | |
| 1464 | |
| 1465 | class ConversationBot: |
| 1466 | def __init__(self, load_dict): |
| 1467 | # load_dict = {'VisualQuestionAnswering':'cuda:0', 'ImageCaptioning':'cuda:1',...} |
| 1468 | print(f"Initializing VisualChatGPT, load_dict={load_dict}") |
| 1469 | if 'ImageCaptioning' not in load_dict: |
| 1470 | raise ValueError("You have to load ImageCaptioning as a basic function for VisualChatGPT") |
| 1471 | |
| 1472 | self.models = {} |
| 1473 | # Load Basic Foundation Models |
| 1474 | for class_name, device in load_dict.items(): |
| 1475 | self.models[class_name] = globals()[class_name](device=device) |
| 1476 | |
| 1477 | # Load Template Foundation Models |
| 1478 | for class_name, module in globals().items(): |
| 1479 | if getattr(module, 'template_model', False): |
| 1480 | template_required_names = {k for k in inspect.signature(module.__init__).parameters.keys() if k!='self'} |
| 1481 | loaded_names = set([type(e).__name__ for e in self.models.values()]) |
| 1482 | if template_required_names.issubset(loaded_names): |
| 1483 | self.models[class_name] = globals()[class_name]( |
| 1484 | **{name: self.models[name] for name in template_required_names}) |
| 1485 | |
| 1486 | print(f"All the Available Functions: {self.models}") |
| 1487 | |
| 1488 | self.tools = [] |
| 1489 | for instance in self.models.values(): |
| 1490 | for e in dir(instance): |
| 1491 | if e.startswith('inference'): |
| 1492 | func = getattr(instance, e) |
| 1493 | self.tools.append(Tool(name=func.name, description=func.description, func=func)) |
| 1494 | self.llm = OpenAI(temperature=0) |
| 1495 | self.memory = ConversationBufferMemory(memory_key="chat_history", output_key='output') |
| 1496 | |
| 1497 | def init_agent(self, lang): |
| 1498 | self.memory.clear() #clear previous history |
| 1499 | if lang=='English': |
| 1500 | PREFIX, FORMAT_INSTRUCTIONS, SUFFIX = VISUAL_CHATGPT_PREFIX, VISUAL_CHATGPT_FORMAT_INSTRUCTIONS, VISUAL_CHATGPT_SUFFIX |
| 1501 | place = "Enter text and press enter, or upload an image" |
| 1502 | label_clear = "Clear" |
| 1503 | else: |
| 1504 | PREFIX, FORMAT_INSTRUCTIONS, SUFFIX = VISUAL_CHATGPT_PREFIX_CN, VISUAL_CHATGPT_FORMAT_INSTRUCTIONS_CN, VISUAL_CHATGPT_SUFFIX_CN |
| 1505 | place = "输入文字并回车，或者上传图片" |
| 1506 | label_clear = "清除" |
| 1507 | self.agent = initialize_agent( |
| 1508 | self.tools, |
| 1509 | self.llm, |
| 1510 | agent="conversational-react-description", |
| 1511 | verbose=True, |
| 1512 | memory=self.memory, |
| 1513 | return_intermediate_steps=True, |
| 1514 | agent_kwargs={'prefix': PREFIX, 'format_instructions': FORMAT_INSTRUCTIONS, |
| 1515 | 'suffix': SUFFIX}, ) |
| 1516 | return gr.update(visible = True), gr.update(visible = False), gr.update(placeholder=place), gr.update(value=label_clear) |
| 1517 | |
| 1518 | def run_text(self, text, state): |
| 1519 | self.agent.memory.buffer = cut_dialogue_history(self.agent.memory.buffer, keep_last_n_words=500) |
| 1520 | res = self.agent({"input": text.strip()}) |
| 1521 | res['output'] = res['output'].replace("\\", "/") |
| 1522 | response = re.sub('(image/[-\w]*.png)', lambda m: f'})*{m.group(0)}*', res['output']) |