Main function - starts a user prompt loop, and then kicks off three threads in parallel on CPU, GPU and NPU.
()
| 99 | |
| 100 | |
| 101 | def run_bot(): |
| 102 | |
| 103 | """ Main function - starts a user prompt loop, and then kicks off |
| 104 | three threads in parallel on CPU, GPU and NPU. """ |
| 105 | |
| 106 | set_env(input_panel_fixed=False, output_animation=False) |
| 107 | put_markdown("""# Multimedia Bot with LLMWare, OpenVINO, & PyWebio""") |
| 108 | |
| 109 | img_counter = 0 |
| 110 | start_bot = True |
| 111 | |
| 112 | while start_bot: |
| 113 | |
| 114 | # user input chat box |
| 115 | |
| 116 | form = input_group('', [ |
| 117 | textarea(name='msg', placeholder='Ask LLMWare Bot', rows=3), |
| 118 | actions(name='cmd', buttons=['Send', 'Exit']) |
| 119 | ]) |
| 120 | |
| 121 | if form['cmd'] == "Exit": |
| 122 | start_bot = False |
| 123 | break |
| 124 | |
| 125 | user_msg = form['msg'] |
| 126 | |
| 127 | # display the user prompt |
| 128 | put_info(user_msg) |
| 129 | |
| 130 | # thread 1 - CPU - text gen |
| 131 | text_gen_thread = threading.Thread(target=text_gen_bot, |
| 132 | kwargs={"user_msg": user_msg, |
| 133 | "img_counter": img_counter}) |
| 134 | text_gen_thread.start() |
| 135 | |
| 136 | # thread 2 - GPU - text to image gen |
| 137 | image_gen_thread = threading.Thread(target=image_gen_bot, |
| 138 | kwargs={"user_msg": user_msg, |
| 139 | "img_counter": img_counter}) |
| 140 | image_gen_thread.start() |
| 141 | |
| 142 | # load the npu model in main and pass to thread |
| 143 | npu_model = ModelCatalog().load_model("slim-topics-npu-ov", |
| 144 | sample=False,temperature=0.0, |
| 145 | device="NPU") |
| 146 | |
| 147 | image_gen_thread.join() |
| 148 | text_gen_thread.join() |
| 149 | |
| 150 | # pull the text output file created in the text gen thread |
| 151 | fp = os.path.join(LLMWareConfig().get_llmware_path(), "txt_tmp.txt") |
| 152 | text_output = "" |
| 153 | if os.path.exists(fp): |
| 154 | text_output = open(fp, "r").read() |
| 155 | |
| 156 | # kick off NPU thread |
| 157 | npu_gen_thread = threading.Thread(target=classifier_agent_bot, |
| 158 | kwargs={"text_output": text_output, |
no test coverage detected