| 245 | return False |
| 246 | |
| 247 | class Chat: |
| 248 | def __init__( |
| 249 | self, |
| 250 | model_path, |
| 251 | device, |
| 252 | num_gpus=1, |
| 253 | load_8bit=False, |
| 254 | temperature=0.7, |
| 255 | max_new_tokens=512, |
| 256 | lora_path=None, |
| 257 | ): |
| 258 | model, tokenizer = load_model( |
| 259 | model_path, device, num_gpus, load_8bit=load_8bit, lora_weights=lora_path |
| 260 | ) |
| 261 | |
| 262 | self.model = model |
| 263 | # self.model.language_model = deepspeed.init_inference( |
| 264 | # self.model.language_model, mp_size=1, dtype=torch.float16, checkpoint=None, replace_with_kernel_inject=True) |
| 265 | self.tokenizer = tokenizer |
| 266 | num_queries = model.config.num_query_tokens |
| 267 | |
| 268 | self.device = device |
| 269 | self.dtype = model.dtype |
| 270 | |
| 271 | stop_words = ["Human: ", "Assistant: ", "###", "\n\n"] |
| 272 | stop_words_ids = [tokenizer(stop_word, return_tensors='pt')['input_ids'].squeeze() for stop_word in stop_words] |
| 273 | stopping_criteria = StoppingCriteriaList([StoppingCriteriaSub(stops=stop_words_ids)]) |
| 274 | |
| 275 | self.conv = get_conv_template("husky") |
| 276 | |
| 277 | self.image_query = DEFAULT_IMG_START_TOKEN + DEFAULT_IMG_END_TOKEN |
| 278 | self.video_query = DEFAULT_VIDEO_START_TOKEN + DEFAULT_VIDEO_END_TOKEN |
| 279 | |
| 280 | self.generation_config = GenerationConfig( |
| 281 | bos_token_id=1, |
| 282 | pad_token_id=0, |
| 283 | do_sample=True, |
| 284 | top_k=20, |
| 285 | top_p=0.9, |
| 286 | temperature=temperature, |
| 287 | max_new_tokens=max_new_tokens, |
| 288 | stopping_criteria=stopping_criteria |
| 289 | ) |
| 290 | |
| 291 | def reduce_boxes(self,scores,boxes,names,iou_threshold = 0.5): |
| 292 | # print("debug!!!!$$$$$$$$$$$$$$$$$$$") |
| 293 | # print(scores) |
| 294 | # print(boxes) |
| 295 | # print(names) |
| 296 | keep_boxes = torch.ones(len(boxes), dtype=torch.bool) |
| 297 | for i in range(len(boxes)): |
| 298 | for j in range(i+1, len(boxes)): |
| 299 | if iou(boxes[i], boxes[j]) > iou_threshold and keep_boxes[i] and keep_boxes[j]: |
| 300 | if scores[i] > scores[j]: |
| 301 | keep_boxes[j] = False |
| 302 | else: |
| 303 | keep_boxes[i] = False |
| 304 | # print("keep_boxes",keep_boxes) |