(model_name, questions_file, answers_file)
| 31 | |
| 32 | @torch.inference_mode() |
| 33 | def eval_model(model_name, questions_file, answers_file): |
| 34 | # Model |
| 35 | disable_torch_init() |
| 36 | model_name = os.path.expanduser(model_name) |
| 37 | tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False) |
| 38 | model = AutoModelForCausalLM.from_pretrained(model_name, |
| 39 | torch_dtype=torch.float16).cuda() |
| 40 | |
| 41 | |
| 42 | ques_file = open(os.path.expanduser(questions_file), "r") |
| 43 | ans_file = open(os.path.expanduser(answers_file), "w") |
| 44 | for i, line in enumerate(tqdm(ques_file)): |
| 45 | idx = json.loads(line)["question_id"] |
| 46 | qs = json.loads(line)["text"] |
| 47 | cat = json.loads(line)["category"] |
| 48 | conv = default_conversation.copy() |
| 49 | conv.append_message(conv.roles[0], qs) |
| 50 | prompt = conv.get_prompt() |
| 51 | inputs = tokenizer([prompt]) |
| 52 | input_ids = torch.as_tensor(inputs.input_ids).cuda() |
| 53 | stopping_criteria = KeywordsStoppingCriteria([conv.sep], tokenizer, input_ids) |
| 54 | output_ids = model.generate( |
| 55 | input_ids, |
| 56 | do_sample=True, |
| 57 | use_cache=True, |
| 58 | temperature=0.7, |
| 59 | max_new_tokens=1024, |
| 60 | stopping_criteria=[stopping_criteria]) |
| 61 | outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0] |
| 62 | try: |
| 63 | index = outputs.index(conv.sep, len(prompt)) |
| 64 | except ValueError: |
| 65 | outputs += conv.sep |
| 66 | index = outputs.index(conv.sep, len(prompt)) |
| 67 | |
| 68 | outputs = outputs[len(prompt) + len(conv.roles[1]) + 2:index].strip() |
| 69 | ans_id = shortuuid.uuid() |
| 70 | ans_file.write(json.dumps({"question_id": idx, |
| 71 | "text": outputs, |
| 72 | "answer_id": ans_id, |
| 73 | "model_id": model_name, |
| 74 | "metadata": {}}) + "\n") |
| 75 | ans_file.flush() |
| 76 | ans_file.close() |
| 77 | |
| 78 | if __name__ == "__main__": |
| 79 | parser = argparse.ArgumentParser() |
no test coverage detected