| 10 | |
| 11 | |
| 12 | class GPTInference: |
| 13 | def __init__(self): |
| 14 | self.client = AzureOpenAI( |
| 15 | azure_endpoint="", |
| 16 | api_key=AZURE_OPENAI_KEY, |
| 17 | api_version="") |
| 18 | |
| 19 | def post_process(self, text): |
| 20 | match = re.search(r"(?i)(?<=\banswer:\s).*", text) |
| 21 | if match: |
| 22 | return match.group(0) |
| 23 | else: |
| 24 | return text |
| 25 | |
| 26 | def predict(self, prompt, temperature=0, max_tokens=1000, retry=3, delay=5): |
| 27 | message_text = [ |
| 28 | {"role": "user", "content": prompt}, |
| 29 | ] |
| 30 | for attempt in range(retry): |
| 31 | try: |
| 32 | response = self.client.chat.completions.create( |
| 33 | model="gpt-4o", # model = "deployment_name" |
| 34 | messages=message_text, |
| 35 | temperature=temperature, |
| 36 | max_tokens=max_tokens, |
| 37 | seed=12345, |
| 38 | ) |
| 39 | break |
| 40 | except: |
| 41 | time.sleep(delay) |
| 42 | response = self.post_process(response.choices[0].message.content) |
| 43 | return response |
| 44 | |
| 45 | def predict_close_book(self, question, demo_file_path, num_demo=16): |
| 46 | demo = load_json_file(demo_file_path) |
| 47 | prompt = ("Here are some examples of questions and their corresponding answer, each with a 'Question' field and an 'Answer' field. " |
| 48 | "Answer the question directly and don't output other thing. ") |
| 49 | for item in demo[:num_demo]: |
| 50 | prompt += f"Question: {item['question']} Answer: {item['short_answers'][0]}\n" |
| 51 | prompt += f"Question: {question} Answer: " |
| 52 | answer = self.predict(prompt) |
| 53 | return answer |
| 54 | |
| 55 | def predict_nq(self, context, question, titles): |
| 56 | titles = ['"' + title + '"' for title in titles] |
| 57 | prompt = (f"Go through the following context and then extract the answer of the question from the context. " |
| 58 | f"Answer the question directly. Your answer should be very concise. " |
| 59 | f"The context is a list of Wikipedia documents, ordered by title: {titles}. " |
| 60 | f"Each Wikipedia document contains a 'title' field and a 'text' field. " |
| 61 | f"The context is: {context}. " |
| 62 | f"The question: {question}. ") |
| 63 | long_answer = self.predict(prompt) |
| 64 | short_answer = self.extract_answer(question, long_answer) |
| 65 | return long_answer, short_answer |
| 66 | |
| 67 | def predict_hotpotqa(self, context, question, titles): |
| 68 | prompt = (f"Go through the following context and then answer the question " |
| 69 | f"The context is a list of Wikipedia documents titled: {titles}. " |