| 109 | controller_addr_pool = [ip_address + str(i) for i in range(post_from, post_to + 1)] |
| 110 | |
| 111 | def request_answer(max_new_tokens): |
| 112 | data = { |
| 113 | "inputs": prompt, |
| 114 | "parameters": { |
| 115 | "max_new_tokens": max_new_tokens, |
| 116 | "do_sample": False |
| 117 | } |
| 118 | } |
| 119 | headers = {"Content-Type": "application/json"} |
| 120 | return requests.post( |
| 121 | random.sample(controller_addr_pool, 1)[0] + "/generate", |
| 122 | headers=headers, |
| 123 | data=json.dumps(data), |
| 124 | timeout=120, |
| 125 | ) |
| 126 | response = request_answer(512) |
| 127 | answer = clean_answer(response.json()["generated_text"]) |
| 128 | return answer |