| 37 | |
| 38 | @retry(stop=stop_after_attempt(10), wait=wait_exponential(min=4, max=60)) |
| 39 | async def get_chat_completion(message, semaphore) -> str: |
| 40 | try: |
| 41 | async with semaphore: |
| 42 | response = await client.chat.completions.create( |
| 43 | model=model, |
| 44 | messages=[{"role": "user", "content": message["question"]}], |
| 45 | timeout=80 |
| 46 | ) |
| 47 | message["pred"] = response.choices[0].message.content |
| 48 | return message |
| 49 | except Exception as e: |
| 50 | print(f"Error in get_chat_completion for message {type(e).__name__} - {str(e)}") |
| 51 | raise |
| 52 | |
| 53 | async def request_model(prompts): |
| 54 | semaphore = asyncio.Semaphore(MAX_CONCURRENT) |