| 140 | # coroutine with Semaphore |
| 141 | sem = asyncio.Semaphore(multiworker) |
| 142 | async def sample_with_statistics(url, header, llm, temperature, top_p, check, tool_number, sampler, tools, method, figure_dir, wf, statistics, now, dependency_type): |
| 143 | async with sem: # semaphore limits num of simultaneous sampling |
| 144 | if statistics["total"] % 100 == 0 and statistics["total"] != 0: |
| 145 | logger.info(json.dumps(statistics, indent=2)) |
| 146 | statistics_wf.write(json.dumps(statistics) + "\n") |
| 147 | try: |
| 148 | await sample(url, header, llm, temperature, top_p, check, tool_number, sampler, tools, method, figure_dir, wf, dependency_type) |
| 149 | except Exception as e: |
| 150 | statistics["total"] += 1 |
| 151 | statistics["fail"] += 1 |
| 152 | if str(type(e)) not in statistics: |
| 153 | statistics[str(type(e))] = 0 |
| 154 | statistics[str(type(e))] += 1 |
| 155 | raise e |
| 156 | statistics["total"] += 1 |
| 157 | statistics["success"] += 1 |
| 158 | statistics["avg_time_per_sample"] = str((datetime.now() - now) / statistics["success"]) |
| 159 | |
| 160 | async def run(url, header, llm, temperature, top_p, check, sampler, tools, figure_dir, wf, statistics, now, dependency_type): |
| 161 | method = random.choices(list(method_weights.keys()), weights=list(method_weights.values()))[0] |