(model, dataset, output_path)
| 13 | |
| 14 | |
| 15 | def infer_openai_model(model, dataset, output_path): |
| 16 | model = RemoteModel(model) |
| 17 | dataset = Dataset(dataset, data_path = os.path.join("test_datasets", dataset.replace("/", "_")), full = False) |
| 18 | |
| 19 | res_dir_path = os.path.join(output_path, dataset.name.replace("/", "_")) |
| 20 | res_path = os.path.join(res_dir_path, model.model + ".json") |
| 21 | |
| 22 | if os.path.exists(res_path): |
| 23 | results = json.load(open(res_path, "r")) |
| 24 | else: |
| 25 | results = {} |
| 26 | |
| 27 | finish = False |
| 28 | index = 0 |
| 29 | |
| 30 | while(not finish): |
| 31 | index += 1 |
| 32 | instance, finish = dataset.next() |
| 33 | prompt = dataset.get_prompt(instance) |
| 34 | if prompt in results and results[prompt][-1] == True and isinstance(results[prompt][0], list): |
| 35 | continue |
| 36 | try: |
| 37 | res = model.run(prompt) |
| 38 | results[prompt] = [res, True] |
| 39 | except Exception as e: |
| 40 | logger.error("Dataset: {}\nModel: {}\nPrompt:\n{}\n".format(dataset.name, model.model, prompt) + str(e)) |
| 41 | results[prompt] = [str(e), False] |
| 42 | print("\r{}/{} ".format(index, dataset.length()), end="", flush=True) |
| 43 | |
| 44 | |
| 45 | |
| 46 | if not os.path.exists(res_dir_path): |
| 47 | os.mkdir(res_dir_path) |
| 48 | |
| 49 | with open(res_path, "w", encoding="utf-8") as of: |
| 50 | of.write(json.dumps(results, sort_keys=True, indent=4, separators=(',', ': '))) |
| 51 | |
| 52 | |
| 53 | def infer_openai_model_for_all(model, output_path): |
no test coverage detected