(args)
| 13 | return completion |
| 14 | |
| 15 | def main(args): |
| 16 | |
| 17 | paper_name = args.paper_name |
| 18 | pdf_json_path = args.pdf_json_path |
| 19 | output_dir = args.output_dir |
| 20 | target_repo_dir = args.target_repo_dir |
| 21 | eval_result_dir = args.eval_result_dir |
| 22 | gpt_version = args.gpt_version |
| 23 | generated_n = args.generated_n |
| 24 | data_dir = args.data_dir |
| 25 | eval_type = args.eval_type |
| 26 | is_papercoder = True if args.papercoder else False |
| 27 | |
| 28 | gold_repo_dir = args.gold_repo_dir |
| 29 | |
| 30 | # paper |
| 31 | with open(f'{pdf_json_path}') as f: |
| 32 | paper_json = json.load(f) |
| 33 | |
| 34 | codes = "" |
| 35 | if is_papercoder: |
| 36 | # python files |
| 37 | target_files_dict = read_python_files(target_repo_dir) |
| 38 | |
| 39 | # configuration |
| 40 | with open(f'{output_dir}/planning_config.yaml') as f: |
| 41 | config_yaml = f.read() |
| 42 | |
| 43 | context_lst = extract_planning(f'{output_dir}/planning_trajectories.json') |
| 44 | |
| 45 | if os.path.exists(f'{output_dir}/task_list.json'): |
| 46 | with open(f'{output_dir}/task_list.json') as f: |
| 47 | task_list = json.load(f) |
| 48 | else: |
| 49 | task_list = content_to_json(context_lst[2]) |
| 50 | |
| 51 | todo_file_lst = task_list['Task list'] |
| 52 | for todo_file in todo_file_lst: |
| 53 | if todo_file.endswith(".yaml"): |
| 54 | continue |
| 55 | codes += f"```python\n## File name: {todo_file}\n{target_files_dict[todo_file]}\n```\n\n" |
| 56 | |
| 57 | codes += f"```yaml\n## File name: config.yaml\n{config_yaml}\n```\n\n" |
| 58 | else: |
| 59 | target_files_dict = read_all_files(target_repo_dir, allowed_ext=[".py", ".yaml", ".yml", ".md", ".sh", ".bash"], is_print=False) |
| 60 | for file_name, code in target_files_dict.items(): |
| 61 | codes += f"```## File name: {file_name}\n{code}\n```\n\n" |
| 62 | |
| 63 | |
| 64 | prompt = open(f"{data_dir}/prompts/{eval_type}.txt").read() |
| 65 | |
| 66 | cur_prompt = prompt.replace('{{Paper}}', f"{paper_json}").replace('{{Code}}', codes) |
| 67 | |
| 68 | # refernce-based |
| 69 | if "ref_based" == eval_type and len(gold_repo_dir) > 0: |
| 70 | all_files_dict = read_all_files(gold_repo_dir, allowed_ext=[".py", ".yaml", ".yml", ".md", ".sh", ".bash"], is_print=False) |
| 71 | |
| 72 | goldcodes = "" |
no test coverage detected