()
| 133 | |
| 134 | |
| 135 | def main(): |
| 136 | from openai import OpenAI |
| 137 | |
| 138 | parser = argparse.ArgumentParser() |
| 139 | parser.add_argument("--json", required=True) |
| 140 | parser.add_argument("--api_key", required=True) |
| 141 | args = parser.parse_args() |
| 142 | |
| 143 | global client |
| 144 | client = OpenAI(api_key=args.api_key) |
| 145 | |
| 146 | json_path = Path(args.json) |
| 147 | if not json_path.exists(): |
| 148 | sys.exit(f"File {json_path} not found.") |
| 149 | |
| 150 | with json_path.open(encoding="utf-8") as f: |
| 151 | records = json.load(f) |
| 152 | |
| 153 | story_prompts = [] |
| 154 | generated_answers = [] |
| 155 | for rec in records: |
| 156 | prefix = rec["prefix"] |
| 157 | full_text = rec["full_text"] |
| 158 | assert full_text.startswith(prefix), "full_text does not start with prefix" |
| 159 | story_prompts.append(prefix) |
| 160 | generated_answers.append(full_text[len(prefix):]) |
| 161 | |
| 162 | scores = get_gpt_eval(story_prompts, generated_answers) |
| 163 | print_mean_scores(scores[1]) |
| 164 | |
| 165 | |
| 166 | if __name__ == "__main__": |
no test coverage detected