Process a single sample from the dataset.
(sample: Dict[str, Any])
| 52 | return [int(idx) for idx in ranking_str.split(",")] |
| 53 | |
| 54 | async def process_sample(sample: Dict[str, Any]) -> Dict[str, Any]: |
| 55 | """Process a single sample from the dataset.""" |
| 56 | prompt = sample["turns"][0]["content"] |
| 57 | results = [] |
| 58 | |
| 59 | # Generate responses for each approach |
| 60 | for approach in APPROACHES: |
| 61 | response = await generate_response(prompt, approach) |
| 62 | results.append({"approach": approach, **response}) |
| 63 | |
| 64 | random.shuffle(results) |
| 65 | # Rank the responses |
| 66 | rankings = await rank_responses(prompt, results) |
| 67 | |
| 68 | # Add rankings to results |
| 69 | print(rankings) |
| 70 | for rank, idx in enumerate(rankings): |
| 71 | results[idx]["rank"] = rank |
| 72 | |
| 73 | return { |
| 74 | "prompt": prompt, |
| 75 | "results": results, |
| 76 | } |
| 77 | |
| 78 | async def generate_dataset(num_samples: int, output_file: str): |
| 79 | """Generate the dataset and save it to a JSONL file.""" |
no test coverage detected