Function generate_dataset

scripts/gen_optillm_dataset.py:78–88

Generate the dataset and save it to a JSONL file.

(num_samples: int, output_file: str)

    }

async def generate_dataset(num_samples: int, output_file: str):
    """Generate the dataset and save it to a JSONL file."""
    dataset = load_dataset("lmsys/arena-hard-auto-v0.1", split="train")
    
    with open(output_file, "w") as f:
        for sample in tqdm(dataset.select(range( num_samples)), total=num_samples):
            try:
                result = await process_sample(sample)
                f.write(json.dumps(result) + "\n")
            except Exception as e:
                print(f"Skip over this item due to error {str(e)}")

def main():
    parser = argparse.ArgumentParser(description="Generate OptILM dataset")