Function fetchDatasetSample

browse/test/security-bench.test.ts:45–60

()

type BenchRow = { content: string; label: 'yes' | 'no' };

async function fetchDatasetSample(): Promise<BenchRow[]> {
  const rows: BenchRow[] = [];
  // HF datasets-server caps at 100 rows per request.
  for (let offset = 0; rows.length < SAMPLE_SIZE; offset += 100) {
    const length = Math.min(100, SAMPLE_SIZE - rows.length);
    const url = `${HF_API}&offset=${offset}&length=${length}`;
    const res = await fetch(url);
    if (!res.ok) throw new Error(`HF API ${res.status}: ${url}`);
    const data = (await res.json()) as { rows: Array<{ row: BenchRow }> };
    if (!data.rows?.length) break;
    for (const r of data.rows) {
      rows.push({ content: r.row.content, label: r.row.label as 'yes' | 'no' });
    }
  }
  return rows;
}

async function loadOrFetchRows(): Promise<BenchRow[]> {
  if (fs.existsSync(CACHE_FILE)) {