Function evalFixture

src/eval.ts:85–149

(fixturePath: string)

  const tmpDir = join(
    (await import("node:os")).tmpdir(),
    `codesight-eval-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
  );

  for (const [filePath, content] of Object.entries(fixture.files)) {
    const fullPath = join(tmpDir, filePath);
    await mkdir(dirname(fullPath), { recursive: true });
    await writeFile(fullPath, content as string);
  }

  return tmpDir;
}

async function evalFixture(fixturePath: string): Promise<FixtureResult> {
  const repoJson = JSON.parse(await readFile(join(fixturePath, "repo.json"), "utf-8"));
  const groundTruth: GroundTruth = JSON.parse(
    await readFile(join(fixturePath, "ground-truth.json"), "utf-8")
  );

  // Create temp repo from fixture
  const tmpDir = await createTempRepo(repoJson);

  const startTime = Date.now();

  try {
    // Run codesight detectors
    const project = await detectProject(tmpDir);
    const files = await collectFiles(tmpDir, 10);
    const [routes, schemas, components, config, middleware, graph] = await Promise.all([
      detectRoutes(files, project),
      detectSchemas(files, project),
      detectComponents(files, project),
      detectConfig(files, project),
      detectMiddleware(files, project),
      detectDependencyGraph(files, project),
    ]);

    const runtime = Date.now() - startTime;

    // Compare routes: method:path
    const detectedRoutes = new Set(routes.map((r) => `${r.method}:${r.path}`));
    const expectedRoutes = new Set((groundTruth.routes || []).map((r) => `${r.method}:${r.path}`));

    // Compare models: name
    const detectedModels = new Set(schemas.map((s) => s.name.toLowerCase()));
    const expectedModels = new Set((groundTruth.models || []).map((m) => m.name.toLowerCase()));

    // Compare env vars
    const detectedEnvVars = new Set(config.envVars.map((e) => e.name));
    const expectedEnvVars = new Set(groundTruth.envVars || []);

    const result: FixtureResult = {
      name: repoJson.name,
      routes: calcMetrics(detectedRoutes, expectedRoutes),
      models: calcMetrics(detectedModels, expectedModels),
      envVars: calcMetrics(detectedEnvVars, expectedEnvVars),
      runtime,
    };