Function listBatchFiles

api/lib/hf-reader.ts:127–150

(forceRefresh = false)

}

export async function listBatchFiles(forceRefresh = false): Promise<BatchInfo[]> {
  const now = Date.now()

  // Check cache
  if (!forceRefresh && fileListCache && fileListCache.expires > now) {
    return parseBatchInfos(fileListCache.data)
  }

  // Fetch both directories in parallel
  const [metaFiles, dataFiles] = await Promise.all([
    fetchFileList('metadata'),
    fetchFileList('dataset'),
  ])

  const allFiles = [
    ...metaFiles.map(f => ({ ...f, path: f.path || `metadata/${f.path}` })),
    ...dataFiles.map(f => ({ ...f, path: f.path || `dataset/${f.path}` })),
  ]

  // Cache the combined list
  fileListCache = { data: allFiles, expires: now + FILE_LIST_TTL }

  return parseBatchInfos(allFiles)
}

function parseBatchInfos(files: HFFileEntry[]): BatchInfo[] {
  return files