Function getAllLlmPages

showcase/shell-docs/src/lib/llm-text.ts:122–234

()

 * `content/docs/integrations/<folder>/`.
 */
export function getAllLlmPages(): LlmPage[] {
  const pages: LlmPage[] = [];
  const seenUrls = new Set<string>();

  const push = (page: LlmPage): void => {
    if (seenUrls.has(page.url)) return;
    seenUrls.add(page.url);
    pages.push(page);
  };

  // ROOT_FRAMEWORK's authored pages win at bare root URLs (the same
  // resolution the live pages use — see UnscopedDocsPage). Walk its
  // folder once so the bare loop below can swap in the override.
  const rootFolder = getDocsFolder(ROOT_FRAMEWORK);
  const rootOverrides = new Map<string, string>(); // slug → filePath
  const rootDir = path.join(CONTENT_DIR, "integrations", rootFolder);
  if (fs.existsSync(rootDir)) {
    for (const { slug, filePath } of walkMdx(rootDir)) {
      rootOverrides.set(slug, filePath);
    }
  }

  // 1. Bare unscoped docs (`src/content/docs/**.mdx`, minus `integrations/`).
  for (const { slug, filePath } of walkMdx(
    DOCS_CONTENT_DIR,
    new Set(["integrations"]),
  )) {
    if (!slug) continue;
    // The root `built-in-agent.mdx` topic page's bare URL permanently
    // redirects to `/` (the retired framework prefix); it stays
    // reachable under other frameworks' scopes only.
    if (slug === ROOT_FRAMEWORK) continue;
    const overridePath = rootOverrides.get(slug);
    const meta = readMetaFromFile(overridePath ?? filePath);
    push({
      url: slug,
      title: meta.title ?? slug,
      description: meta.description,
      filePath: overridePath ?? filePath,
      loadSlug: overridePath ? `integrations/${rootFolder}/${slug}` : slug,
      framework: overridePath ? ROOT_FRAMEWORK : undefined,
    });
  }

  // 2. Per-framework override pages — files under
  //    `content/docs/integrations/<folder>/` that don't have a
  //    root-level equivalent. We emit them at `/<framework>/<topic>`,
  //    or at `/<framework>` for the folder's `index.mdx`.
  //
  //    Note: `walkMdx` strips trailing `/index` from yielded slugs, so
  //    a folder's `index.mdx` arrives here as `slug === ""`. The previous
  //    `if (!slug) continue` guard silently skipped framework root URLs
  //    from `/llms.txt`, leaving LLM crawlers unable to find e.g.
  //    `/langgraph-python`. Treat empty slug as the framework root and
  //    emit it as the bare integration URL.
  const integrations = getIntegrations();
  for (const integration of integrations) {
    if (integration.docs_mode === "hidden") continue;