* Scans the MDX pages in packages/dev/s2-docs/pages and produces a text-based markdown variant of * each file. React-specific JSX elements such as and are replaced * with plain markdown equivalents so that the resulting *.md files can be consumed by LLMs.
()
| 3760 | * with plain markdown equivalents so that the resulting *.md files can be consumed by LLMs. |
| 3761 | */ |
| 3762 | async function main() { |
| 3763 | const mdxFiles = ( |
| 3764 | await glob('*/**/*.mdx', { |
| 3765 | cwd: S2_DOCS_PAGES_ROOT, |
| 3766 | absolute: true |
| 3767 | }) |
| 3768 | ).sort((a, b) => a.localeCompare(b)); |
| 3769 | |
| 3770 | // Collect generated markdown filenames and headings for each library so we can build llms.txt files. |
| 3771 | const docsByLibrary = { |
| 3772 | s2: [], |
| 3773 | 'react-aria': [], |
| 3774 | internationalized: [], |
| 3775 | root: [] |
| 3776 | }; |
| 3777 | |
| 3778 | for (const filePath of mdxFiles) { |
| 3779 | const rawContent = fs.readFileSync(filePath, 'utf8'); |
| 3780 | |
| 3781 | // Skip redirect pages |
| 3782 | if (rawContent.includes('<meta http-equiv="refresh"')) { |
| 3783 | continue; |
| 3784 | } |
| 3785 | |
| 3786 | // Skip error pages |
| 3787 | if (path.basename(filePath) === 'error.mdx') { |
| 3788 | continue; |
| 3789 | } |
| 3790 | |
| 3791 | let mdContent = rawContent.replace(LICENSE_COMMENT_REGEX, ''); |
| 3792 | |
| 3793 | // Inline the S2 Routers MDX content once at the bottom of the page. |
| 3794 | // Avoids rendering the same content for every framework. |
| 3795 | if (mdContent.includes('<Routers components={props.components} />')) { |
| 3796 | mdContent = mdContent.replace(ROUTERS_PLACEHOLDER_REGEX, 'See the Routers section below.'); |
| 3797 | mdContent += '\n\n## Routers\n\n' + getRoutersMdxContent(); |
| 3798 | } |
| 3799 | |
| 3800 | const processor = unified() |
| 3801 | .use(remarkParse) |
| 3802 | .use(remarkMdx) |
| 3803 | .use(remarkRemoveImportsExports) |
| 3804 | .use(remarkDocsComponentsToMarkdown); |
| 3805 | |
| 3806 | const file = {value: mdContent, path: filePath}; |
| 3807 | const tree = processor.parse(file); |
| 3808 | const transformed = await processor.run(tree, file); |
| 3809 | let markdown = toMarkdown(transformed, { |
| 3810 | fences: true, |
| 3811 | bullet: '-', |
| 3812 | listItemIndent: 'one', |
| 3813 | extensions: mdxToMarkdown.extensions |
| 3814 | }); |
| 3815 | |
| 3816 | // Convert markdown links ending in .html to .md (relative links only) |
| 3817 | markdown = markdown.replace(/\[([^\]]+)\]\(([^)]+\.html)\)/g, (match, text, url) => { |
| 3818 | if (!url.startsWith('http') && !url.startsWith('//')) { |
| 3819 | return `[${text}](${url.replace(/\.html$/, '.md')})`; |
no test coverage detected