* Scans the MDX pages in packages/dev/s2-docs/pages and produces a text-based markdown variant of * each file. React-specific JSX elements such as and are replaced * with plain markdown equivalents so that the resulting *.md files can be consumed by LLMs.
()
| 3769 | * with plain markdown equivalents so that the resulting *.md files can be consumed by LLMs. |
| 3770 | */ |
| 3771 | async function main() { |
| 3772 | const mdxFiles = ( |
| 3773 | await glob('*/**/*.mdx', { |
| 3774 | cwd: S2_DOCS_PAGES_ROOT, |
| 3775 | absolute: true |
| 3776 | }) |
| 3777 | ).sort((a, b) => a.localeCompare(b)); |
| 3778 | |
| 3779 | // Collect generated markdown filenames and headings for each library so we can build llms.txt files. |
| 3780 | const docsByLibrary = { |
| 3781 | s2: [], |
| 3782 | 'react-aria': [], |
| 3783 | internationalized: [], |
| 3784 | root: [] |
| 3785 | }; |
| 3786 | |
| 3787 | for (const filePath of mdxFiles) { |
| 3788 | const rawContent = fs.readFileSync(filePath, 'utf8'); |
| 3789 | |
| 3790 | // Skip redirect pages |
| 3791 | if (rawContent.includes('<meta http-equiv="refresh"')) { |
| 3792 | continue; |
| 3793 | } |
| 3794 | |
| 3795 | // Skip error pages |
| 3796 | if (path.basename(filePath) === 'error.mdx') { |
| 3797 | continue; |
| 3798 | } |
| 3799 | |
| 3800 | let mdContent = rawContent.replace(LICENSE_COMMENT_REGEX, ''); |
| 3801 | |
| 3802 | // Inline the S2 Routers MDX content once at the bottom of the page. |
| 3803 | // Avoids rendering the same content for every framework. |
| 3804 | if (mdContent.includes('<Routers components={props.components} />')) { |
| 3805 | mdContent = mdContent.replace(ROUTERS_PLACEHOLDER_REGEX, 'See the Routers section below.'); |
| 3806 | mdContent += '\n\n## Routers\n\n' + getRoutersMdxContent(); |
| 3807 | } |
| 3808 | |
| 3809 | const processor = unified() |
| 3810 | .use(remarkParse) |
| 3811 | .use(remarkMdx) |
| 3812 | .use(remarkRemoveImportsExports) |
| 3813 | .use(remarkDocsComponentsToMarkdown); |
| 3814 | |
| 3815 | const file = {value: mdContent, path: filePath}; |
| 3816 | const tree = processor.parse(file); |
| 3817 | const transformed = await processor.run(tree, file); |
| 3818 | let markdown = toMarkdown(transformed, { |
| 3819 | fences: true, |
| 3820 | bullet: '-', |
| 3821 | listItemIndent: 'one', |
| 3822 | extensions: mdxToMarkdown.extensions |
| 3823 | }); |
| 3824 | |
| 3825 | // Convert markdown links ending in .html to .md (relative links only) |
| 3826 | markdown = markdown.replace(/\[([^\]]+)\]\(([^)]+\.html)\)/g, (match, text, url) => { |
| 3827 | if (!url.startsWith('http') && !url.startsWith('//')) { |
| 3828 | return `[${text}](${url.replace(/\.html$/, '.md')})`; |
no test coverage detected