Function addCrawlPageTask

packages/start-plugin-core/src/prerender.ts:111–228

(page: Page)

    return Array.from(prerendered)

    function addCrawlPageTask(page: Page) {
      if (seen.has(page.path)) return

      seen.add(page.path)

      if (page.fromCrawl) {
        startConfig.pages.push(page)
      }

      if (!(page.prerender?.enabled ?? true)) return

      if (
        startConfig.prerender?.filter &&
        !startConfig.prerender.filter(page)
      ) {
        return
      }

      const prerenderOptions = {
        ...startConfig.prerender,
        ...page.prerender,
      }

      queue.add(async () => {
        logger.info(`Crawling: ${page.path}`)
        const retries = retriesByPath.get(page.path) || 0

        try {
          const res = await requestWithRedirects(
            withTrailingSlash(withBase(page.path, routerBasePath)),
            {
              headers: {
                ...(prerenderOptions.headers ?? {}),
              },
            },
            prerenderOptions.maxRedirects,
          )

          if (!res.ok) {
            if (isRedirectResponse(res)) {
              logger.warn(`Max redirects reached for ${page.path}`)
            }

            throw new Error(`Failed to fetch ${page.path}: ${res.statusText}`, {
              cause: res,
            })
          }

          const cleanPagePath = (
            prerenderOptions.outputPath || page.path
          ).split(/[?#]/)[0]!

          const contentType = res.headers.get('content-type') || ''
          const isImplicitHTML =
            !cleanPagePath.endsWith('.html') && contentType.includes('html')

          const routeWithIndex = cleanPagePath.endsWith('/')
            ? cleanPagePath + 'index'