Function fetchAllUrlsFromSitemaps

helpers/api/sitemap/parse.js:100–141

( urlString )

}

export async function fetchAllUrlsFromSitemaps ( urlString ) {
    // const domain = getDomainFromString( urlString )

    const allUrls = new Map()

    for ( const sitemapFile of sitemapFilesToTry ) {

        const sitemapUrl = new URL( sitemapFile, urlString )

        // console.log( 'sitemapUrl', sitemapUrl.href )

        // Just do a quich HEAD request to see if the file exists with getting the whole body
        const exists = await headOk( sitemapUrl.href )

        // console.log( 'exists', exists )

        // Skip if sitemap doesn't exist
        if ( !exists ) continue

        // Fetch the whole sitemap
        const urls = await getAllUrlsFromSitemap( sitemapUrl.href , {
            getMethod: async sitemapPath => {
                const sitemapUrl = new URL( sitemapPath, urlString )

                const sitemapXml = await getText( sitemapUrl.href )

                return sitemapXml
            }
        })

        // console.log( 'Sitemap urls', urls )

        // Set our urls to the map
        urls.forEach( tag => allUrls.set( tag.loc, new URL( tag.loc ) ) )

        // Store sitemap urls to context
        // const urlsMap = new Map( urls.map( tag => [ tag.loc, new URL( tag.loc )] ) )

    }

    return allUrls
}