Function generateSitemap

packages/core/src/sitemap.ts:76–135

()

	return new Date(getTimestamp(snowflake));
}
export async function generateSitemap() {
	let hasMore;
	let offset = 0;
	const safetyLimit = 200;
	const limit = 10000;
	const allThreads: Channel[] = [];
	do {
		// eslint-disable-next-line no-await-in-loop
		const { threads, hasMore: hm } = await listPublicThreads({
			offset,
			limit,
		});
		allThreads.push(...threads);
		hasMore = hm;
		offset++;
		console.log(`Gathered ${allThreads.length} threads`);
	} while (hasMore && offset < safetyLimit);
	console.log(`Finished collecting ${allThreads.length} threads`);

	// Go from newest to oldest
	const chunks = chunk(allThreads, limit).reverse();
	if (chunks.length == 0 || allThreads.length == 0) {
		return;
	}
	let i = 0;
	for await (const chunk of chunks) {
		const sitemap = new Sitemap('https://www.answeroverflow.com', 'url');
		sitemap.addMany(
			chunk.map((thread) => ({
				loc: `/m/${thread.id}`,
				// We really don't expect archived threads to be updated
				priority: 0.9,
				lastmod: thread.archivedTimestamp
					? new Date(Number(thread.archivedTimestamp))
					: getDate(thread.id),
			})),
		);
		await uploadFile({
			contentType: 'text/xml',
			filename: `sitemaps/sitemap-${i}.xml`,
			stream: sitemap.toXml(),
		});
		console.log(
			`Uploaded sitemaps/sitemap-${i}.xml with ${chunk.length} entries`,
		);
		i++;
	}
	const sitemapIndex = new Sitemap('https://www.answeroverflow.com', 'sitemap');
	for (let j = 0; j < i; j++) {
		sitemapIndex.add({
			loc: `/sitemap${j}.xml`,
		});
	}
	await uploadFile({
		contentType: 'text/xml',
		filename: 'sitemaps/sitemap.xml',
		stream: sitemapIndex.toXml(),
	});