Function scrapeData

apps/web/lib/search.ts:18–48

({ trackId }: { trackId: string })

const VECTOR_SIZE = parseInt(process.env.VECTOR_SIZE!) || 768;

export async function scrapeData({ trackId }: { trackId: string }) {
  const notion = getNotionClient();
  const track = await getTrack(trackId);
  const data = await Promise.all(
    track?.problems.map(async (problem: any) => {
      const notionDocId = problem.notionDocId;
      const notionPage = await fetchNotionPage(notion, notionDocId);
      const titles = Object.values<any>(notionPage?.block ?? {})
        .map((block) => {
          const title = block?.value?.properties?.title;
          if (title && title[0] && title[0][0]) {
            return title[0][0];
          } else {
            const source = block?.value?.properties?.source;
            if (source && source[0] && source[0][0]) {
              return source[0][0];
            } else {
              return null;
            }
          }
        })
        .filter((title) => title);
      problem["titles"] = titles.flat().join(" ");
      problem["trackId"] = track.id;
      problem["trackTitle"] = track.title;
      problem["image"] = track.image;
      return problem;
    }) || []
  );
  return data;
}

export async function createCollection(){
  await client.createCollection("DailyCode",{