Function getBody

scripts/fill.ts:36–109

()

}

const getBody = async () => {
  const fileStream = fs.createReadStream("scripts/out.jsonl");
  const rl = readline.createInterface({
    input: fileStream,
  });

  const body = [] as any[];
  rl.on("line", (line: string) => {
    try {
      const parsedLine = JSON.parse(line);
      const subcategory_slug = parsedLine.custom_id;
      const response = JSON.parse(
        parsedLine.response.body.choices[0].message.content,
      );

      const products = response.products;

      const productsToAdd = products.map(
        (product: { name: string; description: string }) => {
          const price = parseFloat((Math.random() * 20 + 5).toFixed(1));
          return {
            slug: slugify(product.name, { lower: true }),
            name: product.name,
            description: product.description ?? "",
            price,
            subcategory_slug,
          };
        },
      );
      body.push(...productsToAdd);
    } catch (err) {
      console.error("Error parsing JSON:", err);
      fs.appendFile("scripts/errors.txt", line + "\n", (err: any) => {
        if (err) {
          console.error(err);
          return;
        }
      });
    }
  });

  rl.on("close", async () => {
    console.log(body.length);
    for (let i = 0; i < body.length; i += 10000) {
      const chunk = body.slice(i, i + 10000);
      await db.insert(products).values(chunk).onConflictDoNothing();
      console.log(`Inserted products ${i} to ${i + chunk.length}`);
      await new Promise((resolve) => setTimeout(resolve, 100)); // 100 ms
    }

    // const data = [] as any[];
    // const subcategories = await getEmptySubcategories();
    // subcategories.forEach((subcat) => {
    //   // get 30 random products from body, regardless of subcategory_slug
    //   const products = getRandomObjects(body, 30).map((product) => {
    //     return {
    //       ...product,
    //       subcategory_slug: subcat,