Function extractMainContent

express-api/index.js:82–87

(html, link)

  };
  // 18. Extract main content from the HTML page
  function extractMainContent(html, link) {
    console.log(`8. Extracting main content from HTML for ${link}`);
    const $ = html.length ? cheerio.load(html) : null
    $("script, style, head, nav, footer, iframe, img").remove();
    return $("body").text().replace(/\s+/g, " ").trim();
  }
  // 19. Process and vectorize the content
  let vectorCount = 0;
  const fetchAndProcess = async (item) => {