Function fetchPageContent

express-api/index.js:67–80

(link)

  }
  // 17. Fetch page content
  const fetchPageContent = async (link) => {
    console.log(`7. Fetching page content for ${link}`);
    try {
      const response = await fetch(link);
      if (!response.ok) {
        return ""; // skip if fetch fails
      }
      const text = await response.text();
      return extractMainContent(text, link);
    } catch (error) {
      console.error(`Error fetching page content for ${link}:`, error);
      return '';
    }
  };
  // 18. Extract main content from the HTML page
  function extractMainContent(html, link) {
    console.log(`8. Extracting main content from HTML for ${link}`);