Function handleScrape

api/src/modules/actions/actions.controller.ts:22–338

(
  sessionService: SessionService,
  browserService: CDPService,
  request: ScrapeRequest,
  reply: FastifyReply,
)

import { safeGoto } from "../../utils/scrape/safeGoTo.js";

export const handleScrape = async (
  sessionService: SessionService,
  browserService: CDPService,
  request: ScrapeRequest,
  reply: FastifyReply,
) => {
  const startTime = Date.now();
  let times: Record<string, number> = {};
  const { url, format, screenshot, pdf, proxyUrl, logUrl, delay, removeBase64Images } =
    request.body;

  let proxy: IProxyServer | null = null;
  let context: BrowserContext | null = null;

  try {
    if (proxyUrl) {
      proxy = await sessionService.proxyFactory(proxyUrl);
      await proxy.listen();
    }
    times.proxyTime = Date.now() - startTime;

    let page: Page;
    let response: HTTPResponse | null = null;
    let pdfResponse: HTTPResponse | null = null;
    let isPdfNavigation = false;

    if (!browserService.isRunning()) {
      await browserService.launch();
    }

    if (proxy) {
      // If a proxy is used, we proceed with browser navigation; implementing proxy-aware Node fetch
      // would require an HTTP agent and is outside current scope.
      context = await browserService.createBrowserContext(proxy.url);
      page = await context.newPage();
      times.proxyPageTime = Date.now() - startTime - times.proxyTime;
    } else {
      page = await browserService.getPrimaryPage();
      times.pageTime = Date.now() - startTime - times.proxyTime;
    }

    // PDF retrieval will use node fetch with session cookies; removed CDP tracking

    let normalizedUrl: string | null = null;
    if (url) {
      normalizedUrl = normalizeUrl(url);
      if (!normalizedUrl) {
        throw new Error(`Invalid URL: ${url}`);
      }
    }

    const safeResponse = normalizedUrl
      ? await safeGoto(page, normalizedUrl, {
          timeout: 30000,
          waitUntil: "domcontentloaded",
        })
      : { response: null, isPdf: false, pdfResponse: null };