Function getDomainBaseline

background.js:713–759

(baseUrl)

const domainCache = new Map();

async function getDomainBaseline(baseUrl) {
  if (domainCache.has(baseUrl)) {
    const cached = domainCache.get(baseUrl);
    if (Date.now() - cached.timestamp < 300000) { // 5 min cache
      return cached;
    }
  }

  try {
    // Get baseline response
    const baseResponse = await rateLimitedFetch(baseUrl);
    const baseText = await baseResponse.text();

    // Get soft-404 fingerprint (request a random non-existent path)
    const randomPath = `/${Math.random().toString(36).substring(7)}-${Date.now()}`;
    let soft404Fingerprint = null;
    let soft404Length = 0;

    try {
      const soft404Response = await rateLimitedFetch(baseUrl + randomPath);
      const soft404Text = await soft404Response.text();
      soft404Length = soft404Text.length;
      // Create a fingerprint based on content structure, not exact content
      soft404Fingerprint = {
        status: soft404Response.status,
        length: soft404Text.length,
        hasTitle: /<title>/i.test(soft404Text),
        isSoft404: isSoft404(soft404Text),
      };
    } catch (e) {
      // Couldn't get soft-404, that's fine
    }

    const cached = {
      baseText,
      baseLength: baseText.length,
      soft404Fingerprint,
      soft404Length,
      timestamp: Date.now(),
    };

    domainCache.set(baseUrl, cached);
    return cached;
  } catch (e) {
    return null;
  }
}

function matchesSoft404(response, text, fingerprint) {
  if (!fingerprint) return false;