Function indexURL

src/archivist.js:481–652

({targetInfo:info = {}, sessionId, waitingForDebugger} = {})

      }

      async function indexURL({targetInfo:info = {}, sessionId, waitingForDebugger} = {}) {
        if ( waitingForDebugger ) {
          console.warn(info);
          throw new TypeError(`Target not ready for install`);
        }
        if ( Mode == 'serve' ) return;
        if ( info.type != 'page' ) return;
        if ( ! info.url  || info.url == 'about:blank' ) return;
        if ( info.url.startsWith('chrome') ) return;
        if ( dontCache(info) ) return;

        DEBUG.verboseSlow && console.log('Index URL', info);

        DEBUG.verboseSlow && console.log('Index URL called', info);

        if ( State.Indexing.has(info.targetId) ) return;
        State.Indexing.add(info.targetId);

        if ( ! sessionId ) {
          sessionId = await untilHas(
            State.Sessions, info.targetId, 
            {timeout: State.crawling && State.crawlTimeout}
          );
        }

        if ( !State.Installations.has(sessionId) ) {
          await untilHas(
            State.Installations, sessionId, 
            {timeout: State.crawling && State.crawlTimeout}
          );
        }

        send("DOMSnapshot.enable", {}, sessionId);

        await sleep(500);

        const flatDoc = await send("DOMSnapshot.captureSnapshot", {
          computedStyles: [],
        }, sessionId);
        const pageText = processDoc(flatDoc).replace(STRIP_CHARS, ' ');

        if ( State.crawling ) {
          const has = await untilTrue(() => State.CrawlData.has(info.targetId));

          const {url} = Targets.get(sessionId);
          if ( ! dontCache({url}) ) {
            if ( has ) {
              const {depth,links} = State.CrawlData.get(info.targetId);
              DEBUG.verboseSlow && console.log(info, {depth,links});

              const {result:{value:{title,links:crawlLinks}}} = await send("Runtime.evaluate", {
                expression: `(function () { 
                  return {
                    links: Array.from(
                      document.querySelectorAll('a[href]')
                    ).map(a => a.href),
                    title: document.title
                  };