Function startCrawl

src/archivist.js:1876–1952

({
    urls, timeout, depth, saveToFile: saveToFile = false,
    batchSize,
    minPageCrawlTime, 
    maxPageCrawlTime,
    program,
  } = {})

  }

  export async function startCrawl({
    urls, timeout, depth, saveToFile: saveToFile = false,
    batchSize,
    minPageCrawlTime, 
    maxPageCrawlTime,
    program,
  } = {}) {
    if ( State.crawling ) {
      console.log('Already crawling...');
      return;
    }
    if ( saveToFile ) {
      logName = `crawl-${(new Date).toISOString()}.urls.txt`; 
      logStream = Fs.createWriteStream(Path.resolve(args.CONFIG_DIR, logName), {flags:'as+'});
    }
    console.log('StartCrawl', {urls, timeout, depth, batchSize, saveToFile, minPageCrawlTime, maxPageCrawlTime, program});
    State.crawling = true;
    State.crawlDepth = depth;
    State.crawlTimeout = timeout;
    State.visited = new Set();
    Object.assign(State,{
      batchSize,
      minPageCrawlTime,
      maxPageCrawlTime
    });
    const batch_sz = State.batchSize || BATCH_SIZE;
    let totalBytes = 0;
    setTimeout(async () => {
      try {
        while(urls.length >= batch_sz) {
          const jobs = [];
          const batch = urls.splice(urls.length-batch_sz,batch_sz);
          console.log({urls, batch});
          for( let i = 0; i < batch_sz; i++ ) {
            const {depth,url} = batch.shift();
            const pr = archiveAndIndexURL(
              url, 
              {crawl: true, depth, timeout, createIfMissing:true, getLinks: depth >= 1, program}
            );
            jobs.push(pr);
          }
          const links = (await Promise.all(jobs)).flat().filter(({url}) => !Q.has(url));
          if ( links.length ) {
            urls.push(...links);
            links.forEach(({url}) => Q.add(url)); 
          }
        }
        while(urls.length) {
          const {depth,url} = urls.pop();
          const links = (await archiveAndIndexURL(
            url, 
            {crawl: true, depth, timeout, createIfMissing:true, getLinks: depth >= 1, program}
          )).filter(({url}) => !Q.has(url));
          console.log(links, Q);
          if ( links.length ) {
            urls.push(...links);
            links.forEach(({url}) => Q.add(url)); 
          }