Function getAllFiles

lib/crawler/dircrawl.js:16–149

(catList)

const BATCH_SIZE = 1000; // Process files in batches for better performance

export default async function getAllFiles(catList) {
  var proctime = new Timer();
  const url = "https://myrient.erista.me/files/";
  let parentRows = await getTableRows({ url: url, base: "" });
  let parents = [];
  for (let x = 0; x < parentRows.html.length; x++) {
    parents.push(
      await parseOutFile({
        file: parentRows.html[x],
        base: "",
        url: url,
        catList: catList,
      })
    );
  }
  let dirWork = splitFilesAndFolders(parents);
  // First run should only have directories. Is there a reason this could change in the future?
  let dirs = dirWork.directories;
  let fetchTasks = [];
  let resolvedFetchTasks = [];
  let parseTasks = [];
  let fileCount = 0;
  let currentBatch = [];

  while (
    dirs.length > 0 ||
    fetchTasks.length > 0 ||
    parseTasks.length > 0 ||
    resolvedFetchTasks.length > 0
  ) {
    let dirStatus = "";
    if (dirs.length > 0) {
      debugPrint(`Queueing: ${dirs[0].name}`);
      fetchTasks.push(
        piscina
          .run(
            { url: dirs[0].path, base: dirs[0].name },
            { name: "getTableRows" }
          )
          .catch((err) => {
            console.error(err);
          })
      );
      dirs.shift();
    }

    if (
      fetchTasks.length >= parseInt(process.env.MAX_FETCH_JOBS) ||
      ((fetchTasks.length > 0 || resolvedFetchTasks.length > 0) &&
        parseTasks.length == 0)
    ) {
      debugPrint(`Resolving ${fetchTasks.length} fetch tasks.`);
      let settledTasks = await Promise.all(fetchTasks);
      resolvedFetchTasks.push(...settledTasks);
      while (resolvedFetchTasks.length > 0) {
        if (piscina.queueSize >= parseInt(process.env.MAX_JOB_QUEUE)) {
          break;
        }