Function getTopOneMillionDomains

Build/validate-gfwlist.ts:16–55

()

import { GLOBAL } from '../Source/non_ip/global';

export async function getTopOneMillionDomains() {
  const { parse: csvParser } = await import('csv-parse');

  const topDomainTrie = new HostnameSmolTrie();
  const csvParse = csvParser({ columns: false, skip_empty_lines: true });

  const topDomainsZipBody = await (await $$fetch('https://tranco-list.eu/top-1m.csv.zip', {
    headers: {
      accept: '*/*',
      'user-agent': 'curl/8.12.1'
    }
  })).arrayBuffer();
  let entry: yauzl.Entry | null = null;
  for await (const e of await yauzl.fromBuffer(Buffer.from(topDomainsZipBody))) {
    if (e.filename === 'top-1m.csv') {
      entry = e;
      break;
    }
  }

  const { promise, resolve, reject } = Promise.withResolvers<HostnameSmolTrie>();

  const readable = await nullthrow(entry, 'top-1m.csv entry not found').openReadStream();
  const parser = readable.pipe(csvParse);
  parser.on('readable', () => {
    let record;
    while ((record = parser.read()) !== null) {
      topDomainTrie.add(record[1]);
    }
  });

  parser.on('end', () => {
    resolve(topDomainTrie);
  });
  parser.on('error', (err) => {
    reject(err);
  });

  return promise;
}

export async function parseGfwList() {
  const whiteSet = new Set<string>();