Function fetchJSON

packages/scripts/maintenance/lib/upstream.ts:115–190

(
  url: string,
  opts: FetchOptions = {}
)

 * gives up produces a partial diff that looks like real data loss.
 */
export async function fetchJSON<T = any>(
  url: string,
  opts: FetchOptions = {}
): Promise<T | null> {
  const cacheFile = join(
    CACHE_DIR,
    createHash("sha256").update(url).digest("hex").slice(0, 32) + ".json"
  );

  if (!opts.noCache && existsSync(cacheFile)) {
    try {
      const cached = JSON.parse(readFileSync(cacheFile, "utf-8"));
      if (Date.now() - cached.at < DEFAULT_TTL_MS) return cached.body as T;
    } catch {
      // A corrupt cache entry is never worth failing over: refetch.
    }
  }

  const headers: Record<string, string> = {
    accept: "application/json",
    "user-agent": "stack-hub-maintenance (+https://stack.lol)",
  };
  if (url.startsWith("https://api.github.com/")) {
    const token = resolveGithubToken();
    if (token) headers.authorization = `Bearer ${token}`;
    headers.accept = "application/vnd.github+json";
    headers["x-github-api-version"] = "2022-11-28";
  }

  let lastError = "";
  for (let attempt = 0; attempt < 5; attempt++) {
    let res: Response;
    try {
      res = await fetch(url, { headers });
    } catch (e) {
      lastError = String(e);
      await sleep(1000 * 2 ** attempt);
      continue;
    }

    if (res.ok) {
      const body = (await res.json()) as T;
      mkdirSync(CACHE_DIR, { recursive: true });
      writeFileSync(cacheFile, JSON.stringify({ at: Date.now(), url, body }));
      return body;
    }

    // 422 is GitHub's answer to "that ref does not exist" on commit lookups,
    // and 451 is a DMCA takedown: both are answers, not failures.
    if (res.status === 404 || res.status === 422 || res.status === 451) {
      if (opts.allow404) return null;
      throw new HttpError(res.status, url, `${res.status} on ${url}`);
    }

    // Primary (x-ratelimit-remaining: 0) and secondary (retry-after) limits.
    const retryAfter = Number(res.headers.get("retry-after") ?? 0);
    const reset = Number(res.headers.get("x-ratelimit-reset") ?? 0);
    const remaining = res.headers.get("x-ratelimit-remaining");