MCPcopy
hub / github.com/apify/crawlee / ImpitHttpClient

Class ImpitHttpClient

packages/impit-client/src/index.ts:25–255  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

23 * A HTTP client implementation based on the `impit library.
24 */
25export class ImpitHttpClient implements BaseHttpClient {
26 private impitOptions: ImpitOptions;
27 private maxRedirects: number;
28 private followRedirects: boolean;
29
30 /**
31 * Enables reuse of `impit` clients for the same set of options.
32 * This is useful for performance reasons, as creating
33 * a new client for each request breaks TCP connection
34 * (and other resources) reuse.
35 */
36 private clientCache: LruCache<{ client: Impit; cookieJar: ToughCookieJar }> = new LruCache({ maxLength: 10 });
37
38 private getClient(options: ImpitOptions) {
39 const { cookieJar, ...rest } = options;
40
41 const cacheKey = JSON.stringify(rest);
42 const existingClient = this.clientCache.get(cacheKey);
43
44 if (existingClient && (!cookieJar || existingClient.cookieJar === cookieJar)) {
45 return existingClient.client;
46 }
47
48 const client = new Impit(options);
49 this.clientCache.add(cacheKey, { client, cookieJar: cookieJar as ToughCookieJar });
50
51 return client;
52 }
53
54 constructor(options?: Omit<ImpitOptions, 'proxyUrl'> & { maxRedirects?: number }) {
55 this.impitOptions = options ?? {};
56
57 this.maxRedirects = options?.maxRedirects ?? 10;
58 this.followRedirects = options?.followRedirects ?? true;
59 }
60
61 /**
62 * Flattens the headers of a `HttpRequest` to a format that can be passed to `impit`.
63 * @param headers `SimpleHeaders` object
64 * @returns `Record<string, string>` object
65 */
66 private intoHeaders<TResponseType extends keyof ResponseTypes>(
67 headers?: Exclude<HttpRequest<TResponseType>['headers'], undefined>,
68 ): Headers | undefined {
69 if (!headers) {
70 return undefined;
71 }
72
73 const result = new Headers();
74
75 for (const headerName of Object.keys(headers)) {
76 const headerValue = headers[headerName];
77
78 for (const value of Array.isArray(headerValue) ? headerValue : [headerValue]) {
79 if (value === undefined) continue;
80
81 result.append(headerName, value);
82 }

Callers

nothing calls this directly

Calls

no outgoing calls

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…