MCPcopy
hub / github.com/apify/crawlee / KeyValueFileSystemEntry

Class KeyValueFileSystemEntry

packages/memory-storage/src/fs/key-value-store/fs.ts:15–95  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

13import type { CreateStorageImplementationOptions } from '.';
14
15export class KeyValueFileSystemEntry implements StorageImplementation<InternalKeyRecord> {
16 private storeDirectory: string;
17 private writeMetadata: boolean;
18
19 private filePath!: string;
20 private fileMetadataPath!: string;
21 private rawRecord!: Omit<InternalKeyRecord, 'value'>;
22 private fsQueue = new AsyncQueue();
23
24 constructor(options: CreateStorageImplementationOptions) {
25 this.storeDirectory = options.storeDirectory;
26 this.writeMetadata = options.writeMetadata;
27 }
28
29 async get(): Promise<InternalKeyRecord> {
30 await this.fsQueue.wait();
31 let file: Buffer | string;
32
33 try {
34 file = await readFile(this.filePath);
35 } catch {
36 try {
37 // Try without extension
38 file = await readFile(resolve(this.storeDirectory, this.rawRecord.key));
39 memoryStorageLog.warning(
40 [
41 `Key-value entry "${this.rawRecord.key}" for store ${basename(
42 this.storeDirectory,
43 )} does not have a file extension, assuming it as text.`,
44 'If you want to have correct interpretation of the file, you should add a file extension to the entry.',
45 ].join('\n'),
46 );
47 file = file.toString('utf-8');
48 } catch {
49 // This is impossible to happen, but just in case
50 throw new Error(`Could not find file at ${this.filePath}`);
51 }
52 } finally {
53 this.fsQueue.shift();
54 }
55
56 return {
57 ...this.rawRecord,
58 value: file,
59 };
60 }
61
62 async update(data: InternalKeyRecord) {
63 await this.fsQueue.wait();
64 const contentType = mime.contentType(data.key);
65 const fileName =
66 // the content type might include charset, e.g. `text/html; charset=utf-8`, so we check via `startsWith` instead of `===`
67 contentType && data.contentType && contentType.startsWith(data.contentType)
68 ? data.key
69 : `${data.key}.${data.extension}`;
70
71 this.filePath ??= resolve(this.storeDirectory, fileName);
72 this.fileMetadataPath ??= resolve(this.storeDirectory, `${data.key}.__metadata__.json`);

Callers

nothing calls this directly

Calls

no outgoing calls

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…