| 269 | } |
| 270 | |
| 271 | public async load(): Promise<Document[]> { |
| 272 | let text: string |
| 273 | let metadata: Record<string, string> |
| 274 | if (typeof this.filePathOrBlob === 'string') { |
| 275 | const { readFile } = await TextLoader.imports() |
| 276 | text = await readFile(this.filePathOrBlob, 'utf8') |
| 277 | metadata = { source: this.filePathOrBlob } |
| 278 | } else { |
| 279 | text = await this.filePathOrBlob.text() |
| 280 | metadata = { source: 'blob', blobType: this.filePathOrBlob.type } |
| 281 | } |
| 282 | const parsed = await this.parse(text) |
| 283 | parsed.forEach((parsedData, i) => { |
| 284 | const { pageContent } = parsedData |
| 285 | if (typeof pageContent !== 'string') { |
| 286 | throw new Error(`Expected string, at position ${i} got ${typeof pageContent}`) |
| 287 | } |
| 288 | }) |
| 289 | return parsed.map((parsedData, i) => { |
| 290 | const { pageContent, metadata: additionalMetadata } = parsedData |
| 291 | return new Document({ |
| 292 | pageContent, |
| 293 | metadata: |
| 294 | parsed.length === 1 |
| 295 | ? { ...metadata, ...additionalMetadata } |
| 296 | : { |
| 297 | ...metadata, |
| 298 | line: i + 1, |
| 299 | ...additionalMetadata |
| 300 | } |
| 301 | }) |
| 302 | }) |
| 303 | } |
| 304 | |
| 305 | static async imports(): Promise<{ |
| 306 | readFile: typeof ReadFileT |