| 25 | } |
| 26 | |
| 27 | protected async mergeSplits( |
| 28 | splits: string[], |
| 29 | separator: string |
| 30 | ): Promise<string[]> { |
| 31 | const docs: string[] = []; |
| 32 | const currentDoc: string[] = []; |
| 33 | let total = 0; |
| 34 | for (const d of splits) { |
| 35 | const _len = await this.lengthFunction(d); |
| 36 | if ( |
| 37 | total + _len + (currentDoc.length > 0 ? separator.length : 0) > |
| 38 | this.chunkSize |
| 39 | ) { |
| 40 | if (total > this.chunkSize) { |
| 41 | console.warn( |
| 42 | `Created a chunk of size ${total}, which is longer than the specified ${this.chunkSize}` |
| 43 | ); |
| 44 | } |
| 45 | if (currentDoc.length > 0) { |
| 46 | const doc = this.joinDocs(currentDoc, separator); |
| 47 | if (doc !== null) { |
| 48 | docs.push(doc); |
| 49 | } |
| 50 | while ( |
| 51 | total > this.chunkOverlap || |
| 52 | (total + _len > this.chunkSize && total > 0) |
| 53 | ) { |
| 54 | total -= await this.lengthFunction(currentDoc[0]); |
| 55 | currentDoc.shift(); |
| 56 | } |
| 57 | } |
| 58 | } |
| 59 | currentDoc.push(d); |
| 60 | total += _len; |
| 61 | } |
| 62 | const doc = this.joinDocs(currentDoc, separator); |
| 63 | if (doc !== null) { |
| 64 | docs.push(doc); |
| 65 | } |
| 66 | return docs; |
| 67 | } |
| 68 | |
| 69 | private joinDocs(docs: string[], separator: string): string | null { |
| 70 | const text = docs.join(separator).trim(); |