(text: string)
| 11 | } |
| 12 | |
| 13 | export function cleanText(text: string): string { |
| 14 | return text |
| 15 | .replace(/\r\n/g, '\n') |
| 16 | .replace(/\r/g, '\n') |
| 17 | .replace(/\n{3,}/g, '\n\n') |
| 18 | .replace(/\t/g, ' ') |
| 19 | .replace(/ {2,}/g, ' ') |
| 20 | .trim() |
| 21 | } |
| 22 | |
| 23 | export function addOverlap(chunks: string[], overlapChars: number): string[] { |
| 24 | if (overlapChars <= 0 || chunks.length <= 1) { |