( text, maxLength = 8000, startLength = 800, endLength = 600, tiktoken = true, )
| 29 | } |
| 30 | |
| 31 | export async function cropText( |
| 32 | text, |
| 33 | maxLength = 8000, |
| 34 | startLength = 800, |
| 35 | endLength = 600, |
| 36 | tiktoken = true, |
| 37 | ) { |
| 38 | const userConfig = await getUserConfig() |
| 39 | if (!userConfig.cropText) return text |
| 40 | |
| 41 | const k = modelNameToDesc( |
| 42 | userConfig.apiMode ? apiModeToModelName(userConfig.apiMode) : userConfig.modelName, |
| 43 | null, |
| 44 | userConfig.customModelName, |
| 45 | ).match(/[- (]*([0-9]+)k/)?.[1] |
| 46 | if (k) { |
| 47 | maxLength = Number(k) * 1000 |
| 48 | maxLength -= 100 + clamp(userConfig.maxResponseTokenLength, 1, maxLength - 2000) |
| 49 | } else { |
| 50 | maxLength -= 100 + clamp(userConfig.maxResponseTokenLength, 1, maxLength - 2000) |
| 51 | } |
| 52 | |
| 53 | const splits = text.split(/[,，。?？!！;；]/).map((s) => s.trim()) |
| 54 | const splitsLength = splits.map((s) => (tiktoken ? encode(s).length : s.length)) |
| 55 | const length = splitsLength.reduce((sum, length) => sum + length, 0) |
| 56 | |
| 57 | const cropLength = length - startLength - endLength |
| 58 | const cropTargetLength = maxLength - startLength - endLength |
| 59 | const cropPercentage = cropTargetLength / cropLength |
| 60 | const cropStep = Math.max(0, 1 / cropPercentage - 1) |
| 61 | |
| 62 | if (cropStep === 0) return text |
| 63 | |
| 64 | let croppedText = '' |
| 65 | let currentLength = 0 |
| 66 | let currentIndex = 0 |
| 67 | let currentStep = 0 |
| 68 | |
| 69 | for (; currentIndex < splits.length; currentIndex++) { |
| 70 | if (currentLength + splitsLength[currentIndex] + 1 <= startLength) { |
| 71 | croppedText += splits[currentIndex] + ',' |
| 72 | currentLength += splitsLength[currentIndex] + 1 |
| 73 | } else if (currentLength + splitsLength[currentIndex] + 1 + endLength <= maxLength) { |
| 74 | if (currentStep < cropStep) { |
| 75 | currentStep++ |
| 76 | } else { |
| 77 | croppedText += splits[currentIndex] + ',' |
| 78 | currentLength += splitsLength[currentIndex] + 1 |
| 79 | currentStep = currentStep - cropStep |
| 80 | } |
| 81 | } else { |
| 82 | break |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | let endPart = '' |
| 87 | let endPartLength = 0 |
| 88 | for (let i = splits.length - 1; endPartLength + splitsLength[i] <= endLength; i--) { |
no test coverage detected