( messages: T[], config?: PreprocessConfig, logger: PreprocessLogger = defaultLogger )
| 18 | } |
| 19 | |
| 20 | export function preprocessMessages<T extends PreprocessableMessage>( |
| 21 | messages: T[], |
| 22 | config?: PreprocessConfig, |
| 23 | logger: PreprocessLogger = defaultLogger |
| 24 | ): T[] { |
| 25 | if (!config || !hasAnyEnabled(config)) return messages |
| 26 | if (messages.length === 0) return messages |
| 27 | |
| 28 | const inputCount = messages.length |
| 29 | let result: T[] = [...messages] |
| 30 | const applied: string[] = [] |
| 31 | |
| 32 | if (config.dataCleaning !== false) { |
| 33 | const cleaned = applyDataCleaning(result) |
| 34 | if (cleaned.changed > 0) { |
| 35 | result = cleaned.messages |
| 36 | applied.push(`dataCleaning: ${cleaned.changed} messages cleaned`) |
| 37 | } |
| 38 | } |
| 39 | |
| 40 | if (config.blacklistKeywords.length > 0) { |
| 41 | const before = result.length |
| 42 | result = applyBlacklistFilter(result, config.blacklistKeywords) |
| 43 | applied.push(`blacklist: ${before} → ${result.length} (-${before - result.length})`) |
| 44 | } |
| 45 | |
| 46 | if (config.denoise) { |
| 47 | const before = result.length |
| 48 | result = applyDenoise(result) |
| 49 | applied.push(`denoise: ${before} → ${result.length} (-${before - result.length})`) |
| 50 | } |
| 51 | |
| 52 | if (config.mergeConsecutive) { |
| 53 | const before = result.length |
| 54 | result = applyMergeConsecutive(result, config.mergeWindowSeconds ?? MERGE_WINDOW_DEFAULT) |
| 55 | applied.push(`merge: ${before} → ${result.length} (-${before - result.length})`) |
| 56 | } |
| 57 | |
| 58 | if (config.desensitize) { |
| 59 | const enabledRules = (config.desensitizeRules || []).filter((r) => r.enabled) |
| 60 | if (enabledRules.length > 0) { |
| 61 | result = applyDesensitize(result, enabledRules, logger) |
| 62 | applied.push(`desensitize: ${enabledRules.length} rules applied`) |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | logger.info('Preprocess', `Pipeline: ${inputCount} → ${result.length} messages`, { |
| 67 | strategies: applied, |
| 68 | }) |
| 69 | |
| 70 | return result |
| 71 | } |
| 72 | |
| 73 | function hasAnyEnabled(config: PreprocessConfig): boolean { |
| 74 | return ( |
no test coverage detected