Function chunkMessages

packages/node-runtime/src/semantic-index/chunker.ts:293–336

(input: ChunkMessagesInput)

}

export function chunkMessages(input: ChunkMessagesInput): ChunkResult {
  const config = input.config ?? DEFAULT_CHUNKER_CONFIG
  const chunkerConfigHash = computeChunkerConfigHash(config)

  const parents = segmentParents(input.messages, config)
  const chunks: ChildChunk[] = []
  let skippedCount = 0

  for (const parent of parents) {
    const parentId = deriveParentId({
      startMessageId: parent[0].id,
      endMessageId: parent[parent.length - 1].id,
      gapSeconds: config.parentGapSeconds,
      chunkerVersion: CHUNKER_VERSION,
      chunkerConfigHash,
    })

    for (const draft of buildChildDrafts(parent, config)) {
      const effectiveChars = sumEffectiveChars(draft)
      if (effectiveChars < config.semanticVoidSkipThreshold) {
        skippedCount++
        continue
      }
      const startMessageId = draft[0].id
      const endMessageId = draft[draft.length - 1].id
      const embeddingInput = buildEmbeddingInput(input.source, draft)
      chunks.push({
        localChunkId: `${parentId}#${startMessageId}-${endMessageId}`,
        parentId,
        startMessageId,
        endMessageId,
        startTs: draft[0].ts,
        endTs: draft[draft.length - 1].ts,
        messageCount: draft.length,
        effectiveChars,
        rawContentHash: rawContentHashOf(draft),
        embeddingInputHash: sha256Hex(embeddingInput),
        embeddingInput,
      })
    }
  }

  return { chunkerVersion: CHUNKER_VERSION, chunkerConfigHash, chunks, skippedCount, parentCount: parents.length }
}