Function createChunk

apps/sim/lib/knowledge/chunks/service.ts:106–244

(
  knowledgeBaseId: string,
  documentId: string,
  docTags: Record<string, string | number | boolean | Date | null>,
  chunkData: CreateChunkData,
  requestId: string,
  workspaceId?: string | null
)

 * Create a new chunk for a document
 */
export async function createChunk(
  knowledgeBaseId: string,
  documentId: string,
  docTags: Record<string, string | number | boolean | Date | null>,
  chunkData: CreateChunkData,
  requestId: string,
  workspaceId?: string | null
): Promise<ChunkData> {
  logger.info(`[${requestId}] Generating embedding for manual chunk`)
  const kbRow = await db
    .select({ embeddingModel: knowledgeBase.embeddingModel })
    .from(knowledgeBase)
    .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt)))
    .limit(1)
  if (kbRow.length === 0) {
    throw new Error('Knowledge base not found')
  }
  const kbEmbeddingModel = kbRow[0].embeddingModel
  const { embeddings } = await generateEmbeddings(
    [chunkData.content],
    kbEmbeddingModel,
    workspaceId
  )

  const tokenCount = estimateTokenCount(
    chunkData.content,
    getEmbeddingModelInfo(kbEmbeddingModel).tokenizerProvider
  )

  const chunkId = generateId()
  const now = new Date()

  // Use transaction to atomically get next index and insert chunk
  const newChunk = await db.transaction(async (tx) => {
    const activeDocument = await tx
      .select({ id: document.id })
      .from(document)
      .innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id))
      .where(
        and(
          eq(document.id, documentId),
          eq(document.knowledgeBaseId, knowledgeBaseId),
          isNull(document.archivedAt),
          isNull(document.deletedAt),
          isNull(knowledgeBase.deletedAt)
        )
      )
      .limit(1)

    if (activeDocument.length === 0) {
      throw new Error('Document not found')
    }

    // Get the next chunk index atomically within the transaction
    const lastChunk = await tx
      .select({ chunkIndex: embedding.chunkIndex })
      .from(embedding)
      .where(eq(embedding.documentId, documentId))