Function process_all_chunks

lpm_kernel/api/domains/documents/routes.py:144–191

Process chunks for all documents in batch

()

@document_bp.route("/documents/chunks/process", methods=["POST"])
def process_all_chunks():
    """Process chunks for all documents in batch"""
    try:
        config = Config.from_env()
        chunker = DocumentChunker(
            chunk_size=int(config.get("DOCUMENT_CHUNK_SIZE")),
            overlap=int(config.get("DOCUMENT_CHUNK_OVERLAP")),
        )

        documents = document_service.list_documents()
        processed, failed = 0, 0

        chunk_service = ChunkService()
        for doc in documents:
            try:
                if not doc.raw_content:
                    logger.warning(f"Document {doc.id} has no content, skipping...")
                    failed += 1
                    continue

                # Split into chunks and save
                chunks = chunker.split(doc.raw_content)
                for chunk in chunks:
                    chunk.document_id = doc.id
                    chunk_service.save_chunk(chunk)

                processed += 1
                logger.info(
                    f"Document {doc.id} processed: {len(chunks)} chunks created"
                )

            except Exception as e:
                logger.error(f"Failed to process document {doc.id}: {str(e)}")
                failed += 1

        return jsonify(
            APIResponse.success(
                data={
                    "total": len(documents),
                    "processed": processed,
                    "failed": failed,
                }
            )
        )

    except Exception as e:
        logger.error(f"Chunk processing failed: {str(e)}")
        return jsonify(APIResponse.error(message=f"Chunk processing failed: {str(e)}"))


@document_bp.route("/documents/<int:document_id>/chunk/embedding", methods=["POST"])