Function run_indexing_with_progress

rag_system/api_server_with_progress.py:145–236

Enhanced indexing function with real-time progress tracking

(file_paths: List[str], session_id: str)

        ServerSentEventsHandler.send_event(self.session_id, "progress", event_data)

def run_indexing_with_progress(file_paths: List[str], session_id: str):
    """Enhanced indexing function with real-time progress tracking"""
    from rag_system.pipelines.indexing_pipeline import IndexingPipeline
    from rag_system.utils.ollama_client import OllamaClient
    import json
    
    try:
        # Send initial status
        ServerSentEventsHandler.send_event(session_id, "status", {
            "message": "Initializing indexing pipeline...",
            "session_id": session_id
        })
        
        # Load configuration
        config_file = "batch_indexing_config.json"
        try:
            with open(config_file, 'r') as f:
                config = json.load(f)
        except FileNotFoundError:
            # Fallback to default config
            config = {
                "embedding_model_name": "Qwen/Qwen3-Embedding-0.6B",
                "indexing": {
                    "embedding_batch_size": 50,
                    "enrichment_batch_size": 10,
                    "enable_progress_tracking": True
                },
                "contextual_enricher": {"enabled": True, "window_size": 1},
                "retrievers": {
                    "dense": {"enabled": True, "lancedb_table_name": "default_text_table"},
                    "bm25": {"enabled": True, "index_name": "default_bm25_index"}
                },
                "storage": {
                    "chunk_store_path": "./index_store/chunks/chunks.pkl",
                    "lancedb_uri": "./index_store/lancedb",
                    "bm25_path": "./index_store/bm25"
                }
            }
        
        # Initialize components
        ollama_client = OllamaClient()
        ollama_config = {
            "generation_model": "llama3.2:1b",
            "embedding_model": "mxbai-embed-large"
        }
        
        # Create enhanced pipeline
        pipeline = IndexingPipeline(config, ollama_client, ollama_config)
        
        # Create progress tracker for the overall process
        total_steps = 6  # Rough estimate of pipeline steps
        step_tracker = RealtimeProgressTracker(total_steps, "Document Indexing", session_id)
        
        with timer("Complete Indexing Pipeline"):
            try:
                # Step 1: Document Processing
                step_tracker.update(1, current_step="Processing documents...")