Process single file content
(&self, file_path: &Path, _content: &str, ts_parser: &mut TreeSitterParser)
| 503 | |
| 504 | /// Process single file content |
| 505 | async fn process_file_content(&self, file_path: &Path, _content: &str, ts_parser: &mut TreeSitterParser) -> Result<usize, Box<dyn std::error::Error>> { |
| 506 | // Delete existing LanceDB embeddings for this file to prevent duplicates |
| 507 | self.delete_file_embeddings(&file_path.to_string_lossy()).await?; |
| 508 | |
| 509 | // Also clean up BM25 index entries for this file before re-adding |
| 510 | if let Some(ref bm25) = self.bm25_index { |
| 511 | let fp = file_path.to_string_lossy(); |
| 512 | if let Err(e) = bm25.remove_by_path(&fp).await { |
| 513 | error!("Failed to remove BM25 entries for {}: {}", fp, e); |
| 514 | } |
| 515 | } |
| 516 | |
| 517 | // Parse with TreeSitter |
| 518 | let symbols = ts_parser.parse_file(&file_path.to_path_buf())?; |
| 519 | |
| 520 | let mut vectors_created = 0; |
| 521 | let mut points = Vec::new(); |
| 522 | let mut bm25_chunks: Vec<CodeChunk> = Vec::new(); |
| 523 | // Collect cache-miss items for batch embedding (20-50x speedup) |
| 524 | let mut cache_miss_queue: Vec<(String, String, String, String, String, usize, usize, String)> = Vec::new(); |
| 525 | const BATCH_SIZE: usize = 20; |
| 526 | |
| 527 | for symbol in symbols { |
| 528 | // Extract data and drop guard immediately |
| 529 | let extracted = { |
| 530 | let symbol_guard = symbol.read(); |
| 531 | let symbol_ref = symbol_guard.as_ref(); |
| 532 | |
| 533 | match symbol_ref.symbol_type() { |
| 534 | crate::codegraph::treesitter::structs::SymbolType::StructDeclaration | |
| 535 | crate::codegraph::treesitter::structs::SymbolType::FunctionDeclaration => { |
| 536 | let symbol_info = symbol_ref.symbol_info_struct(); |
| 537 | let code_block = symbol_info.get_content_from_file_blocked() |
| 538 | .unwrap_or_else(|e| { |
| 539 | eprintln!("Warning: Failed to get content for {}: {}", symbol_ref.name(), e); |
| 540 | symbol_ref.name().to_string() |
| 541 | }); |
| 542 | |
| 543 | Some(( |
| 544 | code_block, |
| 545 | symbol_ref.name().to_string(), |
| 546 | format!("{:?}", symbol_ref.symbol_type()), |
| 547 | format!("{:?}", symbol_ref.language()), |
| 548 | symbol_ref.full_range().start_point.row, |
| 549 | symbol_ref.full_range().end_point.row, |
| 550 | )) |
| 551 | } |
| 552 | _ => None, |
| 553 | } |
| 554 | }; |
| 555 | |
| 556 | if let Some((code_block, name, symbol_type_str, language_str, start_row, end_row)) = extracted { |
| 557 | // P0: Skip short code blocks to improve retrieval quality |
| 558 | // See: docs/retrieval-quality-analysis.md |
| 559 | if code_block.trim().chars().count() < self.min_code_block_length { |
| 560 | debug!("Skipping short symbol '{}' ({} chars, min: {})", |
| 561 | name, code_block.len(), self.min_code_block_length); |
| 562 | continue; |