Vectorize directory
(&self, dir_path: &str, existing_hashes: Option<&std::collections::HashMap<String, String>>)
| 368 | |
| 369 | /// Vectorize directory |
| 370 | pub async fn vectorize_directory(&self, dir_path: &str, existing_hashes: Option<&std::collections::HashMap<String, String>>) -> Result<std::collections::HashMap<String, String>, Box<dyn std::error::Error>> { |
| 371 | info!("Starting vectorization of directory: {}", dir_path); |
| 372 | |
| 373 | let mut parser = CodeParser::new(); |
| 374 | let mut ts_parser = TreeSitterParser::new(); |
| 375 | |
| 376 | let path = Path::new(dir_path); |
| 377 | let files = parser.scan_directory(path); |
| 378 | |
| 379 | info!("Found {} files to vectorize", files.len()); |
| 380 | let mut total_vectors = 0; |
| 381 | let mut new_hashes = std::collections::HashMap::new(); |
| 382 | |
| 383 | for file_path in files { |
| 384 | // Calculate MD5 |
| 385 | let content = match fs::read_to_string(&file_path) { |
| 386 | Ok(c) => c, |
| 387 | Err(e) => { |
| 388 | error!("Failed to read file {}: {}", file_path.display(), e); |
| 389 | continue; |
| 390 | } |
| 391 | }; |
| 392 | let hash = format!("{:x}", md5::compute(&content)); |
| 393 | let file_key = file_path.to_string_lossy().to_string(); |
| 394 | |
| 395 | new_hashes.insert(file_key.clone(), hash.clone()); |
| 396 | |
| 397 | // Check if modified |
| 398 | if let Some(hashes) = existing_hashes { |
| 399 | if let Some(old_hash) = hashes.get(&file_key) { |
| 400 | if old_hash == &hash { |
| 401 | continue; |
| 402 | } |
| 403 | } |
| 404 | } |
| 405 | |
| 406 | match self.process_file_content(&file_path, &content, &mut ts_parser).await { |
| 407 | Ok(vectors) => { |
| 408 | total_vectors += vectors; |
| 409 | info!("File {} processed successfully with {} vectors", file_path.display(), vectors); |
| 410 | } |
| 411 | Err(e) => { |
| 412 | error!("Failed to process file {}: {}", file_path.display(), e); |
| 413 | } |
| 414 | } |
| 415 | } |
| 416 | |
| 417 | // Clean up embeddings for files that were deleted since last run |
| 418 | if let Some(hashes) = existing_hashes { |
| 419 | let mut deleted_count = 0; |
| 420 | for old_file in hashes.keys() { |
| 421 | if !new_hashes.contains_key(old_file) { |
| 422 | info!("Cleaning up embeddings for deleted file: {}", old_file); |
| 423 | if let Err(e) = self.delete_file_embeddings(old_file).await { |
| 424 | error!("Failed to delete LanceDB embeddings for {}: {}", old_file, e); |
| 425 | } else { |
| 426 | deleted_count += 1; |
| 427 | } |