Loads PDF documents from specified directories.
| 7 | |
| 8 | |
| 9 | class DocumentLoader: |
| 10 | """ |
| 11 | Loads PDF documents from specified directories. |
| 12 | """ |
| 13 | |
| 14 | def __init__(self, corpus_paths: List[str]): |
| 15 | """ |
| 16 | Initialize document loader. |
| 17 | |
| 18 | Args: |
| 19 | corpus_paths: List of directory paths containing PDF files |
| 20 | """ |
| 21 | self.corpus_paths = [Path(path) for path in corpus_paths] |
| 22 | |
| 23 | def load_documents(self) -> List[Path]: |
| 24 | """ |
| 25 | Load all PDF documents from configured corpus paths. |
| 26 | |
| 27 | Returns: |
| 28 | List of PDF file paths |
| 29 | """ |
| 30 | collected_pdfs = [] |
| 31 | |
| 32 | for path in self.corpus_paths: |
| 33 | if not path.exists(): |
| 34 | logger.warning(f"Warning: Directory not found: {path}") |
| 35 | continue |
| 36 | |
| 37 | if not path.is_dir(): |
| 38 | logger.warning(f"Warning: Not a directory: {path}") |
| 39 | continue |
| 40 | |
| 41 | pdf_files = list(path.glob("*.pdf")) |
| 42 | |
| 43 | if not pdf_files: |
| 44 | logger.warning(f"Warning: No PDF files found in: {path}") |
| 45 | continue |
| 46 | |
| 47 | for pdf_file in pdf_files: |
| 48 | logger.info(f"Loaded: {pdf_file}") |
| 49 | collected_pdfs.append(pdf_file) |
| 50 | |
| 51 | return collected_pdfs |
| 52 | |
| 53 | def count_documents(self) -> int: |
| 54 | """ |
| 55 | Count total PDF documents across all corpus paths. |
| 56 | |
| 57 | Returns: |
| 58 | Total number of PDF documents |
| 59 | """ |
| 60 | total = 0 |
| 61 | |
| 62 | for path in self.corpus_paths: |
| 63 | if not path.exists() or not path.is_dir(): |
| 64 | continue |
| 65 | |
| 66 | pdf_files = list(path.glob("*.pdf")) |