Function index_directory

tools/kb_scraper.py:120–166

Index all matching files in a directory recursively. Args: dirpath: Root directory to scan category: Category label for all files extensions: File extensions to include max_files: Safety cap (prevents runaway indexing of huge repos) Returns: Tot

(
    dirpath: str,
    category: str = "docs",
    extensions: tuple = DEFAULT_EXTENSIONS,
    max_files: int = 500,
)

def index_directory(
    dirpath: str,
    category: str = "docs",
    extensions: tuple = DEFAULT_EXTENSIONS,
    max_files: int = 500,
) -> int:
    """
    Index all matching files in a directory recursively.

    Args:
        dirpath: Root directory to scan
        category: Category label for all files
        extensions: File extensions to include
        max_files: Safety cap (prevents runaway indexing of huge repos)

    Returns:
        Total number of chunks indexed
    """
    root = Path(dirpath)
    if not root.is_dir():
        print(f"[kb_scraper] Not a directory: {dirpath}")
        return 0

    total_chunks = 0
    files_indexed = 0

    for path in sorted(root.rglob("*")):
        if files_indexed >= max_files:
            print(f"[kb_scraper] Reached max_files={max_files}, stopping.")
            break
        if not path.is_file():
            continue
        if path.suffix.lower() not in extensions:
            continue
        # Skip hidden dirs and common noise
        parts = path.parts
        if any(p.startswith(".") or p in ("__pycache__", "node_modules", ".git") for p in parts):
            continue

        chunks = index_file(str(path), category)
        if chunks:
            total_chunks += len(chunks)
            files_indexed += 1
            print(f"  [kb_scraper] {path.name}: {len(chunks)} chunks")

    print(f"[kb_scraper] Total: {total_chunks} chunks from {files_indexed} files in {dirpath}")
    return total_chunks


def list_indexed() -> list: