Function main

scripts/rag_scraper.py:262–297

(argv: Optional[Sequence[str]] = None)

def main(argv: Optional[Sequence[str]] = None) -> int:
    args = parse_args(argv)
    cfg, specs = load_config(args.config, args.preset)
    if cfg:
        args.cache_dir = Path(str(cfg.get("cacheDir", args.cache_dir)))
        args.max_files_per_repo = int(cfg.get("maxFilesPerRepo", args.max_files_per_repo))
        args.max_snippets_per_repo = int(cfg.get("maxSnippetsPerRepo", args.max_snippets_per_repo))
        args.max_file_bytes = int(cfg.get("maxFileBytes", args.max_file_bytes))
        args.chunk_lines = int(cfg.get("chunkLines", args.chunk_lines))
        args.overlap = int(cfg.get("overlap", args.overlap))
        args.min_chars = int(cfg.get("minChars", args.min_chars))
        if cfg.get("languages") and not args.languages:
            args.languages = ",".join(cfg.get("languages")) if isinstance(cfg.get("languages"), list) else str(cfg.get("languages"))
    for repo in args.repo:
        specs.append(RepoSpec(url=repo, tags=[]))
    if not specs:
        print("No repositories configured. Use --preset starter/broad or add repos to repos.json.", file=sys.stderr)
        return 2

    args.out.parent.mkdir(parents=True, exist_ok=True)
    total = 0
    with args.out.open("w", encoding="utf-8") as fh:
        for spec in specs:
            print(f"scraping {spec.url}", file=sys.stderr)
            try:
                repo_root = ensure_repo(spec, args.cache_dir)
                count = 0
                for rec in scrape_repo(spec, repo_root, args):
                    fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
                    count += 1
                total += count
                print(f"  snippets: {count}", file=sys.stderr)
            except (subprocess.CalledProcessError, OSError) as exc:
                print(f"  skipped: {exc}", file=sys.stderr)
    print(json.dumps({"snippets": total, "out": str(args.out)}))
    return 0


if __name__ == "__main__":