Function scan

scanner.py:576–821

(projects_dir=None, projects_dirs=None, db_path=DB_PATH, verbose=True)

def scan(projects_dir=None, projects_dirs=None, db_path=DB_PATH, verbose=True):
    conn = get_db(db_path)
    init_db(conn)

    if projects_dirs:
        dirs_to_scan = [Path(d) for d in projects_dirs]
    elif projects_dir:
        dirs_to_scan = [Path(projects_dir)]
    else:
        dirs_to_scan = DEFAULT_PROJECTS_DIRS

    jsonl_files = []
    for d in dirs_to_scan:
        if not d.exists():
            continue
        if verbose:
            print(f"Scanning {d} ...")
        jsonl_files.extend(glob.glob(str(d / "**" / "*.jsonl"), recursive=True))
    jsonl_files.sort()

    # One-time topic backfill for DBs whose sessions predate topic support: fill
    # topics from title records in already-processed transcripts that an
    # incremental scan would otherwise never revisit. Runs once, gated by the
    # schema_meta 'topic_backfill_done' marker. It runs before the main loop, so
    # on a fresh DB the sessions table is still empty and this no-ops; only DBs
    # with pre-existing untitled sessions do real work.
    if _meta_get(conn, "topic_backfill_done") != "1":
        filled = _backfill_topics(conn, jsonl_files)
        _meta_set(conn, "topic_backfill_done", "1")
        conn.commit()
        if verbose and filled:
            print(f"Backfilled topic for {filled} existing session(s).")

    new_files = 0
    updated_files = 0
    skipped_files = 0
    total_turns = 0
    total_sessions = set()

    for filepath in jsonl_files:
        try:
            mtime = os.path.getmtime(filepath)
        except OSError:
            continue

        row = conn.execute(
            "SELECT mtime, lines FROM processed_files WHERE path = ?",
            (filepath,)
        ).fetchone()

        if row and abs(row["mtime"] - mtime) < 0.01:
            skipped_files += 1
            continue

        is_new = row is None
        if verbose:
            status = "NEW" if is_new else "UPD"
            print(f"  [{status}] {filepath}")