加载单个 JSONL 文件，支持日期过滤
(input_dir: Path, subdir: str, filename: str, start_date: Optional[str] = None, end_date: Optional[str] = None)
| 30 | |
| 31 | |
| 32 | def load_jsonl_file(input_dir: Path, subdir: str, filename: str, start_date: Optional[str] = None, end_date: Optional[str] = None) -> List[Dict]: |
| 33 | """加载单个 JSONL 文件，支持日期过滤""" |
| 34 | records = [] |
| 35 | |
| 36 | if subdir: |
| 37 | file_path = input_dir / subdir / filename |
| 38 | else: |
| 39 | file_path = input_dir / filename |
| 40 | |
| 41 | if not file_path.exists(): |
| 42 | print(f" [Warning] File not found: {file_path}") |
| 43 | return records |
| 44 | |
| 45 | with file_path.open("r", encoding="utf-8") as f: |
| 46 | for line in f: |
| 47 | line = line.strip() |
| 48 | if not line: |
| 49 | continue |
| 50 | |
| 51 | record = json.loads(line) |
| 52 | record_date = record.get("date", "") |
| 53 | |
| 54 | # 日期过滤 |
| 55 | if start_date and record_date < start_date: |
| 56 | continue |
| 57 | if end_date and record_date > end_date: |
| 58 | continue |
| 59 | |
| 60 | records.append(record) |
| 61 | |
| 62 | print(f" Loaded {len(records)} records from {file_path.relative_to(input_dir.parent)}") |
| 63 | return records |
| 64 | |
| 65 | |
| 66 | def load_all_jsonl_files(input_dir: Path, subdir: str, start_date: Optional[str] = None, end_date: Optional[str] = None) -> List[Dict]: |