Step 4: Compare fetched data with current data layer.
(standards: list[dict])
| 427 | |
| 428 | |
| 429 | def show_diff(standards: list[dict]): |
| 430 | """Step 4: Compare fetched data with current data layer.""" |
| 431 | print("\n=== Step 4: Diff vs current data layer ===") |
| 432 | |
| 433 | if not standards: |
| 434 | standards = _load_from_cache() |
| 435 | if not standards: |
| 436 | return |
| 437 | |
| 438 | # Classify for display |
| 439 | for s in standards: |
| 440 | if "category" not in s or s["category"] == "other": |
| 441 | s["category"] = classify_standard(s) |
| 442 | |
| 443 | # Load current data layer |
| 444 | current_standards = {} |
| 445 | for json_file in DATA_DIR.glob("standards-*.json"): |
| 446 | with open(json_file, "r", encoding="utf-8") as f: |
| 447 | data = json.load(f) |
| 448 | for entry in data.get("entries", []): |
| 449 | current_standards[entry["number"]] = entry |
| 450 | |
| 451 | fetched_numbers = {s["number"] for s in standards} |
| 452 | current_numbers = set(current_standards.keys()) |
| 453 | |
| 454 | added = fetched_numbers - current_numbers |
| 455 | removed = current_numbers - fetched_numbers |
| 456 | common = fetched_numbers & current_numbers |
| 457 | |
| 458 | print(f"\n Current data layer: {len(current_numbers)} standards") |
| 459 | print(f" Fetched from EUR-Lex: {len(fetched_numbers)} standards") |
| 460 | print(f" ---") |
| 461 | print(f" New (to add): {len(added)}") |
| 462 | print(f" Removed (to delete): {len(removed)}") |
| 463 | print(f" Common (to verify): {len(common)}") |
| 464 | |
| 465 | if added: |
| 466 | print(f"\n +++ NEW STANDARDS ({len(added)}):") |
| 467 | for num in sorted(added): |
| 468 | s = next((x for x in standards if x["number"] == num), {}) |
| 469 | cat = s.get("category", "?") |
| 470 | review = " [NEEDS REVIEW]" if s.get("needs_review") else "" |
| 471 | print(f" + {num} [{cat}]{review}") |
| 472 | if s.get("title"): |
| 473 | print(f" {s['title'][:100]}") |
| 474 | |
| 475 | if removed: |
| 476 | print(f"\n --- REMOVED STANDARDS ({len(removed)}):") |
| 477 | for num in sorted(removed): |
| 478 | print(f" - {num}") |
| 479 | |
| 480 | needs_review = [s for s in standards if s.get("needs_review")] |
| 481 | if needs_review: |
| 482 | print(f"\n !!! NEEDS HUMAN REVIEW ({len(needs_review)}):") |
| 483 | for s in needs_review: |
| 484 | print(f" ! {s['number']} - title missing or from amendment") |
| 485 | |
| 486 | # Generate review report |
no test coverage detected