()
| 25 | |
| 26 | |
| 27 | def analyze(): |
| 28 | with open(INDEX_PATH) as f: |
| 29 | entries = json.load(f).get("entries", []) |
| 30 | with open(DISC_PATH) as f: |
| 31 | discovered = json.load(f) |
| 32 | |
| 33 | total = len(entries) |
| 34 | with_ft = sum(1 for e in entries if fulltext_exists(e)) |
| 35 | missing = [e for e in entries if not fulltext_exists(e)] |
| 36 | |
| 37 | print(f"Total entries: {total}") |
| 38 | print(f"With fulltext: {with_ft}") |
| 39 | print(f"Missing fulltext: {len(missing)}") |
| 40 | |
| 41 | has_urls = 0 |
| 42 | no_urls = 0 |
| 43 | by_doc_number = Counter() |
| 44 | |
| 45 | for e in missing: |
| 46 | eid = e["id"] |
| 47 | disc = discovered.get(eid, {}) |
| 48 | urls = disc.get("urls", []) |
| 49 | if not urls: |
| 50 | old = disc.get("url", "") |
| 51 | if old: |
| 52 | urls = [old] |
| 53 | if urls: |
| 54 | has_urls += 1 |
| 55 | else: |
| 56 | no_urls += 1 |
| 57 | dn = e.get("doc_number", "") |
| 58 | if dn: |
| 59 | by_doc_number[dn] += 1 |
| 60 | |
| 61 | print(f"\nMissing entries with discovered URLs: {has_urls}") |
| 62 | print(f"Missing entries without any URLs: {no_urls}") |
| 63 | |
| 64 | if by_doc_number: |
| 65 | print(f"\nMissing entries by doc_number (top 20):") |
| 66 | for dn, count in by_doc_number.most_common(20): |
| 67 | print(f" {count:3d} {dn}") |
| 68 | |
| 69 | has_source_url = sum(1 for e in missing if e.get("source_url")) |
| 70 | print(f"\nMissing entries with source_url: {has_source_url}") |
| 71 | |
| 72 | print(f"\nSample missing entries (first 10):") |
| 73 | for e in missing[:10]: |
| 74 | title = e.get("title", {}) |
| 75 | t = title.get("zh", "") if isinstance(title, dict) else str(title) |
| 76 | eid = e["id"] |
| 77 | disc = discovered.get(eid, {}) |
| 78 | urls = disc.get("urls", []) |
| 79 | source = e.get("source_url", "") |
| 80 | print(f" [{eid[:12]}] {t[:50]} | urls={len(urls)} source={bool(source)}") |
| 81 | |
| 82 | print(f"\nMissing entries WITH source_url (first 10):") |
| 83 | for e in missing: |
| 84 | if not e.get("source_url"): |
no test coverage detected