Load valid L2 and L3 codes from classification catalog. Returns (valid_l2, valid_l3, l3_to_l1_map)
(catalog_path: Path)
| 43 | |
| 44 | |
| 45 | def load_valid_codes(catalog_path: Path) -> tuple[set, set, dict]: |
| 46 | """Load valid L2 and L3 codes from classification catalog. |
| 47 | Returns (valid_l2, valid_l3, l3_to_l1_map) |
| 48 | """ |
| 49 | with open(catalog_path, encoding="utf-8") as f: |
| 50 | catalog = json.load(f) |
| 51 | |
| 52 | valid_l2 = set() |
| 53 | valid_l3 = set() |
| 54 | code_to_l1 = {} |
| 55 | |
| 56 | for item in catalog.get("items", []): |
| 57 | l2 = item.get("l2_code", "") |
| 58 | l3 = item.get("code", "") |
| 59 | l1 = item.get("l1_code", "") |
| 60 | if l2: |
| 61 | valid_l2.add(l2) |
| 62 | code_to_l1[l2] = l1 |
| 63 | if l3: |
| 64 | valid_l3.add(l3) |
| 65 | code_to_l1[l3] = l1 |
| 66 | |
| 67 | return valid_l2, valid_l3, code_to_l1 |
| 68 | |
| 69 | |
| 70 | def _remove_substring_false_positives(codes: set, text: str, valid_l3: set) -> set: |