| 108 | |
| 109 | @dataclass |
| 110 | class Classifier: |
| 111 | s: store.Store |
| 112 | overwrite: bool |
| 113 | filter_specs: list[tuple[str, str | None]] |
| 114 | small_only: bool |
| 115 | large_only: bool |
| 116 | size_threshold: int |
| 117 | normalized_inputs: set[str] |
| 118 | |
| 119 | _hash_to_canonical: dict[str, str] = field(init=False, default_factory=dict) |
| 120 | _filter_index: dict[str, tuple[str, models.ExtractionMeta]] = field( |
| 121 | init=False, default_factory=dict |
| 122 | ) |
| 123 | |
| 124 | def __post_init__(self) -> None: |
| 125 | # When --overwrite is set we want every canonical to be re-extracted, |
| 126 | # so we don't seed the dedup map from the DB (in-run dedup still applies). |
| 127 | if not self.overwrite: |
| 128 | for sha, source in self.s.known_sha256s().items(): |
| 129 | self._hash_to_canonical[_dedup_key(sha, source)] = source |
| 130 | if self.filter_specs: |
| 131 | self._filter_index = self.s.extractor_info_index() |
| 132 | |
| 133 | def classify(self, gz_path: str) -> Decision: |
| 134 | short_path = config.source_from_path(gz_path) |
| 135 | |
| 136 | if self.small_only or self.large_only: |
| 137 | size = os.path.getsize(gz_path) |
| 138 | if self.small_only and size > self.size_threshold: |
| 139 | return SizeSkip(gz_path, short_path, size, self.size_threshold, "small") |
| 140 | if self.large_only and size <= self.size_threshold: |
| 141 | return SizeSkip(gz_path, short_path, size, self.size_threshold, "large") |
| 142 | |
| 143 | if os.path.islink(gz_path): |
| 144 | canonical_path = os.path.realpath(gz_path) |
| 145 | canonical_source = config.source_from_path(canonical_path) |
| 146 | if canonical_source != short_path: |
| 147 | return Symlink( |
| 148 | gz_path=gz_path, |
| 149 | short_path=short_path, |
| 150 | canonical_source=canonical_source, |
| 151 | stale_in_db=self.s.has_manpage_source(short_path), |
| 152 | canonical_in_inputs=canonical_path in self.normalized_inputs, |
| 153 | ) |
| 154 | |
| 155 | if self.overwrite and self.filter_specs: |
| 156 | existing = self._filter_index.get(short_path) |
| 157 | if existing is not None: |
| 158 | stored_extractor, stored_meta = existing |
| 159 | if _matches_filter( |
| 160 | self.filter_specs, |
| 161 | stored_extractor, |
| 162 | stored_meta, |
| 163 | ): |
| 164 | # Matching row: queue for re-extraction. Deliberately skip |
| 165 | # the dedup branch — and don't seed _hash_to_canonical — |
| 166 | # so a same-hash sibling doesn't silently alias onto this |
| 167 | # row's stale parsed_manpages. |
no outgoing calls