Check repository consistency
(self, repair=False, max_duration=0)
| 291 | return info |
| 292 | |
| 293 | def check(self, repair=False, max_duration=0): |
| 294 | """Check repository consistency""" |
| 295 | |
| 296 | def log_error(msg): |
| 297 | nonlocal obj_corrupted |
| 298 | obj_corrupted = True |
| 299 | logger.error(f"Repo object {info.name} is corrupted: {msg}") |
| 300 | |
| 301 | def check_object(obj): |
| 302 | """Check if obj looks valid.""" |
| 303 | hdr_size = RepoObj.obj_header.size |
| 304 | obj_size = len(obj) |
| 305 | if obj_size >= hdr_size: |
| 306 | hdr = RepoObj.ObjHeader(*RepoObj.obj_header.unpack(obj[:hdr_size])) |
| 307 | meta = obj[hdr_size : hdr_size + hdr.meta_size] |
| 308 | if hdr.meta_size != len(meta): |
| 309 | log_error("metadata size incorrect.") |
| 310 | elif hdr.meta_hash != xxh64(meta).digest(): |
| 311 | log_error("metadata does not match checksum.") |
| 312 | data = obj[hdr_size + hdr.meta_size : hdr_size + hdr.meta_size + hdr.data_size] |
| 313 | if hdr.data_size != len(data): |
| 314 | log_error("data size incorrect.") |
| 315 | elif hdr.data_hash != xxh64(data).digest(): |
| 316 | log_error("data does not match checksum.") |
| 317 | else: |
| 318 | log_error("too small.") |
| 319 | |
| 320 | # TODO: progress indicator, ... |
| 321 | partial = bool(max_duration) |
| 322 | assert not (repair and partial) |
| 323 | mode = "partial" if partial else "full" |
| 324 | LAST_KEY_CHECKED = "cache/last-key-checked" |
| 325 | logger.info(f"Starting {mode} repository check") |
| 326 | if partial: |
| 327 | # continue a past partial check (if any) or from a checkpoint or start one from beginning |
| 328 | try: |
| 329 | last_key_checked = self.store.load(LAST_KEY_CHECKED).decode() |
| 330 | except StoreObjectNotFound: |
| 331 | last_key_checked = "" |
| 332 | else: |
| 333 | # start from the beginning and also forget about any potential past partial checks |
| 334 | last_key_checked = "" |
| 335 | try: |
| 336 | self.store.delete(LAST_KEY_CHECKED) |
| 337 | except StoreObjectNotFound: |
| 338 | pass |
| 339 | if last_key_checked: |
| 340 | logger.info(f"Skipping to keys after {last_key_checked}.") |
| 341 | else: |
| 342 | logger.info("Starting from beginning.") |
| 343 | t_start = time.monotonic() |
| 344 | t_last_checkpoint = t_start |
| 345 | objs_checked = objs_errors = 0 |
| 346 | chunks = ChunkIndex() |
| 347 | # we don't do refcounting anymore, neither we can know here whether any archive |
| 348 | # is using this object, but we assume that this is the case. |
| 349 | # As we don't do garbage collection here, this is not a problem. |
| 350 | # We also don't know the plaintext size, so we set it to 0. |