Process some chunks (usually: recompress).
(repository, repo_objs, stats, recompress_ids, olevel)
| 35 | |
| 36 | |
| 37 | def process_chunks(repository, repo_objs, stats, recompress_ids, olevel): |
| 38 | """Process some chunks (usually: recompress).""" |
| 39 | compr_keys = stats["compr_keys"] |
| 40 | if compr_keys == 0: # work around defaultdict(int) |
| 41 | compr_keys = stats["compr_keys"] = set() |
| 42 | for id, chunk in zip(recompress_ids, repository.get_many(recompress_ids, read_data=True)): |
| 43 | old_size = len(chunk) |
| 44 | stats["old_size"] += old_size |
| 45 | meta, data = repo_objs.parse(id, chunk, ro_type=ROBJ_DONTCARE) |
| 46 | ro_type = meta.pop("type", None) |
| 47 | compr_old = meta["ctype"], meta["clevel"], meta.get("olevel", -1) |
| 48 | if olevel == -1: |
| 49 | # if the chunk was obfuscated, but should not be in future, remove related metadata |
| 50 | meta.pop("olevel", None) |
| 51 | meta.pop("psize", None) |
| 52 | chunk = repo_objs.format(id, meta, data, ro_type=ro_type) |
| 53 | compr_done = meta["ctype"], meta["clevel"], meta.get("olevel", -1) |
| 54 | if compr_done != compr_old: |
| 55 | # we actually changed something |
| 56 | repository.put(id, chunk, wait=False) |
| 57 | repository.async_response(wait=False) |
| 58 | stats["new_size"] += len(chunk) |
| 59 | compr_keys.add(compr_done) |
| 60 | stats[compr_done] += 1 |
| 61 | stats["recompressed_count"] += 1 |
| 62 | else: |
| 63 | # It might be that the old chunk used compression none or lz4 (for whatever reason, |
| 64 | # including the old compressor being a DecidingCompressor) AND we used a |
| 65 | # DecidingCompressor now, which did NOT compress like we wanted, but decided |
| 66 | # to use the same compression (and obfuscation) we already had. |
| 67 | # In this case, we just keep the old chunk and do not rewrite it - |
| 68 | # This is important to avoid rewriting such chunks **again and again**. |
| 69 | stats["new_size"] += old_size |
| 70 | compr_keys.add(compr_old) |
| 71 | stats[compr_old] += 1 |
| 72 | stats["kept_count"] += 1 |
| 73 | |
| 74 | |
| 75 | def format_compression_spec(ctype, clevel, olevel): |
no test coverage detected