(
repository, chunks, *, incremental=True, clear=False, force_write=False, delete_other=False, delete_these=None
)
| 725 | |
| 726 | |
| 727 | def write_chunkindex_to_repo_cache( |
| 728 | repository, chunks, *, incremental=True, clear=False, force_write=False, delete_other=False, delete_these=None |
| 729 | ): |
| 730 | # for now, we don't want to serialize the flags or the size, just the keys (chunk IDs): |
| 731 | cleaned_value = ChunkIndexEntry(flags=ChunkIndex.F_NONE, size=0) |
| 732 | chunks_to_write = ChunkIndex() |
| 733 | # incremental==True: |
| 734 | # the borghash code has no means to only serialize the F_NEW table entries, |
| 735 | # thus we copy only the new entries to a temporary table. |
| 736 | # incremental==False: |
| 737 | # maybe copying the stuff into a new ChunkIndex is not needed here, |
| 738 | # but for simplicity, we do it anyway. |
| 739 | for key, _ in chunks.iteritems(only_new=incremental): |
| 740 | chunks_to_write[key] = cleaned_value |
| 741 | with io.BytesIO() as f: |
| 742 | chunks_to_write.write(f) |
| 743 | data = f.getvalue() |
| 744 | logger.debug(f"caching {len(chunks_to_write)} chunks (incremental={incremental}).") |
| 745 | chunks_to_write.clear() # free memory of the temporary table |
| 746 | if clear: |
| 747 | # if we don't need the in-memory chunks index anymore: |
| 748 | chunks.clear() # free memory, immediately |
| 749 | new_hash = xxh64(data, seed=CHUNKINDEX_HASH_SEED).hexdigest() |
| 750 | cached_hashes = list_chunkindex_hashes(repository) |
| 751 | if force_write or new_hash not in cached_hashes: |
| 752 | # when an updated chunks index is stored into the cache, we also store its hash as part of the name. |
| 753 | # when a client is loading the chunks index from a cache, it has to compare its xxh64 |
| 754 | # hash against the hash in its name. if it is the same, the cache is valid. |
| 755 | # if it is different, the cache is either corrupted or out of date and has to be discarded. |
| 756 | # when some functionality is DELETING chunks from the repository, it has to delete |
| 757 | # all existing cache/chunks.* and maybe write a new, valid cache/chunks.<hash>, |
| 758 | # so that all clients will discard any client-local chunks index caches. |
| 759 | cache_name = f"cache/chunks.{new_hash}" |
| 760 | logger.debug(f"caching chunks index as {cache_name} in repository...") |
| 761 | repository.store_store(cache_name, data) |
| 762 | # we have successfully stored to the repository, so we can clear all F_NEW flags now: |
| 763 | chunks.clear_new() |
| 764 | # delete some not needed cached chunk indexes, but never the one we just wrote: |
| 765 | if delete_other: |
| 766 | delete_these = set(cached_hashes) - {new_hash} |
| 767 | elif delete_these: |
| 768 | delete_these = set(delete_these) - {new_hash} |
| 769 | else: |
| 770 | delete_these = set() |
| 771 | for hash in delete_these: |
| 772 | cache_name = f"cache/chunks.{hash}" |
| 773 | try: |
| 774 | repository.store_delete(cache_name) |
| 775 | except StoreObjectNotFound: |
| 776 | pass |
| 777 | if delete_these: |
| 778 | logger.debug(f"cached chunk indexes deleted: {delete_these}") |
| 779 | return new_hash |
| 780 | |
| 781 | |
| 782 | def read_chunkindex_from_repo_cache(repository, hash): |
no test coverage detected