Compact sparse segments by copying data into new segments
(self, threshold)
| 718 | raise self.InsufficientFreeSpaceError(formatted_required, formatted_free) |
| 719 | |
| 720 | def compact_segments(self, threshold): |
| 721 | """Compact sparse segments by copying data into new segments""" |
| 722 | if not self.compact: |
| 723 | logger.debug("Nothing to do: compact empty") |
| 724 | return |
| 725 | index_transaction_id = self.get_index_transaction_id() |
| 726 | segments = self.segments |
| 727 | unused = [] # list of segments, that are not used anymore |
| 728 | |
| 729 | def complete_xfer(intermediate=True): |
| 730 | # complete the current transfer (when some target segment is full) |
| 731 | nonlocal unused |
| 732 | # commit the new, compact, used segments |
| 733 | segment = self.io.write_commit(intermediate=intermediate) |
| 734 | self.segments.setdefault(segment, 0) |
| 735 | self.compact[segment] += LoggedIO.header_fmt.size |
| 736 | logger.debug( |
| 737 | "complete_xfer: Wrote %scommit at segment %d", "intermediate " if intermediate else "", segment |
| 738 | ) |
| 739 | # get rid of the old, sparse, unused segments. free space. |
| 740 | for segment in unused: |
| 741 | logger.debug("complete_xfer: Deleting unused segment %d", segment) |
| 742 | count = self.segments.pop(segment) |
| 743 | if count != 0: |
| 744 | logger.warning( |
| 745 | "Corrupted segment reference count %d (expected 0) for segment %d - corrupted index or hints", |
| 746 | count, |
| 747 | segment, |
| 748 | ) |
| 749 | self.io.delete_segment(segment) |
| 750 | del self.compact[segment] |
| 751 | unused = [] |
| 752 | |
| 753 | logger.debug("Compaction started (threshold is %i%%).", threshold * 100) |
| 754 | pi = ProgressIndicatorPercent( |
| 755 | total=len(self.compact), msg="Compacting segments %3.0f%%", step=1, msgid="repository.compact_segments" |
| 756 | ) |
| 757 | for segment, freeable_space in sorted(self.compact.items()): |
| 758 | if not self.io.segment_exists(segment): |
| 759 | logger.warning("Segment %d not found, but listed in compaction data", segment) |
| 760 | self.compact.pop(segment, None) |
| 761 | self.segments.pop(segment, None) |
| 762 | pi.show() |
| 763 | self._send_log() |
| 764 | continue |
| 765 | segment_size = self.io.segment_size(segment) |
| 766 | freeable_ratio = 1.0 * freeable_space / segment_size |
| 767 | # we want to compact if: |
| 768 | # - we can free a considerable relative amount of space (freeable_ratio over some threshold) |
| 769 | if not (freeable_ratio > threshold): |
| 770 | logger.debug( |
| 771 | "Not compacting segment %d (maybe freeable: %2.2f%% [%d bytes])", |
| 772 | segment, |
| 773 | freeable_ratio * 100.0, |
| 774 | freeable_space, |
| 775 | ) |
| 776 | pi.show() |
| 777 | self._send_log() |
no test coverage detected