| 750 | |
| 751 | |
| 752 | class ChunksizeAdjuster: |
| 753 | def __init__( |
| 754 | self, |
| 755 | max_size=MAX_SINGLE_UPLOAD_SIZE, |
| 756 | min_size=MIN_UPLOAD_CHUNKSIZE, |
| 757 | max_parts=MAX_PARTS, |
| 758 | ): |
| 759 | self.max_size = max_size |
| 760 | self.min_size = min_size |
| 761 | self.max_parts = max_parts |
| 762 | |
| 763 | def adjust_chunksize(self, current_chunksize, file_size=None): |
| 764 | """Get a chunksize close to current that fits within all S3 limits. |
| 765 | |
| 766 | :type current_chunksize: int |
| 767 | :param current_chunksize: The currently configured chunksize. |
| 768 | |
| 769 | :type file_size: int or None |
| 770 | :param file_size: The size of the file to upload. This might be None |
| 771 | if the object being transferred has an unknown size. |
| 772 | |
| 773 | :returns: A valid chunksize that fits within configured limits. |
| 774 | """ |
| 775 | chunksize = current_chunksize |
| 776 | if file_size is not None: |
| 777 | chunksize = self._adjust_for_max_parts(chunksize, file_size) |
| 778 | return self._adjust_for_chunksize_limits(chunksize) |
| 779 | |
| 780 | def _adjust_for_chunksize_limits(self, current_chunksize): |
| 781 | if current_chunksize > self.max_size: |
| 782 | logger.debug( |
| 783 | "Chunksize greater than maximum chunksize. " |
| 784 | f"Setting to {self.max_size} from {current_chunksize}." |
| 785 | ) |
| 786 | return self.max_size |
| 787 | elif current_chunksize < self.min_size: |
| 788 | logger.debug( |
| 789 | "Chunksize less than minimum chunksize. " |
| 790 | f"Setting to {self.min_size} from {current_chunksize}." |
| 791 | ) |
| 792 | return self.min_size |
| 793 | else: |
| 794 | return current_chunksize |
| 795 | |
| 796 | def _adjust_for_max_parts(self, current_chunksize, file_size): |
| 797 | chunksize = current_chunksize |
| 798 | num_parts = int(math.ceil(file_size / float(chunksize))) |
| 799 | |
| 800 | while num_parts > self.max_parts: |
| 801 | chunksize *= 2 |
| 802 | num_parts = int(math.ceil(file_size / float(chunksize))) |
| 803 | |
| 804 | if chunksize != current_chunksize: |
| 805 | logger.debug( |
| 806 | "Chunksize would result in the number of parts exceeding the " |
| 807 | f"maximum. Setting to {chunksize} from {current_chunksize}." |
| 808 | ) |
| 809 |
no outgoing calls