Patched version that supports incremental updates for CommitScheduler.
(
api_self,
folder_path_or_files: Union[str, Path, List[str], List[Path]],
path_in_repo: str,
allow_patterns: Optional[Union[List[str], str]] = None,
ignore_patterns: Optional[Union[List[str], str]] = None,
)
| 29 | original_prepare = api._prepare_upload_folder |
| 30 | |
| 31 | def patched_prepare_upload_folder( |
| 32 | api_self, |
| 33 | folder_path_or_files: Union[str, Path, List[str], List[Path]], |
| 34 | path_in_repo: str, |
| 35 | allow_patterns: Optional[Union[List[str], str]] = None, |
| 36 | ignore_patterns: Optional[Union[List[str], str]] = None, |
| 37 | ) -> List[Union[tuple, list]]: |
| 38 | """ |
| 39 | Patched version that supports incremental updates for CommitScheduler. |
| 40 | """ |
| 41 | with scheduler_instance.lock: |
| 42 | if isinstance(folder_path_or_files, list): |
| 43 | raise ValueError( |
| 44 | 'Uploading multiple files or folders is not supported for scheduled commit.' |
| 45 | ) |
| 46 | elif os.path.isfile(folder_path_or_files): |
| 47 | raise ValueError( |
| 48 | 'Uploading file is not supported for scheduled commit.') |
| 49 | else: |
| 50 | folder_path = Path(folder_path_or_files).expanduser().resolve() |
| 51 | |
| 52 | logger.debug('Listing files to upload for scheduled commit.') |
| 53 | relpath_to_abspath = { |
| 54 | path.relative_to(folder_path).as_posix(): path |
| 55 | for path in sorted(folder_path.glob('**/*')) if path.is_file() |
| 56 | } |
| 57 | prefix = f"{path_in_repo.strip('/')}/" if path_in_repo else '' |
| 58 | |
| 59 | prepared_repo_objects = [] |
| 60 | files_to_track = {} |
| 61 | |
| 62 | for relpath in RepoUtils.filter_repo_objects( |
| 63 | relpath_to_abspath.keys(), |
| 64 | allow_patterns=allow_patterns, |
| 65 | ignore_patterns=ignore_patterns): |
| 66 | local_path = relpath_to_abspath[relpath] |
| 67 | stat = local_path.stat() |
| 68 | if scheduler_instance.last_uploaded.get( |
| 69 | local_path |
| 70 | ) is None or scheduler_instance.last_uploaded[ |
| 71 | local_path] != stat.st_mtime: |
| 72 | partial_file = PartialFileIO(local_path, stat.st_size) |
| 73 | prepared_repo_objects.append( |
| 74 | (prefix + relpath, partial_file)) |
| 75 | files_to_track[local_path] = stat.st_mtime |
| 76 | |
| 77 | scheduler_instance._pending_tracker_updates = files_to_track |
| 78 | |
| 79 | if not prepared_repo_objects: |
| 80 | logger.debug( |
| 81 | 'No changed files to upload for scheduled commit.') |
| 82 | |
| 83 | return prepared_repo_objects |
| 84 | |
| 85 | try: |
| 86 | api._prepare_upload_folder = types.MethodType( |
nothing calls this directly
no test coverage detected
searching dependent graphs…