Method preupload_weights

modelscope/hub/utils/aigc.py:310–379

Pre-upload aigc model weights to the LFS server. Server may require the sha256 of weights to be registered before creation. This method streams the weight file so the sha gets registered. Args: cookies: Optional requests-style cookies (CookieJar/dict). If provid

(self,
                          *,
                          cookies: Optional[object] = None,
                          timeout: int = 300,
                          headers: Optional[dict] = None,
                          endpoint: Optional[str] = None)

            return False

    def preupload_weights(self,
                          *,
                          cookies: Optional[object] = None,
                          timeout: int = 300,
                          headers: Optional[dict] = None,
                          endpoint: Optional[str] = None) -> None:
        """Pre-upload aigc model weights to the LFS server.

        Server may require the sha256 of weights to be registered before creation.
        This method streams the weight file so the sha gets registered.

        Args:
            cookies: Optional requests-style cookies (CookieJar/dict). If provided, preferred.
            timeout: Request timeout seconds.
            headers: Optional headers.
        """
        endpoint = endpoint or get_endpoint()
        endpoint_host: str = urlparse(endpoint.strip()).hostname.lstrip('www.')

        # https://lfs.modelscope.cn or https://pre-lfs.modelscope.cn
        base_url: str = f'{MODELSCOPE_URL_SCHEME}lfs.{endpoint_host}' if not endpoint_host.startswith('pre') \
            else f'{MODELSCOPE_URL_SCHEME}pre-lfs.{endpoint_host.lstrip("pre.")}'

        url: str = f'{base_url}/api/v1/models/aigc/weights'

        file_path = getattr(self, 'target_file', None) or self.model_path
        file_path = os.path.abspath(os.path.expanduser(file_path))
        if not os.path.isfile(file_path):
            raise ValueError(f'Pre-upload expects a file, got: {file_path}')

        cookies = dict(cookies) if cookies else None
        if cookies is None:
            raise ValueError('Token does not exist, please login first.')

        headers.update({'Cookie': f"m_session_id={cookies['m_session_id']}"})

        file_size = os.path.getsize(file_path)

        def read_in_chunks(file_object,
                           pbar,
                           chunk_size: int = 1 * 1024 * 1024):
            while True:
                ck = file_object.read(chunk_size)
                if not ck:
                    break
                pbar.update(len(ck))
                yield ck

        with tqdm(
                total=file_size,
                unit='B',
                unit_scale=True,
                dynamic_ncols=True,
                desc='[Pre-uploading] ') as pbar:
            with open(file_path, 'rb') as f:
                r = requests.put(
                    url,
                    headers=headers,