Method create_local_model_repo

llmware/resources.py:4206–4262

Pulls down and caches models from public llmware public S3 repo

(self, access_key=None,secret_key=None)

        return downloaded_files

    def create_local_model_repo(self, access_key=None,secret_key=None):

        """Pulls down and caches models from public llmware public S3 repo """

        # list of models retrieved from cloud repo
        models_retrieved = []

        # check for llmware path & create if not already set up
        if not os.path.exists(LLMWareConfig.get_llmware_path()):
            # if not explicitly set up by user, then create folder directory structure
            LLMWareConfig.setup_llmware_workspace()

        # confirm that local model repo path has been created
        local_model_repo_path = LLMWareConfig.get_model_repo_path()
        if not os.path.exists(local_model_repo_path):
            os.mkdir(local_model_repo_path)

        aws_repo_bucket = LLMWareConfig.get_config("llmware_public_models_bucket")

        # if specific model_list passed, then only retrieve models on the list

        bucket = boto3.resource('s3', aws_access_key_id=access_key,
                                aws_secret_access_key=secret_key).Bucket(aws_repo_bucket)

        files = bucket.objects.all()

        s3 = boto3.client('s3', aws_access_key_id=access_key, aws_secret_access_key=secret_key)

        # bucket = s3.Bucket(bucket_name)
        # files = bucket.objects.all()

        for file in files:

            name_parts = file.key.split(os.sep)

            # confirm that file.key is correctly structure as [0] model name, and [1] model component
            if len(name_parts) == 2:

                logger.info(f"update: identified models in model_repo: {name_parts}")

                if name_parts[0] and name_parts[1]:

                    model_folder = os.path.join(local_model_repo_path,name_parts[0])

                    if not os.path.exists(model_folder):
                        os.mkdir(model_folder)
                        models_retrieved.append(name_parts[0])

                    logger.info(f"update: downloading file from s3 bucket - "
                                f"{name_parts[1]} - {file.key}")

                    s3.download_file(aws_repo_bucket, file.key, os.path.join(model_folder,name_parts[1]))

        logger.info(f"update: created local model repository - {len(models_retrieved)} models retrieved - "
                    f" model list - {models_retrieved}")

        return models_retrieved