Function _create_model_from_files

bertopic/_bertopic.py:5018–5118

Create a BERTopic model from a variety of inputs. Arguments: topics: A dictionary containing topic metadata, including: - Topic representations, labels, sizes, custom labels, etc. params: BERTopic-specific hyperparams, including HF embedding_model ID

(
    topics: Mapping[str, Any],
    params: Mapping[str, Any],
    tensors: Mapping[str, np.array],
    ctfidf_tensors: Mapping[str, Any] | None = None,
    ctfidf_config: Mapping[str, Any] | None = None,
    images: Mapping[int, Any] | None = None,
    warn_no_backend: bool = True,
)

def _create_model_from_files(
    topics: Mapping[str, Any],
    params: Mapping[str, Any],
    tensors: Mapping[str, np.array],
    ctfidf_tensors: Mapping[str, Any] | None = None,
    ctfidf_config: Mapping[str, Any] | None = None,
    images: Mapping[int, Any] | None = None,
    warn_no_backend: bool = True,
):
    """Create a BERTopic model from a variety of inputs.

    Arguments:
        topics: A dictionary containing topic metadata, including:
                - Topic representations, labels, sizes, custom labels, etc.
        params: BERTopic-specific hyperparams, including HF embedding_model ID
                if given.
        tensors: The topic embeddings
        ctfidf_tensors: The c-TF-IDF representations
        ctfidf_config: The config for CountVectorizer and c-TF-IDF
        images: The images per topic
        warn_no_backend: Whether to warn the user if no backend is given
    """
    params["n_gram_range"] = tuple(params["n_gram_range"])

    if ctfidf_config is not None:
        ngram_range = ctfidf_config["vectorizer_model"]["params"]["ngram_range"]
        ctfidf_config["vectorizer_model"]["params"]["ngram_range"] = tuple(ngram_range)

    params["n_gram_range"] = tuple(params["n_gram_range"])

    # Select HF model through SentenceTransformers
    try:
        from sentence_transformers import SentenceTransformer

        embedding_model = select_backend(SentenceTransformer(params["embedding_model"]))
    except:  # noqa: E722
        embedding_model = BaseEmbedder()

        if warn_no_backend:
            logger.warning(
                "You are loading a BERTopic model without explicitly defining an embedding model."
                " If you want to also load in an embedding model, make sure to use"
                " `BERTopic.load(my_model, embedding_model=my_embedding_model)`."
            )

    if params.get("embedding_model") is not None:
        del params["embedding_model"]

    # Prepare our empty sub-models
    empty_dimensionality_model = BaseDimensionalityReduction()
    empty_cluster_model = BaseCluster()

    # Fit BERTopic without actually performing any clustering
    topic_model = BERTopic(
        embedding_model=embedding_model,
        umap_model=empty_dimensionality_model,
        hdbscan_model=empty_cluster_model,
        **params,