Function test_evaluate_multiple_textcat_separate

spacy/tests/test_language.py:186–235

Test that evaluate can evaluate multiple textcat components separately with custom scorers.

(en_vocab)

def test_evaluate_multiple_textcat_separate(en_vocab):
    """Test that evaluate can evaluate multiple textcat components separately
    with custom scorers."""

    def custom_textcat_score(examples, **kwargs):
        scores = Scorer.score_cats(
            examples,
            "cats",
            multi_label=False,
            **kwargs,
        )
        return {f"custom_{k}": v for k, v in scores.items()}

    @spacy.registry.scorers("test_custom_textcat_scorer")
    def make_custom_textcat_scorer():
        return custom_textcat_score

    nlp = Language(en_vocab)
    textcat = nlp.add_pipe(
        "textcat",
        config={"scorer": {"@scorers": "test_custom_textcat_scorer"}},
    )
    for label in ("POSITIVE", "NEGATIVE"):
        textcat.add_label(label)
    textcat_multilabel = nlp.add_pipe("textcat_multilabel")
    for label in ("FEATURE", "REQUEST", "BUG", "QUESTION"):
        textcat_multilabel.add_label(label)
    nlp.initialize()

    annots = {
        "cats": {
            "POSITIVE": 1.0,
            "NEGATIVE": 0.0,
            "FEATURE": 1.0,
            "QUESTION": 1.0,
            "POSITIVE": 1.0,
            "NEGATIVE": 0.0,
        }
    }
    doc = nlp.make_doc("hello world")
    example = Example.from_dict(doc, annots)
    scores = nlp.evaluate([example])
    # check custom scores for the textcat pipe
    assert "custom_cats_f_per_type" in scores
    labels = nlp.get_pipe("textcat").labels
    assert set(scores["custom_cats_f_per_type"].keys()) == set(labels)
    # check default scores for the textcat_multilabel pipe
    assert "cats_f_per_type" in scores
    labels = nlp.get_pipe("textcat_multilabel").labels
    assert set(scores["cats_f_per_type"].keys()) == set(labels)


def vector_modification_pipe(doc):