Function get_overlap

data_preprocessing/utils.py:108–112

(source: str, target: str, ignore_stopwords: bool = False, ignore_entities: bool = False)

    return set([token.lemma_.lower() for token in tokens])

def get_overlap(source: str, target: str, ignore_stopwords: bool = False, ignore_entities: bool = False):
    source_words = get_words(source, ignore_stopwords=ignore_stopwords, ignore_entities=ignore_entities)
    target_words = get_words(target, ignore_stopwords=ignore_stopwords, ignore_entities=ignore_entities)
    overlap = target_words.intersection(source_words)
    return overlap, (len(overlap) / len(target_words)) if overlap else 0, (len(overlap) / len(source_words)) if overlap else 0

def has_overlap(source: str, target: str, threshold: float, ignore_stopwords: bool = False, ignore_entities: bool = False):
    _, overlap_level1, overlap_level2 = get_overlap(source, target, ignore_stopwords=ignore_stopwords, ignore_entities=ignore_entities)