(text: str, ignore_stopwords: bool = False, ignore_entities: bool = False, placeholders=None)
| 89 | return f.read().splitlines() |
| 90 | |
| 91 | def get_words(text: str, ignore_stopwords: bool = False, ignore_entities: bool = False, placeholders=None): |
| 92 | doc = get_doc(text.strip()) |
| 93 | |
| 94 | if not placeholders: |
| 95 | placeholders = [] |
| 96 | |
| 97 | tokens = [token for token in doc if token.text not in placeholders and not token.is_punct and len(token.text.strip()) > 0] |
| 98 | |
| 99 | if ignore_stopwords: |
| 100 | tokens = [token for token in tokens if not token.is_stop] |
| 101 | |
| 102 | if ignore_entities: |
| 103 | ents = [ent.text for ent in doc.ents] |
| 104 | tokens = [token for token in tokens if token.text not in ents] |
| 105 | |
| 106 | return set([token.lemma_.lower() for token in tokens]) |
| 107 | |
| 108 | def get_overlap(source: str, target: str, ignore_stopwords: bool = False, ignore_entities: bool = False): |
| 109 | source_words = get_words(source, ignore_stopwords=ignore_stopwords, ignore_entities=ignore_entities) |
no test coverage detected