| 43 | # pass |
| 44 | # return list(match_words) |
| 45 | class Corpus(object): |
| 46 | def __init__(self, doc_name, source_id, corpus): |
| 47 | self.doc_name = doc_name |
| 48 | self.source_id = source_id |
| 49 | self.corpus = corpus |
| 50 | |
| 51 | def get_match_words(self, entities: list): |
| 52 | match_words = {"doc_name": self.doc_name, "source_id": self.source_id, "text": self.corpus, "match_words": []} |
| 53 | blacklist = ["table. ", "tab. ", "fig. ", "figure. "] |
| 54 | match_words["match_words"] = self.auto_match(entities) |
| 55 | |
| 56 | return match_words |
| 57 | |
| 58 | def auto_match(self, entities, lower_case=True): |
| 59 | entities = list(set(entities)) ## 去重 |
| 60 | match_words = set() |
| 61 | A = Automaton() |
| 62 | for entity in entities: |
| 63 | # 中英文兼容的小写转换, 替换keyword.lower()为自定义函数custom_lower_fast(keyword) |
| 64 | entity_key = utils.custom_lower_fast(entity) if lower_case else entity |
| 65 | # 检索entity，输出对应的(subject, entity) |
| 66 | A.add_word(entity_key, entity) |
| 67 | A.make_automaton() # 构造自动机 |
| 68 | # 初始化match_raw：idx记录文本id, text_len记录文本长度，unique_count记录匹配到的entity个数 |
| 69 | _text = utils.custom_lower_fast(self.corpus) if lower_case else self.corpus |
| 70 | try: |
| 71 | for end_index, entity in A.iter(_text): |
| 72 | end_index += 1 |
| 73 | start_index = end_index - len(entity) |
| 74 | # 如果检测到的不是单词边界，则跳过 |
| 75 | if utils.is_word_boundary(_text, start_index, end_index): |
| 76 | match_words.add(entity) |
| 77 | except Exception as e: |
| 78 | pass |
| 79 | return list(match_words) |
no outgoing calls
no test coverage detected