(s)
| 3 | from collections import Counter |
| 4 | |
| 5 | def preprocess_string(s): |
| 6 | cleaned = re.sub(r"[^\u4e00-\u9fa5a-zA-Z0-9\sàâäéèêëîïôöùûüçÀÂÄÉÈÊËÎÏÔÖÙÛÜÇ]", '', s) |
| 7 | if contains_chinese(cleaned): |
| 8 | pattern = re.compile(r"[\u4e00-\u9fa5a-zA-Z0-9àâäéèêëîïôöùûüçÀÂÄÉÈÊËÎÏÔÖÙÛÜÇ]") |
| 9 | s = ''.join(pattern.findall(s)) |
| 10 | |
| 11 | return s.strip() |
| 12 | |
| 13 | normalized = re.sub(r'\s+', ' ', cleaned) |
| 14 | |
| 15 | return normalized.strip() |
| 16 | |
| 17 | def clean_and_remove_hallucinations(texts): |
| 18 | # keywords_list can be added to process ocr results to a cleaner version |
no test coverage detected