| 121 | |
| 122 | |
| 123 | def strip_markdown(text: str) -> str: |
| 124 | text = text.replace("\r\n", "\n").replace("\r", "\n").lstrip("\ufeff") |
| 125 | text = re.sub(r"\A---\n.*?\n---\n", "\n", text, flags=re.S) |
| 126 | text = re.sub(r"```.*?```", "\n", text, flags=re.S) |
| 127 | text = re.sub(r"<!--.*?-->", "\n", text, flags=re.S) |
| 128 | |
| 129 | lines: list[str] = [] |
| 130 | for line in text.splitlines(): |
| 131 | stripped = line.strip() |
| 132 | if not stripped: |
| 133 | lines.append("") |
| 134 | continue |
| 135 | if stripped.startswith(("> 日期", "> Date", "> date")): |
| 136 | continue |
| 137 | if stripped.startswith("![") or stripped.lower().startswith("<img"): |
| 138 | continue |
| 139 | if re.fullmatch(r"\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?", stripped): |
| 140 | continue |
| 141 | lines.append(line) |
| 142 | |
| 143 | text = "\n".join(lines) |
| 144 | text = re.sub(r"!\[[^\]]*]\([^)]*\)", " ", text) |
| 145 | text = re.sub(r"\[([^\]]+)]\([^)]*\)", r"\1", text) |
| 146 | text = re.sub(r"https?://\S+", " ", text) |
| 147 | text = re.sub(r"<[^>]+>", " ", text) |
| 148 | text = re.sub(r"[`*_~>#-]+", " ", text) |
| 149 | text = re.sub(r"[ \t]+", " ", text) |
| 150 | text = re.sub(r"\n{3,}", "\n\n", text) |
| 151 | return text.strip() |
| 152 | |
| 153 | |
| 154 | def meaningful_chars(text: str) -> int: |