MCPcopy Create free account
hub / github.com/RASAAS/docmcp-knowledge / _remove_toc

Function _remove_toc

scripts/clean_fda_fulltext.py:59–91  ·  view source on GitHub ↗

Remove Table of Contents block.

(text: str)

Source from the content-addressed store, hash-verified

57
58
59def _remove_toc(text: str) -> str:
60 """Remove Table of Contents block."""
61 toc_patterns = [
62 r"Table of Contents\n(?:.*?\.{4,}.*?\n)+",
63 r"Table of Contents\s*\n(?:(?:[IVXLCDM]+\.\s|[A-Z]\.\s|\d+\.\s|Appendix).*?\n)+",
64 ]
65 for pat in toc_patterns:
66 text = re.sub(pat, "", text, flags=re.DOTALL)
67
68 lines = text.split("\n")
69 cleaned = []
70 skip = False
71 for line in lines:
72 stripped = line.strip()
73 if stripped == "Table of Contents":
74 skip = True
75 continue
76 if skip:
77 if re.search(r"\.{3,}\s*\d+\s*$", stripped):
78 continue
79 if re.match(r"^[IVXLCDM]+\.\s*$", stripped):
80 continue
81 if re.match(r"^[A-Z]\.\s*$", stripped):
82 continue
83 if re.match(r"^\d+\.\s*$", stripped):
84 continue
85 if stripped == "":
86 continue
87 if re.search(r"\.{3,}", stripped):
88 continue
89 skip = False
90 cleaned.append(line)
91 return "\n".join(cleaned)
92
93
94def _remove_preface_and_cover(text: str) -> str:

Callers 1

clean_fulltextFunction · 0.85

Calls 1

searchMethod · 0.45

Tested by

no test coverage detected