MCPcopy Index your code
hub / github.com/grapeot/devin.cursorrules / process_element

Function process_element

tools/web_scraper.py:60–94  ·  view source on GitHub ↗

Process an element and its children recursively.

(elem, depth=0)

Source from the content-addressed store, hash-verified

58 return False
59
60 def process_element(elem, depth=0):
61 """Process an element and its children recursively."""
62 if should_skip_element(elem):
63 return
64
65 # Handle text content
66 if hasattr(elem, 'text') and elem.text:
67 text = elem.text.strip()
68 if text and text not in seen_texts:
69 # Check if this is an anchor tag
70 if elem.tag == '{http://www.w3.org/1999/xhtml}a':
71 href = None
72 for attr, value in elem.items():
73 if attr.endswith('href'):
74 href = value
75 break
76 if href and not href.startswith(('#', 'javascript:')):
77 # Format as markdown link
78 link_text = f"[{text}]({href})"
79 result.append(" " * depth + link_text)
80 seen_texts.add(text)
81 else:
82 result.append(" " * depth + text)
83 seen_texts.add(text)
84
85 # Process children
86 for child in elem:
87 process_element(child, depth + 1)
88
89 # Handle tail text
90 if hasattr(elem, 'tail') and elem.tail:
91 tail = elem.tail.strip()
92 if tail and tail not in seen_texts:
93 result.append(" " * depth + tail)
94 seen_texts.add(tail)
95
96 # Start processing from the body tag
97 body = document.find('.//{http://www.w3.org/1999/xhtml}body')

Callers 1

parse_htmlFunction · 0.85

Calls 1

should_skip_elementFunction · 0.85

Tested by

no test coverage detected