Function get_interactive_elements

sanitizer.py:4–69

Parses Android Accessibility XML and returns a lean list of interactive elements. Calculates center coordinates (x, y) for every clickable element.

(xml_content: str)

from typing import List, Dict, Optional

def get_interactive_elements(xml_content: str) -> List[Dict]:
    """
    Parses Android Accessibility XML and returns a lean list of interactive elements.
    Calculates center coordinates (x, y) for every clickable element.
    """
    try:
        root = ET.fromstring(xml_content)
    except ET.ParseError:
        print("⚠️ Error parsing XML. The screen might be loading.")
        return []

    elements = []
    
    # Recursively find all nodes
    for node in root.iter():
        # Filter: We only care about elements that are interactive or have information
        is_clickable = node.attrib.get("clickable") == "true"
        # Check for actual text input fields (not just focusable elements)
        element_class = node.attrib.get("class", "")
        is_editable = (
            "EditText" in element_class or
            "AutoCompleteTextView" in element_class or
            node.attrib.get("editable") == "true"
        )
        text = node.attrib.get("text", "")
        desc = node.attrib.get("content-desc", "")
        resource_id = node.attrib.get("resource-id", "")
        
        # Skip empty layout containers that do nothing
        if not is_clickable and not is_editable and not text and not desc:
            continue
            
        # Parse Bounds: "[140,200][400,350]" -> Center X, Y
        bounds = node.attrib.get("bounds")
        if bounds:
            try:
                # Extract coordinates
                coords = bounds.replace("][", ",").replace("[", "").replace("]", "").split(",")
                x1, y1, x2, y2 = map(int, coords)
                
                center_x = (x1 + x2) // 2
                center_y = (y1 + y2) // 2
                
                # Determine suggested action based on element type
                if is_editable:
                    suggested_action = "type"
                elif is_clickable:
                    suggested_action = "tap"
                else:
                    suggested_action = "read"

                element = {
                    "id": resource_id,
                    "text": text or desc,  # Fallback to content-desc if text is empty
                    "type": node.attrib.get("class", "").split(".")[-1],
                    "bounds": bounds,
                    "center": (center_x, center_y),
                    "clickable": is_clickable,