(node_list, index, simplify=True, add_tags = True)
| 108 | return attrs_str |
| 109 | |
| 110 | def tree2text(node_list, index, simplify=True, add_tags = True): |
| 111 | text = '' |
| 112 | text_len = 0 |
| 113 | node = node_list[index] |
| 114 | is_leaf = True # if the node is a leaf node |
| 115 | # valid_child = 0 |
| 116 | #print(len(node_list)) |
| 117 | for child_idx in node.get_children(): |
| 118 | #print(child_idx) |
| 119 | if node_list[child_idx].get_tag() != 'text': |
| 120 | is_leaf = False |
| 121 | child_seq, child_text_len = tree2text(node_list, child_idx, simplify, add_tags) |
| 122 | # if child_text_len > 0: |
| 123 | # valid_child += 1 |
| 124 | text = text + ' ' + child_seq.strip() + ' ' |
| 125 | text_len += child_text_len |
| 126 | |
| 127 | text_len += len(node.get_str()) |
| 128 | |
| 129 | # if (not add_tags) or (node.get_tag() == 'text') or (simplify and text_len == 0) or (simplify and valid_child <= 1 and not is_leaf): |
| 130 | # text = text + f" {node.get_str()} " |
| 131 | # else: |
| 132 | if node.get_tag() == 'text': |
| 133 | text = text + f" {node.get_str()} " |
| 134 | else: |
| 135 | text = f"<{node.get_tag() + attrs_dict2str(node.get_attr())}> " + text + f" {node.get_str()} </{node.get_tag()}>" |
| 136 | |
| 137 | node_list[index].set_text(text) |
| 138 | node_list[index].set_tokens(list(text)) |
| 139 | return text, text_len |
| 140 | |
| 141 | def web2text(html_content, simplify=True, prettify = True, add_tags = True, attrs=[]): |
| 142 | parse_tree = web2tree(html_content, attrs=attrs) |
no test coverage detected