Tree Edit Distance basead Similarity
| 53 | |
| 54 | |
| 55 | class TEDS(object): |
| 56 | ''' Tree Edit Distance basead Similarity |
| 57 | ''' |
| 58 | def __init__(self, structure_only=False, n_jobs=1, ignore_nodes=None): |
| 59 | assert isinstance(n_jobs, int) and (n_jobs >= 1), 'n_jobs must be an integer greather than 1' |
| 60 | self.structure_only = structure_only |
| 61 | self.n_jobs = n_jobs |
| 62 | self.ignore_nodes = ignore_nodes |
| 63 | self.__tokens__ = [] |
| 64 | |
| 65 | def tokenize(self, node): |
| 66 | ''' Tokenizes table cells |
| 67 | ''' |
| 68 | self.__tokens__.append('<%s>' % node.tag) |
| 69 | if node.text is not None: |
| 70 | self.__tokens__ += list(node.text) |
| 71 | for n in node.getchildren(): |
| 72 | self.tokenize(n) |
| 73 | if node.tag != 'unk': |
| 74 | self.__tokens__.append('</%s>' % node.tag) |
| 75 | if node.tag != 'td' and node.tail is not None: |
| 76 | self.__tokens__ += list(node.tail) |
| 77 | |
| 78 | def load_html_tree(self, node, parent=None): |
| 79 | ''' Converts HTML tree to the format required by apted |
| 80 | ''' |
| 81 | global __tokens__ |
| 82 | if node.tag == 'td': |
| 83 | if self.structure_only: |
| 84 | cell = [] |
| 85 | else: |
| 86 | self.__tokens__ = [] |
| 87 | self.tokenize(node) |
| 88 | cell = self.__tokens__[1:-1].copy() |
| 89 | new_node = TableTree(node.tag, |
| 90 | int(node.attrib.get('colspan', '1')), |
| 91 | int(node.attrib.get('rowspan', '1')), |
| 92 | cell, *deque()) |
| 93 | else: |
| 94 | new_node = TableTree(node.tag, None, None, None, *deque()) |
| 95 | if parent is not None: |
| 96 | parent.children.append(new_node) |
| 97 | if node.tag != 'td': |
| 98 | for n in node.getchildren(): |
| 99 | self.load_html_tree(n, new_node) |
| 100 | if parent is None: |
| 101 | return new_node |
| 102 | |
| 103 | def evaluate(self, pred, true): |
| 104 | ''' Computes TEDS score between the prediction and the ground truth of a |
| 105 | given sample |
| 106 | ''' |
| 107 | if (not pred) or (not true): |
| 108 | return 0.0 |
| 109 | pred = "<html>" + pred + "</html>" |
| 110 | true = "<html>" + true + "</html>" |
| 111 | parser = html.HTMLParser(remove_comments=True, encoding='utf-8') |
| 112 | pred = html.fromstring(pred, parser=parser) |