Computes TEDS score between the prediction and the ground truth of a batch of samples @params pred_json: {'FILENAME': 'HTML CODE', ...} @params true_json: {'FILENAME': {'html': 'HTML CODE'}, ...} @output: {'FILENAME': 'TEDS SCORE', ...}
(self, pred_json, true_json)
| 128 | return 0.0 |
| 129 | |
| 130 | def batch_evaluate(self, pred_json, true_json): |
| 131 | ''' Computes TEDS score between the prediction and the ground truth of |
| 132 | a batch of samples |
| 133 | @params pred_json: {'FILENAME': 'HTML CODE', ...} |
| 134 | @params true_json: {'FILENAME': {'html': 'HTML CODE'}, ...} |
| 135 | @output: {'FILENAME': 'TEDS SCORE', ...} |
| 136 | ''' |
| 137 | samples = true_json.keys() |
| 138 | if self.n_jobs == 1: |
| 139 | scores = [self.evaluate(pred_json.get(filename, ''), true_json[filename]['html']) for filename in tqdm(samples)] |
| 140 | else: |
| 141 | inputs = [{'pred': pred_json.get(filename, ''), 'true': true_json[filename]['html']} for filename in samples] |
| 142 | scores = parallel_process(inputs, self.evaluate, use_kwargs=True, n_jobs=self.n_jobs, front_num=1) |
| 143 | total_score_simple = 0 |
| 144 | num_simple = 0 |
| 145 | total_score_complex = 0 |
| 146 | num_complex = 0 |
| 147 | total_score = 0 |
| 148 | num_total = 0 |
| 149 | for filename,score in zip(samples, scores): |
| 150 | print(filename) |
| 151 | print(score) |
| 152 | print('') |
| 153 | if true_json[filename]['type'] == 'simple': |
| 154 | total_score_simple += score |
| 155 | num_simple += 1 |
| 156 | elif true_json[filename]['type'] == 'complex': |
| 157 | total_score_complex += score |
| 158 | num_complex += 1 |
| 159 | else: |
| 160 | raise ValueError('Unknown type: %s' % true_json[filename]['type']) |
| 161 | total_score += score |
| 162 | num_total += 1 |
| 163 | if num_simple > 0: |
| 164 | avg_score_simple = total_score_simple / num_simple |
| 165 | else: |
| 166 | avg_score_simple = 0 |
| 167 | if num_complex > 0: |
| 168 | avg_score_complex = total_score_complex / num_complex |
| 169 | else: |
| 170 | avg_score_complex = 0 |
| 171 | avg_score = total_score / num_total |
| 172 | print({'simple': (num_simple,avg_score_simple), 'complex': (num_complex,avg_score_complex), 'total': (num_total,avg_score)}) |
| 173 | |
| 174 | def main(): |
| 175 | parser = argparse.ArgumentParser(description="Evaluate page_to_markdown task") |
no test coverage detected