| 60 | f_kb.write(line) |
| 61 | |
| 62 | def test_eval(self): |
| 63 | root_dir = self.tmp_dir |
| 64 | params = { |
| 65 | 'cm_width': 10, |
| 66 | 'cm_depth': 10, |
| 67 | 'max_set': 100, |
| 68 | 'entity_emb_size': 64, |
| 69 | 'relation_emb_size': 64, |
| 70 | 'vocab_emb_size': 64, |
| 71 | 'train_entity_emb': False, |
| 72 | 'train_relation_emb': False, |
| 73 | 'intermediate_top_k': 10, |
| 74 | 'use_cm_sketch': True |
| 75 | } |
| 76 | |
| 77 | loader = data_loader.DataLoader( |
| 78 | params=params, name='query2box_ic', |
| 79 | root_dir=root_dir, |
| 80 | kb_file='kb.txt') |
| 81 | |
| 82 | q2b_metrics = emql_eval.Query2BoxMetrics(self.task, root_dir, loader) |
| 83 | ent1, rel1, ent2, rel2, rel3 = 0, 0, 1, 2, 4 |
| 84 | features = np.array([ent1, rel1, ent2, rel2, rel3]) |
| 85 | # Predictions that are not in hard_answers will be skipped for |
| 86 | # evaluation. We refer answers not in hard_answers as easy answers. |
| 87 | # For clarity, we copy over the all_answers and |
| 88 | # hard_answers of the query below: |
| 89 | # all_answers = {3, 4, 5} |
| 90 | # easy_answers = {3, 4} |
| 91 | # hard_answers = {5} |
| 92 | answer_ids = np.array([4, 2, 5]) |
| 93 | tf_prediction = {'query': features, 'answer_ids': answer_ids} |
| 94 | q2b_metrics.eval(tf_prediction) |
| 95 | |
| 96 | # hits@1 is 0 because the first non easy_answers is 2, but it's |
| 97 | # not in correct hard_answers. hits@3 and hits@10 is 1.0 because |
| 98 | # because the second non easy_answer 5 is in correct hard_answers. |
| 99 | self.assertEqual(q2b_metrics.metrics['hits@1'], 0.0) |
| 100 | self.assertEqual(q2b_metrics.metrics['hits@3'], 1.0) |
| 101 | self.assertEqual(q2b_metrics.metrics['hits@10'], 1.0) |
| 102 | self.assertEqual(q2b_metrics.metrics['mrr'], 0.5) |
| 103 | |
| 104 | |
| 105 | if __name__ == '__main__': |