MCPcopy Create free account
hub / github.com/google-research/language / test_eval

Method test_eval

language/emql/eval_test.py:62–102  ·  view source on GitHub ↗
(self)

Source from the content-addressed store, hash-verified

60 f_kb.write(line)
61
62 def test_eval(self):
63 root_dir = self.tmp_dir
64 params = {
65 'cm_width': 10,
66 'cm_depth': 10,
67 'max_set': 100,
68 'entity_emb_size': 64,
69 'relation_emb_size': 64,
70 'vocab_emb_size': 64,
71 'train_entity_emb': False,
72 'train_relation_emb': False,
73 'intermediate_top_k': 10,
74 'use_cm_sketch': True
75 }
76
77 loader = data_loader.DataLoader(
78 params=params, name='query2box_ic',
79 root_dir=root_dir,
80 kb_file='kb.txt')
81
82 q2b_metrics = emql_eval.Query2BoxMetrics(self.task, root_dir, loader)
83 ent1, rel1, ent2, rel2, rel3 = 0, 0, 1, 2, 4
84 features = np.array([ent1, rel1, ent2, rel2, rel3])
85 # Predictions that are not in hard_answers will be skipped for
86 # evaluation. We refer answers not in hard_answers as easy answers.
87 # For clarity, we copy over the all_answers and
88 # hard_answers of the query below:
89 # all_answers = {3, 4, 5}
90 # easy_answers = {3, 4}
91 # hard_answers = {5}
92 answer_ids = np.array([4, 2, 5])
93 tf_prediction = {'query': features, 'answer_ids': answer_ids}
94 q2b_metrics.eval(tf_prediction)
95
96 # hits@1 is 0 because the first non easy_answers is 2, but it's
97 # not in correct hard_answers. hits@3 and hits@10 is 1.0 because
98 # because the second non easy_answer 5 is in correct hard_answers.
99 self.assertEqual(q2b_metrics.metrics['hits@1'], 0.0)
100 self.assertEqual(q2b_metrics.metrics['hits@3'], 1.0)
101 self.assertEqual(q2b_metrics.metrics['hits@10'], 1.0)
102 self.assertEqual(q2b_metrics.metrics['mrr'], 0.5)
103
104
105if __name__ == '__main__':

Callers

nothing calls this directly

Calls 1

evalMethod · 0.95

Tested by

no test coverage detected