This class evaluates an Information Retrieval (IR) setting. Given a set of queries and a large corpus set. It will retrieve for each query the top-k most similar document.
| 56 | |
| 57 | |
| 58 | class APIEvaluator(SentenceEvaluator): |
| 59 | """ |
| 60 | This class evaluates an Information Retrieval (IR) setting. |
| 61 | Given a set of queries and a large corpus set. It will retrieve for each query the top-k most similar document. |
| 62 | """ |
| 63 | |
| 64 | def __init__( |
| 65 | self, |
| 66 | queries: Dict[str, str], # qid => query |
| 67 | corpus: Dict[str, str], # cid => doc |
| 68 | relevant_docs: Dict[str, Set[str]], # qid => Set[cid] |
| 69 | corpus_chunk_size: int = 5, |
| 70 | show_progress_bar: bool = True, |
| 71 | batch_size: int = 1, |
| 72 | write_csv: bool = True, |
| 73 | score_function=cos_sim, # Score function, higher=more similar |
| 74 | ): |
| 75 | self.queries_id = list(queries.keys()) |
| 76 | self.queries = [queries[qid] for qid in self.queries_id] |
| 77 | self.corpus_ids = list(corpus.keys()) |
| 78 | self.corpus = [corpus[cid] for cid in self.corpus_ids] |
| 79 | self.relevant_docs = relevant_docs |
| 80 | self.corpus_chunk_size = corpus_chunk_size |
| 81 | self.show_progress_bar = show_progress_bar |
| 82 | self.batch_size = batch_size |
| 83 | self.write_csv = write_csv |
| 84 | self.score_function = score_function |
| 85 | |
| 86 | self.csv_file: str = "Information-Retrieval_evaluation_results.csv" |
| 87 | self.csv_headers = [ |
| 88 | "epoch", |
| 89 | "steps", |
| 90 | "Average NDCG@1", |
| 91 | "Average NDCG@3", |
| 92 | "Average NDCG@5", |
| 93 | ] |
| 94 | |
| 95 | # for k in accuracy_at_k: |
| 96 | # self.csv_headers.append("Accuracy@{}".format(k)) |
| 97 | |
| 98 | def __call__( |
| 99 | self, |
| 100 | model, |
| 101 | output_path: str = None, |
| 102 | epoch: int = -1, |
| 103 | steps: int = -1, |
| 104 | *args, |
| 105 | **kwargs |
| 106 | ) -> float: |
| 107 | if epoch != -1: |
| 108 | out_txt = ( |
| 109 | " after epoch {}:".format(epoch) |
| 110 | if steps == -1 |
| 111 | else " in epoch {} after {} steps:".format(epoch, steps) |
| 112 | ) |
| 113 | else: |
| 114 | out_txt = ":" |
| 115 | logger.info("Information Retrieval Evaluation" + out_txt) |