| 19 | |
| 20 | |
| 21 | class BasedTriviaQA(ConfigurableTask): |
| 22 | VERSION = "default" |
| 23 | DATASET_PATH = "hazyresearch/based_triviaqa" |
| 24 | DATASET_NAME = None |
| 25 | |
| 26 | def __init__(self): |
| 27 | super().__init__(config={'metadata': {'version': self.VERSION}}) |
| 28 | |
| 29 | def has_training_docs(self): |
| 30 | return False |
| 31 | |
| 32 | def has_validation_docs(self): |
| 33 | return True |
| 34 | |
| 35 | def has_test_docs(self): |
| 36 | return False |
| 37 | |
| 38 | def cleaned_context(self, context): |
| 39 | # remove "[PAR], [DOC], [TLE]" |
| 40 | context = re.sub(r"\[PAR\]|\[DOC\]|\[TLE\]", "", context) |
| 41 | return context |
| 42 | |
| 43 | def validation_docs(self): |
| 44 | return self.dataset["validation"] |
| 45 | |
| 46 | def doc_to_text(self, doc): |
| 47 | context = self.cleaned_context(doc["context"].strip()) |
| 48 | question = doc["question"].strip() |
| 49 | while(context.lower().endswith(question.lower())): |
| 50 | context = context[:-len(question)] |
| 51 | |
| 52 | out = ( |
| 53 | context.strip().strip(".") + ". " + doc["question"].strip() |
| 54 | ) |
| 55 | return out |
| 56 | |
| 57 | def should_decontaminate(self): |
| 58 | return True |
| 59 | |
| 60 | def doc_to_decontamination_query(self, doc): |
| 61 | return doc["context"] |
| 62 | |
| 63 | def doc_to_target(self, doc): |
| 64 | answer_list = doc['answers'] |
| 65 | if len(answer_list) > 0: |
| 66 | answer = answer_list[0] |
| 67 | else: |
| 68 | answer = "unanswerable" |
| 69 | return " " + answer |
| 70 | |
| 71 | def construct_requests(self, doc, ctx, **kwargs): |
| 72 | """Uses RequestFactory to construct Requests and returns an iterable of |
| 73 | Requests which will be sent to the LM. |
| 74 | |
| 75 | :param doc: |
| 76 | The document as returned from training_docs, validation_docs, or test_docs. |
| 77 | :param ctx: str |
| 78 | The context string, generated by fewshot_context. This includes the natural |
nothing calls this directly
no outgoing calls
no test coverage detected