Uses RequestFactory to construct Requests and returns an iterable of Requests which will be sent to the LM. :param doc: The document as returned from training_docs, validation_docs, or test_docs. :param ctx: str The context string, generated by fewsho
(self, doc, ctx)
| 92 | return super().fewshot_examples(k, rnd) |
| 93 | |
| 94 | def construct_requests(self, doc, ctx): |
| 95 | """Uses RequestFactory to construct Requests and returns an iterable of |
| 96 | Requests which will be sent to the LM. |
| 97 | |
| 98 | :param doc: |
| 99 | The document as returned from training_docs, validation_docs, or test_docs. |
| 100 | :param ctx: str |
| 101 | The context string, generated by fewshot_context. This includes the natural |
| 102 | language description, as well as the few shot examples, and the question |
| 103 | part of the document for `doc`. |
| 104 | """ |
| 105 | lls = [] |
| 106 | for option in doc["options"]: |
| 107 | # Following Section 4.4 "Recurrent Language Models" in the CBT paper: |
| 108 | # "we rank candidate [option] c based on p(q1 . . . qk−1, c, qk+1 . . . ql) |
| 109 | # rather than simply p(q1 . . . qk−1, c)." |
| 110 | lls.append(rf.loglikelihood("", ctx.replace("XXXXX", option))[0]) |
| 111 | return lls |
| 112 | |
| 113 | def process_results(self, doc, results): |
| 114 | """Take a single document and the LM results and evaluates, returning a |
nothing calls this directly
no test coverage detected