(self, requests)
| 96 | raise NotImplementedError() |
| 97 | |
| 98 | def loglikelihood(self, requests): |
| 99 | res = [] |
| 100 | for context, continuation in tqdm(requests): |
| 101 | response = textsynth_completion( |
| 102 | url=self.api_url + "/v1/engines/" + self.engine + "/logprob", |
| 103 | headers={"Authorization": "Bearer " + self.api_key}, |
| 104 | json={"context": context, "continuation": continuation}, |
| 105 | ) |
| 106 | resp = response.json() |
| 107 | if "logprob" in resp: |
| 108 | logprob = resp["logprob"] |
| 109 | is_greedy = resp["is_greedy"] |
| 110 | res.append((logprob, is_greedy)) |
| 111 | |
| 112 | self.cache_hook.add_partial( |
| 113 | "loglikelihood", (context, continuation), (logprob, is_greedy) |
| 114 | ) |
| 115 | else: |
| 116 | logger.error( |
| 117 | f"The following response does not contain `logprobs`. Got:\n{resp}" |
| 118 | ) |
| 119 | assert False |
| 120 | return res |
| 121 | |
| 122 | def loglikelihood_rolling(self, requests): |
| 123 | # TODO: The TextSynth API does not support tokenized inputs so we cannot |
nothing calls this directly
no test coverage detected