(self, requests)
| 87 | raise NotImplementedError() |
| 88 | |
| 89 | def loglikelihood(self, requests): |
| 90 | res = [] |
| 91 | for context, continuation in tqdm(requests): |
| 92 | response = textsynth_completion( |
| 93 | url=self.api_url + "/v1/engines/" + self.engine + "/logprob", |
| 94 | headers={"Authorization": "Bearer " + self.api_key}, |
| 95 | json={"context": context, "continuation": continuation}, |
| 96 | ) |
| 97 | resp = response.json() |
| 98 | if "logprob" in resp: |
| 99 | logprob = resp["logprob"] |
| 100 | is_greedy = resp["is_greedy"] |
| 101 | res.append((logprob, is_greedy)) |
| 102 | else: |
| 103 | logger.error( |
| 104 | f"The following response does not contain `logprobs`. Got:\n{resp}" |
| 105 | ) |
| 106 | assert False |
| 107 | return res |
| 108 | |
| 109 | def loglikelihood_rolling(self, requests): |
| 110 | # TODO: The TextSynth API does not support tokenized inputs so we cannot |
nothing calls this directly
no test coverage detected