(self, requests)
| 225 | return self._loglikelihood_tokens(new_reqs) |
| 226 | |
| 227 | def loglikelihood_rolling(self, requests): |
| 228 | # TODO: Implement caching once we've confirmed the perplexity implementation |
| 229 | |
| 230 | # automatic batch size detection for vectorization |
| 231 | adaptive_batch_size = None |
| 232 | if self.batch_size == "auto": |
| 233 | # using rolling window with maximum context |
| 234 | print("Passed argument batch_size = auto. Detecting largest batch size") |
| 235 | batch_size = self._detect_batch_size() |
| 236 | print(f"Determined Largest batch size: {batch_size}") |
| 237 | adaptive_batch_size = batch_size |
| 238 | |
| 239 | loglikelihoods = [] |
| 240 | for (string,) in tqdm(requests): |
| 241 | rolling_token_windows = list( |
| 242 | map( |
| 243 | utils.make_disjoint_window, |
| 244 | utils.get_rolling_token_windows( |
| 245 | token_list=self.tok_encode(string), |
| 246 | prefix_token=self.eot_token_id, |
| 247 | max_seq_len=self.max_length, |
| 248 | context_len=1, |
| 249 | ), |
| 250 | ) |
| 251 | ) |
| 252 | |
| 253 | rolling_token_windows = [(None,) + x for x in rolling_token_windows] |
| 254 | |
| 255 | # TODO: extract out this call so it only gets called once and also somehow figure out partial caching for |
| 256 | # that |
| 257 | string_nll = self._loglikelihood_tokens( |
| 258 | rolling_token_windows, |
| 259 | disable_tqdm=True, |
| 260 | override_bs=adaptive_batch_size, |
| 261 | ) |
| 262 | |
| 263 | # discard is_greedy |
| 264 | string_nll = [x[0] for x in string_nll] |
| 265 | |
| 266 | string_nll = sum(string_nll) |
| 267 | loglikelihoods.append(string_nll) |
| 268 | |
| 269 | return loglikelihoods |
| 270 | |
| 271 | def _loglikelihood_tokens(self, requests, disable_tqdm=False, override_bs=None): |
| 272 | # TODO: implement some kind of efficient-request-middleware that lumps together requests with the same context |
nothing calls this directly
no test coverage detected