(self, requests, disable_tqdm=False, override_bs=None)
| 269 | return loglikelihoods |
| 270 | |
| 271 | def _loglikelihood_tokens(self, requests, disable_tqdm=False, override_bs=None): |
| 272 | # TODO: implement some kind of efficient-request-middleware that lumps together requests with the same context |
| 273 | res = [] |
| 274 | |
| 275 | def _collate(x): |
| 276 | # the negative sign on len(toks) sorts descending - this has a few advantages: |
| 277 | # - time estimates will always be over not underestimates, which is more useful for planning |
| 278 | # - to know the size of a batch when going through the list, you know the first one is always the batch |
| 279 | # padded context length. this is useful to simplify the batching logic and more importantly to make |
| 280 | # automatic adaptive batches much much easier to implement |
| 281 | # - any OOMs will happen right away rather than near the end |
| 282 | |
| 283 | toks = x[1] + x[2] |
| 284 | return -len(toks), tuple(toks) |
| 285 | |
| 286 | re_ord = utils.Reorderer(requests, _collate) |
| 287 | |
| 288 | reordered_requests = re_ord.get_reordered() |
| 289 | n_reordered_requests = len(reordered_requests) |
| 290 | |
| 291 | # automatic (variable) batch size detection for vectorization |
| 292 | # pull longest context sample from request |
| 293 | def _batch_scheduler(pos): |
| 294 | sched = pos // int(n_reordered_requests / self.batch_schedule) |
| 295 | if sched in self.batch_sizes: |
| 296 | return self.batch_sizes[sched] |
| 297 | print( |
| 298 | f"Passed argument batch_size = auto:{self.batch_schedule}. Detecting largest batch size" |
| 299 | ) |
| 300 | self.batch_sizes[sched] = self._detect_batch_size(reordered_requests, pos) |
| 301 | print(f"Determined largest batch size: {self.batch_sizes[sched]}") |
| 302 | return self.batch_sizes[sched] |
| 303 | |
| 304 | for chunk in utils.chunks( |
| 305 | tqdm(reordered_requests, disable=disable_tqdm), |
| 306 | n=self.batch_size |
| 307 | if self.batch_size != "auto" |
| 308 | else override_bs |
| 309 | if override_bs is not None |
| 310 | else 0, |
| 311 | fn=_batch_scheduler |
| 312 | if self.batch_size == "auto" and n_reordered_requests > 0 and not override_bs |
| 313 | else None, |
| 314 | ): |
| 315 | inps = [] |
| 316 | cont_toks_list = [] |
| 317 | inplens = [] |
| 318 | |
| 319 | padding_length = None |
| 320 | |
| 321 | # because vectorizing is annoying, we first convert each (context, continuation) pair to padded |
| 322 | # tensors, then we pack them together into a batch, call the model, and then pick it all apart |
| 323 | # again because vectorizing is annoying |
| 324 | |
| 325 | for _, context_enc, continuation_enc in chunk: |
| 326 | # sanity check |
| 327 | assert len(context_enc) > 0 |
| 328 | assert len(continuation_enc) > 0 |
no test coverage detected