(self)
| 1445 | self.batch.n_tokens += 1 |
| 1446 | |
| 1447 | def _try_decode_batch(self) -> bool: |
| 1448 | n_tokens = int(self.batch.n_tokens) |
| 1449 | if n_tokens <= 0: |
| 1450 | return True |
| 1451 | started_at = time.perf_counter() |
| 1452 | result = int(llama_cpp.llama_decode(self.ctx, self.batch)) |
| 1453 | self.decode_seconds_total += time.perf_counter() - started_at |
| 1454 | self.decode_calls_total += 1 |
| 1455 | self.decode_tokens_total += n_tokens |
| 1456 | if result != 0: |
| 1457 | self.decode_failures_total += 1 |
| 1458 | return False |
| 1459 | return True |
| 1460 | |
| 1461 | def _set_nextn_layer_offset(self, offset: int) -> None: |
| 1462 | if self.chain_heads: |
no outgoing calls
no test coverage detected