(self)
| 1470 | self._set_nextn_layer_offset(0) |
| 1471 | |
| 1472 | def _decode_batch(self) -> None: |
| 1473 | n_tokens = int(self.batch.n_tokens) |
| 1474 | if n_tokens <= 0: |
| 1475 | return |
| 1476 | started_at = time.perf_counter() |
| 1477 | result = int(llama_cpp.llama_decode(self.ctx, self.batch)) |
| 1478 | self.decode_seconds_total += time.perf_counter() - started_at |
| 1479 | self.decode_calls_total += 1 |
| 1480 | self.decode_tokens_total += n_tokens |
| 1481 | if result != 0: |
| 1482 | self.decode_failures_total += 1 |
| 1483 | raise RuntimeError(f"MTP draft decode failed with code {result}") |
| 1484 | |
| 1485 | def _decode_batch_for_mtp_heads( |
| 1486 | self, |
no outgoing calls
no test coverage detected