| 43 | park_target_name(target)); |
| 44 | return false; |
| 45 | } |
| 46 | |
| 47 | GenerateResult LayerSplitBackend::run_from_state(const GenerateRequest & req, |
| 48 | const DaemonIO & io, |
| 49 | int base_pos, |
| 50 | bool reset_state, |
| 51 | const std::vector<int32_t> & history_prefix) { |
| 52 | GenerateResult result; |
| 53 | if (!adapter_) { |
| 54 | result.fail(GenerateErrorCode::AdapterUnavailable); |
| 55 | return result; |
| 56 | } |
| 57 | |
| 58 | DaemonIO out_io = io.with_token_callback(req.on_token); |
| 59 | if (base_pos + (int)req.prompt.size() + req.n_gen + 1 > adapter_->max_context()) { |
| 60 | result.fail(GenerateErrorCode::ContextOverflow); |
| 61 | return result; |
| 62 | } |
| 63 | if (req.do_sample && req.sampler.needs_logit_processing() && |
| 64 | !adapter_->supports_cpu_sampling()) { |
| 65 | result.fail(GenerateErrorCode::SamplingUnsupported); |
| 66 | return result; |
| 67 | } |
| 68 | |
| 69 | adapter_->begin_request(req); |
| 70 | if (reset_state) adapter_->reset_request_state(); |
| 71 | |
| 72 | const int prompt_len = (int)req.prompt.size(); |
| 73 | const int adapter_chunk = adapter_->prefill_chunk_tokens(); |
| 74 | int last_tok = (base_pos > 0 && prompt_len == 0) |
| 75 | ? adapter_->current_last_token() |
| 76 | : -1; |
| 77 | int consumed = 0; |
| 78 | auto t_prefill_start = std::chrono::steady_clock::now(); |
| 79 | while (consumed < prompt_len) { |
| 80 | if (out_io.is_cancelled()) break; |
| 81 | int n_tokens = prompt_len - consumed; |
| 82 | if (adapter_chunk > 0 && n_tokens > adapter_chunk) { |
| 83 | n_tokens = adapter_chunk; |
| 84 | } |
| 85 | if (req.snap_pos >= 0 && req.snap_slot >= 0 && |
| 86 | req.snap_pos > base_pos + consumed && |
| 87 | req.snap_pos < base_pos + consumed + n_tokens) { |
| 88 | n_tokens = req.snap_pos - (base_pos + consumed); |
| 89 | } |
| 90 | std::vector<int32_t> chunk(req.prompt.begin() + consumed, |
| 91 | req.prompt.begin() + consumed + n_tokens); |
| 92 | if (!adapter_->prefill(chunk, base_pos + consumed, last_tok)) { |
| 93 | result.fail(GenerateErrorCode::PrefillFailed); |
| 94 | return result; |
| 95 | } |
| 96 | consumed += n_tokens; |
| 97 | if (req.snap_pos >= 0 && req.snap_slot >= 0 && |
| 98 | base_pos + consumed == req.snap_pos) { |
| 99 | if (adapter_->snapshot_save(req.snap_slot)) { |
| 100 | std::printf("[snap] inline slot=%d cur_pos=%d\n", |
| 101 | req.snap_slot, req.snap_pos); |
| 102 | std::fflush(stdout); |
nothing calls this directly
no test coverage detected