| 610 | if (ntok < 0) { |
| 611 | fprintf(stderr, "ds4: decode failed: %s\n", err); |
| 612 | ds4_session_free(session); |
| 613 | return 1; |
| 614 | } |
| 615 | } else { |
| 616 | size_t piece_len = 0; |
| 617 | char *piece = ds4_token_text(engine, token, &piece_len); |
| 618 | token_printer_write_text(&printer, piece, piece_len); |
| 619 | fflush(stdout); |
| 620 | free(piece); |
| 621 | generated++; |
| 622 | if (generated >= max_tokens || cli_interrupt_requested()) { |
| 623 | continue; |
| 624 | } |
| 625 | |
| 626 | cli_dist_busy_set(cfg, true); |
| 627 | int eval_rc = ds4_session_eval(session, token, err, sizeof(err)); |
| 628 | cli_dist_busy_set(cfg, false); |
| 629 | if (eval_rc != 0) { |
| 630 | fprintf(stderr, "ds4: decode failed: %s\n", err); |
| 631 | ds4_session_free(session); |
| 632 | return 1; |
| 633 | } |
| 634 | continue; |
| 635 | } |
| 636 | |
| 637 | bool stop = false; |
| 638 | for (int j = 0; j < ntok; j++) { |
| 639 | if (ds4_token_is_stop_for_think_mode(engine, toks[j], think_mode)) { |
| 640 | stop = true; |
| 641 | break; |
| 642 | } |
| 643 | size_t piece_len = 0; |
| 644 | char *piece = ds4_token_text(engine, toks[j], &piece_len); |
| 645 | token_printer_write_text(&printer, piece, piece_len); |
| 646 | fflush(stdout); |
| 647 | free(piece); |
| 648 | generated++; |
| 649 | if (generated >= max_tokens) break; |
| 650 | } |
| 651 | if (stop) break; |
| 652 | } |
| 653 | const double t_decode1 = cli_now_sec(); |
| 654 | generation_done(&printer); |
| 655 | if (cli_interrupt_requested()) cli_interrupt_clear(); |
| 656 | |
| 657 | const double prefill_s = t_prefill1 - t_prefill0; |
| 658 | const double decode_s = t_decode1 - t_decode0; |
| 659 | ds4_log(stderr, |
| 660 | DS4_LOG_TIMING, |
| 661 | "ds4: prefill: %.2f t/s, generation: %.2f t/s\n", |
| 662 | prefill_s > 0.0 ? (double)prompt->len / prefill_s : 0.0, |
| 663 | decode_s > 0.0 ? (double)generated / decode_s : 0.0); |
| 664 | |
| 665 | ds4_session_free(session); |
| 666 | return 0; |
| 667 | } |
| 668 | |
| 669 | static bool json_utf8_valid(const char *s, size_t n) { |
no test coverage detected