Local chat: start an ephemeral API server, then connect the TUI to it. This reuses the existing remote chat TUI with zero code duplication.
(ctx: &mut Context)
| 1125 | /// Local chat: start an ephemeral API server, then connect the TUI to it. |
| 1126 | /// This reuses the existing remote chat TUI with zero code duplication. |
| 1127 | pub async fn run_local(ctx: &mut Context) -> Result<()> { |
| 1128 | // Find a free port by temporarily binding to :0 |
| 1129 | let tmp = std::net::TcpListener::bind("127.0.0.1:0")?; |
| 1130 | let port = tmp.local_addr()?.port(); |
| 1131 | drop(tmp); |
| 1132 | |
| 1133 | let addr = format!("127.0.0.1:{port}"); |
| 1134 | let server_url = format!("http://{addr}"); |
| 1135 | |
| 1136 | eprintln!("starting local chat on {server_url}"); |
| 1137 | eprintln!("loading model (logs visible until TUI starts)...\n"); |
| 1138 | |
| 1139 | ctx.args.api = Some(addr); |
| 1140 | |
| 1141 | // Start the master on a background thread. Model loading logs are visible |
| 1142 | // on the terminal until the TUI takes over. |
| 1143 | let ctx_clone = ctx.clone(); |
| 1144 | std::thread::spawn(move || { |
| 1145 | let rt = tokio::runtime::Builder::new_current_thread() |
| 1146 | .enable_all() |
| 1147 | .build() |
| 1148 | .unwrap(); |
| 1149 | let local = tokio::task::LocalSet::new(); |
| 1150 | local.block_on(&rt, async move { |
| 1151 | if let Err(e) = super::run_master(ctx_clone).await { |
| 1152 | eprintln!("inference server error: {e}"); |
| 1153 | } |
| 1154 | }); |
| 1155 | }); |
| 1156 | |
| 1157 | // Wait for the API server to be ready (large MoE models with expert pre-warming can take minutes) |
| 1158 | let client = Client::new(); |
| 1159 | let mut ready = false; |
| 1160 | let max_wait_iters = 1200; // 600 seconds = 10 minutes |
| 1161 | for i in 0..max_wait_iters { |
| 1162 | tokio::time::sleep(std::time::Duration::from_millis(500)).await; |
| 1163 | match client.get(format!("{server_url}/v1/models")).send().await { |
| 1164 | Ok(resp) if resp.status().is_success() => { |
| 1165 | ready = true; |
| 1166 | break; |
| 1167 | } |
| 1168 | _ => { |
| 1169 | if i > 0 && i % 10 == 0 { |
| 1170 | eprintln!("still loading... ({:.0}s)", i as f64 * 0.5); |
| 1171 | } |
| 1172 | } |
| 1173 | } |
| 1174 | } |
| 1175 | |
| 1176 | if !ready { |
| 1177 | anyhow::bail!("local server did not start within {} seconds", max_wait_iters / 2); |
| 1178 | } |
| 1179 | |
| 1180 | // Suppress logs and take over terminal for TUI |
| 1181 | ::log::set_max_level(::log::LevelFilter::Off); |
| 1182 | |
| 1183 | // Clear any log output before entering TUI |
| 1184 | eprint!("\x1b[2J\x1b[H"); // clear screen + home cursor |
no test coverage detected