Create the context from the parsed command line arguments.
(mut args: Args)
| 112 | impl Context { |
| 113 | /// Create the context from the parsed command line arguments. |
| 114 | pub fn from_args(mut args: Args) -> Result<Self> { |
| 115 | #[allow(unused_mut)] // mutated only with vibevoice feature |
| 116 | let mut dtype = parse_dtype_str(args.dtype.as_deref())?; |
| 117 | |
| 118 | let device = utils::get_inference_device(args.cpu, args.device) |
| 119 | .map_err(|e| anyhow!("can't attach to device: {:?}", e))?; |
| 120 | |
| 121 | // Disable cudarc event tracking for CUDA devices: cudarc's CudaStream::wait() |
| 122 | // rejects events from a different CudaContext, which breaks cross-device tensor |
| 123 | // transfers in multi-GPU setups. Safe since we use a single stream per device. |
| 124 | #[cfg(feature = "cuda")] |
| 125 | if let Device::Cuda(cuda_dev) = &device { |
| 126 | unsafe { |
| 127 | cuda_dev.disable_event_tracking(); |
| 128 | } |
| 129 | } |
| 130 | |
| 131 | log::info!( |
| 132 | "[{:?}] dtype={:?} device={:?} mem={}", |
| 133 | args.mode, |
| 134 | &dtype, |
| 135 | &device, |
| 136 | human_bytes::human_bytes( |
| 137 | memory_stats::memory_stats() |
| 138 | .map(|m| m.physical_mem) |
| 139 | .unwrap_or(0) as f64 |
| 140 | ) |
| 141 | ); |
| 142 | |
| 143 | let data_path = PathBuf::from(&args.model); |
| 144 | let data_path = if args.model_type == ModelType::ImageModel { |
| 145 | // Image models (SD, FLUX) download components on-demand via their own |
| 146 | // ModelFile::get() methods. Just use the path or repo ID as-is. |
| 147 | data_path |
| 148 | } else if !data_path.exists() { |
| 149 | if utils::hf::looks_like_hf_repo(&args.model) { |
| 150 | utils::hf::ensure_model_downloaded(&args.model)? |
| 151 | } else { |
| 152 | bail!("model path does not exist: {}", data_path.display()); |
| 153 | } |
| 154 | } else { |
| 155 | data_path |
| 156 | }; |
| 157 | |
| 158 | let mut topology = if let Some(topo) = args.topology_override.take() { |
| 159 | // Zero-config setup already built the topology |
| 160 | topo |
| 161 | } else if let Some(path) = &args.topology { |
| 162 | Topology::from_path(path, &args.model_type)? |
| 163 | } else { |
| 164 | log::warn!("no topology file specified, the entire model will be loaded"); |
| 165 | Topology::new() |
| 166 | }; |
| 167 | |
| 168 | // If the topology has nodes with no layer assignments, automatically |
| 169 | // distribute layers using the TFLOPS-proportional sharding algorithm. |
| 170 | // This lets users specify worker addresses without manual layer ranges. |
| 171 | if topology.needs_auto_sharding() |
nothing calls this directly
no test coverage detected