()
| 16350 | |
| 16351 | |
| 16352 | def main() -> None: |
| 16353 | parser = argparse.ArgumentParser() |
| 16354 | parser.add_argument("-C", "--config-file", required=True) |
| 16355 | args = parser.parse_args() |
| 16356 | config = ConfigFile.load(args.config_file) |
| 16357 | model_path = config.model.resolve_model_path() |
| 16358 | loras = [ |
| 16359 | Model.LoraAdapter(path=lora.resolve_path(), scale=lora.scale) |
| 16360 | for lora in config.model.loras |
| 16361 | ] |
| 16362 | model = Model( |
| 16363 | model_path=model_path, |
| 16364 | model_alias=config.model.alias, |
| 16365 | chat_template=config.model.chat_template, |
| 16366 | loras=loras, |
| 16367 | n_gpu_layers=config.model.n_gpu_layers, |
| 16368 | split_mode=config.model.split_mode, |
| 16369 | main_gpu=config.model.main_gpu, |
| 16370 | tensor_split=config.model.tensor_split, |
| 16371 | vocab_only=config.model.vocab_only, |
| 16372 | use_mmap=config.model.use_mmap, |
| 16373 | use_mlock=config.model.use_mlock, |
| 16374 | kv_overrides=config.model.kv_overrides, |
| 16375 | n_ctx=config.model.n_ctx, |
| 16376 | n_batch=config.model.n_batch, |
| 16377 | n_ubatch=config.model.n_ubatch, |
| 16378 | n_seq_max=config.model.n_seq_max, |
| 16379 | n_threads=config.model.threads, |
| 16380 | n_threads_batch=config.model.threads_batch, |
| 16381 | rope_scaling_type=config.model.rope_scaling_type, |
| 16382 | pooling_type=config.model.pooling_type, |
| 16383 | attention_type=config.model.attention_type, |
| 16384 | embedding=config.model.embedding, |
| 16385 | rope_freq_base=config.model.rope_freq_base, |
| 16386 | rope_freq_scale=config.model.rope_freq_scale, |
| 16387 | yarn_ext_factor=config.model.yarn_ext_factor, |
| 16388 | yarn_attn_factor=config.model.yarn_attn_factor, |
| 16389 | yarn_beta_fast=config.model.yarn_beta_fast, |
| 16390 | yarn_beta_slow=config.model.yarn_beta_slow, |
| 16391 | yarn_orig_ctx=config.model.yarn_orig_ctx, |
| 16392 | offload_kqv=config.model.offload_kqv, |
| 16393 | flash_attn=config.model.flash_attn, |
| 16394 | op_offload=config.model.op_offload, |
| 16395 | swa_full=config.model.swa_full, |
| 16396 | no_perf=config.model.no_perf, |
| 16397 | type_k=config.model.type_k, |
| 16398 | type_v=config.model.type_v, |
| 16399 | kv_unified=config.model.kv_unified, |
| 16400 | max_seq_len=config.model.max_seq_len, |
| 16401 | max_output_tokens=config.model.max_output_tokens, |
| 16402 | draft_model=config.model.draft_model, |
| 16403 | draft_model_path=config.model.resolve_draft_model_path(), |
| 16404 | draft_model_num_pred_tokens=config.model.draft_model_num_pred_tokens, |
| 16405 | draft_model_max_ngram_size=config.model.draft_model_max_ngram_size, |
| 16406 | draft_model_top_k=config.model.draft_model_top_k, |
| 16407 | draft_model_p_min=config.model.draft_model_p_min, |
| 16408 | draft_model_max_batch_size=config.model.draft_model_max_batch_size, |
| 16409 | draft_model_threads=config.model.draft_model_threads, |
no test coverage detected
searching dependent graphs…