(
fd_runner,
model_path,
tensor_parallel_size,
max_num_seqs,
max_model_len,
max_tokens,
quantization,
load_choices,
prompts,
speculative_config={},
result_queue=None,
)
| 79 | |
| 80 | |
| 81 | def form_model_get_output_topp0( |
| 82 | fd_runner, |
| 83 | model_path, |
| 84 | tensor_parallel_size, |
| 85 | max_num_seqs, |
| 86 | max_model_len, |
| 87 | max_tokens, |
| 88 | quantization, |
| 89 | load_choices, |
| 90 | prompts, |
| 91 | speculative_config={}, |
| 92 | result_queue=None, |
| 93 | ): |
| 94 | try: |
| 95 | with fd_runner( |
| 96 | model_path, |
| 97 | tensor_parallel_size=tensor_parallel_size, |
| 98 | max_num_seqs=max_num_seqs, |
| 99 | max_model_len=max_model_len, |
| 100 | load_choices=load_choices, |
| 101 | quantization=quantization, |
| 102 | speculative_config=speculative_config, |
| 103 | ) as fd_model: |
| 104 | fd_outputs = fd_model.generate_topp0(prompts, max_tokens=max_tokens) |
| 105 | result_queue.put(fd_outputs) |
| 106 | except Exception: |
| 107 | print(f"Failed using {load_choices} loader to load model from {model_path}.") |
| 108 | traceback.print_exc() |
| 109 | pytest.fail(f"Failed to initialize LLM model from {model_path}") |
| 110 | |
| 111 | |
| 112 | def kill_process_on_port(port: int): |
nothing calls this directly
no test coverage detected