(batch_size: int)
| 103 | |
| 104 | |
| 105 | def get_fd_config(batch_size: int): |
| 106 | fd_config = FDConfig( |
| 107 | model_config=ModelConfig( |
| 108 | { |
| 109 | "model": build_config_json(), |
| 110 | "max_model_len": 2048, |
| 111 | } |
| 112 | ), |
| 113 | parallel_config=ParallelConfig( |
| 114 | { |
| 115 | "tensor_parallel_size": 1, |
| 116 | "expert_parallel_size": 1, |
| 117 | "expert_parallel_rank": 0, |
| 118 | "data_parallel_size": 1, |
| 119 | } |
| 120 | ), |
| 121 | # quant_config=BlockWiseFP8Config(weight_block_size=[128, 128]), |
| 122 | scheduler_config=SchedulerConfig({"max_num_seqs": batch_size}), |
| 123 | cache_config=CacheConfig({}), |
| 124 | graph_opt_config=GraphOptimizationConfig({}), |
| 125 | load_config=LoadConfig({}), |
| 126 | ips="0.0.0.0", |
| 127 | ) |
| 128 | return fd_config |
| 129 | |
| 130 | |
| 131 | def test_sampler(): |
no test coverage detected