Function test_config

kt-kernel/python/cli/utils/tuna_engine.py:87–281

Test if a configuration with given num_gpu_experts works. Args: num_gpu_experts: Number of GPU experts to test model_path: Path to the model config: Configuration dict with all parameters verbose: Whether to show detailed logs Returns: (success:

(
    num_gpu_experts: int,
    model_path: Path,
    config: dict,
    verbose: bool = False,
)

def test_config(
    num_gpu_experts: int,
    model_path: Path,
    config: dict,
    verbose: bool = False,
) -> tuple[bool, float]:
    """
    Test if a configuration with given num_gpu_experts works.

    Args:
        num_gpu_experts: Number of GPU experts to test
        model_path: Path to the model
        config: Configuration dict with all parameters
        verbose: Whether to show detailed logs

    Returns:
        (success: bool, elapsed_time: float)
        - success: True if server starts and inference works
        - elapsed_time: Time taken for the test
    """
    start_time = time.time()

    # Use random port to avoid conflicts
    test_port = random.randint(30000, 40000)

    # Build command
    cmd = [
        sys.executable,
        "-m",
        "sglang.launch_server",
        "--model",
        str(model_path),
        "--port",
        str(test_port),
        "--host",
        "127.0.0.1",
        "--tensor-parallel-size",
        str(config["tensor_parallel_size"]),
        "--kt-num-gpu-experts",
        str(num_gpu_experts),
        "--max-total-tokens",
        str(config["max_total_tokens"]),
    ]

    # Add kt-kernel options
    if config.get("weights_path"):
        cmd.extend(["--kt-weight-path", str(config["weights_path"])])
    else:
        cmd.extend(["--kt-weight-path", str(model_path)])

    cmd.extend(
        [
            "--kt-cpuinfer",
            str(config.get("cpu_threads", 64)),
            "--kt-threadpool-count",
            str(config.get("numa_nodes", 2)),
            "--kt-method",
            config.get("kt_method", "AMXINT4"),