Function check_compatibility

src/whichllm/engine/compatibility.py:118–261

Check if a model+variant can run on the given hardware.

(
    model: ModelInfo,
    variant: GGUFVariant | None,
    hardware: HardwareInfo,
    context_length: int = 4096,
)

def check_compatibility(
    model: ModelInfo,
    variant: GGUFVariant | None,
    hardware: HardwareInfo,
    context_length: int = 4096,
) -> CompatibilityResult:
    """Check if a model+variant can run on the given hardware."""
    warnings: list[str] = []

    vram_required = estimate_vram(model, variant, context_length)

    usable_ram = effective_usable_ram(hardware.ram_bytes, hardware.ram_budget_bytes)

    # Determine best GPU
    best_gpu: GPUInfo | None = None
    best_gpu_available = 0
    gpu_available_values: list[int] = []
    candidate_gpus = _fit_candidate_gpus(hardware.gpus)
    ram_budget_active = hardware.ram_budget_bytes is not None
    for gpu in candidate_gpus:
        gpu_available = _gpu_available_memory(
            gpu, usable_ram, ram_budget_active=ram_budget_active
        )
        gpu_available_values.append(gpu_available)
        if best_gpu is None or gpu_available > best_gpu_available:
            best_gpu = gpu
            best_gpu_available = gpu_available

    vram_available = sum(gpu_available_values) if gpu_available_values else 0
    fit_vram_available, uses_multi_gpu, multi_gpu_effective_vram = (
        _multi_gpu_effective_vram(candidate_gpus, gpu_available_values, warnings)
    )
    if (
        len(candidate_gpus) > 1
        and not uses_multi_gpu
        and any(gpu.shared_memory or gpu.vendor == "apple" for gpu in candidate_gpus)
    ):
        vram_available = fit_vram_available
    offload_ram_available = (
        0
        if best_gpu and (best_gpu.shared_memory or best_gpu.vendor == "apple")
        else usable_ram
    )

    # Check compute capability for NVIDIA
    if best_gpu and best_gpu.vendor == "nvidia" and best_gpu.compute_capability:
        if best_gpu.compute_capability < MIN_COMPUTE_CAPABILITY_OLLAMA:
            warnings.append(
                f"Compute capability {best_gpu.compute_capability} is below "
                f"minimum {MIN_COMPUTE_CAPABILITY_OLLAMA} for Ollama"
            )

    # Flag legacy Kepler GPUs that have no CUDA support in modern llama.cpp.
    # They can still run, but only through the Vulkan backend on Linux.
    if best_gpu and _is_vulkan_only_gpu(best_gpu):
        warnings.append(
            "Legacy Kepler GPU: no CUDA support in modern llama.cpp; "
            "use the Vulkan backend (Linux) instead"