Function infer

llm_analysis/analysis.py:2251–2351

_summary_ Args: model_name (str, optional): model name to query the pre-defined `model_configs` dict, or model config json file path, if not found, query Hugging Face to construct ModelConfig. Defaults to "facebook_opt-1.3b". gpu_name (str, optional): gpu name to query the pre-d

(
    model_name="facebook_opt-1.3b",
    gpu_name="a100-sxm-40gb",
    dtype_name="w16a16e16",
    log_level="INFO",
    batch_size_per_gpu=1,
    ds_zero: int = 0,
    dp_size: int = 1,
    tp_size: int = 1,
    pp_size: int = 1,
    sp_size: int = None,
    seq_len=512,
    num_tokens_to_generate=32,
    use_kv_cache: bool = True,
    layernorm_dtype_bytes: int = BYTES_FP16,
    kv_cache_dtype_bytes: int = None,
    achieved_tflops: float = None,
    achieved_memory_bandwidth_GBs: float = None,
    flops_efficiency: float = None,
    hbm_memory_efficiency: float = None,
    intra_node_memory_efficiency=INTRA_NODE_MEMORY_EFFICIENCY,
    inter_node_memory_efficiency=INTER_NODE_MEMORY_EFFICIENCY,
    cost_per_gpu_hour: float = None,
    output_dir: str = None,
    output_file_suffix: str = "",
)

def infer(
    model_name="facebook_opt-1.3b",
    gpu_name="a100-sxm-40gb",
    dtype_name="w16a16e16",
    log_level="INFO",
    batch_size_per_gpu=1,
    ds_zero: int = 0,
    dp_size: int = 1,
    tp_size: int = 1,
    pp_size: int = 1,
    sp_size: int = None,
    seq_len=512,
    num_tokens_to_generate=32,
    use_kv_cache: bool = True,
    layernorm_dtype_bytes: int = BYTES_FP16,
    kv_cache_dtype_bytes: int = None,
    achieved_tflops: float = None,
    achieved_memory_bandwidth_GBs: float = None,
    flops_efficiency: float = None,
    hbm_memory_efficiency: float = None,
    intra_node_memory_efficiency=INTRA_NODE_MEMORY_EFFICIENCY,
    inter_node_memory_efficiency=INTER_NODE_MEMORY_EFFICIENCY,
    cost_per_gpu_hour: float = None,
    output_dir: str = None,
    output_file_suffix: str = "",
) -> dict:
    """_summary_

    Args:
        model_name (str, optional): model name to query the pre-defined `model_configs` dict, or model config json file path, if not found, query Hugging Face to construct ModelConfig. Defaults to "facebook_opt-1.3b".
        gpu_name (str, optional): gpu name to query the pre-defined `gpu_configs` dict. Defaults to "a100-sxm-40gb".
        dtype_name (str, optional): data type name to pre-defined `dtype_configs` dict. Defaults to "w16a16e16".
        log_level (str, optional): logging level. Defaults to "INFO".
        batch_size_per_gpu (int, optional): batch size per GPU. Defaults to 1.
        ds_zero (int, optional): which DeepSpeed ZeRO stage to use. See `DSZeRO`. Defaults to 0.
        dp_size (int, optional): data parallelism size. Defaults to None.
        tp_size (int, optional): tensor parallelism size. Defaults to 1.
        pp_size (int, optional): pipeline parallelism size. Defaults to 1.
        sp_size (int, optional): sequence parallelism size. Defaults to tp_size.
        seq_len (int, optional): input sequence length. Defaults to 512.
        num_tokens_to_generate (int, optional): number of tokens to generate for generative models. Defaults to 32.
        use_kv_cache (bool, optional): whether to use kv cache. Defaults to True.
        layernorm_dtype_bytes (int, optional): number of bytes in the data type for the layernorm activations. Defaults to BYTES_FP32. Often has to be at least FP16 in inference to maintain model accuracy.
        kv_cache_dtype_bytes (int, optional): number of bytes in the data type for the kv_cache. Defaults to None. Often has to be at least FP16 in inference to maintain model accuracy.
        achieved_tflops (float, optional): achieved TFLOPS per GPU. If specified, will override the flops_efficiency passed in. Defaults to None.
        achieved_memory_bandwidth_GBs (float, optional): achieved GPU memory bandwidth in GB/s. If specified, will override the hbm_memory_efficiency passed in. Defaults to None.
        flops_efficiency (float, optional): flops efficiency, ranging from 0 to 1. Defaults to None.
        hbm_memory_efficiency (float, optional): GPU HBM memory efficiency, ranging from 0 to 1. Defaults to HBM_MEMORY_EFFICIENCY.
        intra_node_memory_efficiency (float, optional):  intra-node memory efficiency, ranging from 0 to 1. Defaults to INTRA_NODE_MEMORY_EFFICIENCY.
        inter_node_memory_efficiency (float, optional):  inter-node memory efficiency, ranging from 0 to 1. Defaults to INTER_NODE_MEMORY_EFFICIENCY.
        cost_per_gpu_hour (float, optional): dollar cost per GPU hour. Defaults to None.
        output_dir (str, optional): if set to a directory path, write the return summary dict out to the directory with the setup. Defaults to None.. Defaults to None.
        output_file_suffix (str, optional): suffix of the output file. Defaults to "".

    Returns:
        dict: a summary dictionary of the inference analysis
    """