Function long_form_eval

scripts/eval/eval.py:1906–2139

Evaluate ASR model performance on long-form transcription tasks. This evaluation function handles variable-length audio segments (typically longer than 30 seconds) that require different processing than short-form evaluation. It uses the model's built-in transcribe() method with beam se

(
    batch_size: int,
    num_workers: int,
    ckpt: str,
    eval_set: Literal[
        "tedlium",
        "meanwhile",
        "rev16",
        "earnings21",
        "earnings22",
        "coraal",
        "kincaid46",
    ],
    log_dir: str,
    n_mels: int = DEFAULT_N_MELS,
    bootstrap: bool = False,
    wandb_log: bool = False,
    wandb_log_dir: str = "wandb",
    eval_dir: str = "data/eval",
    hf_token: Optional[str] = None,
)

def long_form_eval(
    batch_size: int,
    num_workers: int,
    ckpt: str,
    eval_set: Literal[
        "tedlium",
        "meanwhile",
        "rev16",
        "earnings21",
        "earnings22",
        "coraal",
        "kincaid46",
    ],
    log_dir: str,
    n_mels: int = DEFAULT_N_MELS,
    bootstrap: bool = False,
    wandb_log: bool = False,
    wandb_log_dir: str = "wandb",
    eval_dir: str = "data/eval",
    hf_token: Optional[str] = None,
) -> None:
    """Evaluate ASR model performance on long-form transcription tasks.

    This evaluation function handles variable-length audio segments (typically longer
    than 30 seconds) that require different processing than short-form evaluation.
    It uses the model's built-in transcribe() method with beam search and timestamp
    generation for optimal long-form performance.

    Key differences from short-form evaluation:
    - No mel-spectrogram preprocessing (handled internally by transcribe())
    - Uses beam search decoding with best-of selection
    - Processes full audio length without padding/trimming
    - Enables timestamp generation for alignment verification
    - Single-sample batch processing for stability

    Args:
        batch_size (int): Number of audio samples per batch (typically 1 for long-form)
        num_workers (int): Number of worker processes for data loading
        ckpt (str): Path to model checkpoint file
        eval_set (Literal): Name of evaluation dataset (see supported datasets below)
        log_dir (str): Directory for saving evaluation results and logs
        n_mels (int, optional): Number of mel-spectrogram bins (unused in long-form).
            Defaults to DEFAULT_N_MELS.
        bootstrap (bool, optional): Enable bootstrap sampling for confidence intervals.
            Defaults to False.
        wandb_log (bool, optional): Enable Weights & Biases logging. Defaults to False.
        wandb_log_dir (str, optional): Directory for WandB logs. Defaults to "wandb".
        eval_dir (str, optional): Root directory for evaluation datasets.
            Defaults to "data/eval".
        hf_token (Optional[str], optional): HuggingFace token for private datasets.
            Defaults to None.

    Supported Datasets:
        - tedlium: TED talks with longer audio segments
        - meanwhile: Meanwhile podcast corpus for long-form evaluation
        - rev16: Rev.com transcription corpus subset
        - earnings21: Corporate earnings call transcripts (2021)
        - earnings22: Corporate earnings call transcripts (2022)