Function evaluate

eval_utils/perplexity.py:36–147

(
    args,
    curr_epoch,
    model,
    accelerator,
    dataset_loader,
    logger,
    curr_train_iter=-1,
)

@torch.no_grad()
def evaluate(
    args,
    curr_epoch,
    model,
    accelerator,
    dataset_loader,
    logger,
    curr_train_iter=-1,
):
    
    model.eval()
    net_device = next(model.parameters()).device
    num_batches = len(dataset_loader)
    
    ### parse evaluation status
    if hasattr(dataset_loader.dataset, "dataset_name"):
        dataset_name = dataset_loader.dataset.dataset_name
    else:
        dataset_name = "default"
    task_name_prefix = dataset_name + '_'

    time_delta = SmoothedValue(window_size=10)
    
    accelerator.wait_for_everyone()
    
    epoch_str = f"[{curr_epoch}/{args.max_epoch}]" if curr_epoch > 0 else ""
    
    if accelerator.is_main_process:
        logger.log_messages("==" * 10)
        logger.log_messages(f"Evaluate Epoch [{curr_epoch}/{args.max_epoch}]")
        logger.log_messages("==" * 10)
    
    ### calculate perplexity
    neg_log_likelihood = []
    for curr_iter, batch_data_label in enumerate(dataset_loader):
        
        curr_time = time.time()
        
        # forward pass to calculate per-sequence negative log likelihood
        with accelerator.autocast():    
            outputs = model(batch_data_label, is_eval=True)
        # [(batch,), (batch,), ...]
        neg_log_likelihood.append(outputs['neg_log_likelihood'])

        ### log status
        time_delta.update(time.time() - curr_time)
        
        if accelerator.is_main_process and curr_iter % args.log_every == 0:
            mem_mb = torch.cuda.max_memory_allocated() / (1024 ** 2)
            moving_average_ppl = perplexity(neg_log_likelihood)
            logger.log_messages(
                '; '.join(
                    (
                        f"Evaluate {epoch_str}",
                        f"Batch [{curr_iter}/{num_batches}]",
                        f"perplexity: {moving_average_ppl:0.4f}",
                        f"Evaluating on iter: {curr_train_iter}",
                        f"Iter time {time_delta.avg:0.2f}",