Function evaluate

clipeval/eval_zeroshot.py:29–47

(d, val_loader, templates, labels, model, tokenizer, classnorm=False)

def evaluate(d, val_loader, templates, labels, model, tokenizer, classnorm=False):
    print('Evaluating {}'.format(d))

    is_acc = d not in ['FGVCAircraft', 'OxfordPets', 'Caltech101', 'Flowers102', 'Kinetics700', 'HatefulMemes']

    acc_or_outputs = validate_zeroshot(val_loader, templates, labels, model, tokenizer, is_acc, classnorm)

    if d in ['FGVCAircraft', 'OxfordPets', 'Caltech101', 'Flowers102']:
        metric = mean_per_class(*acc_or_outputs)
    elif d == 'Kinetics700':
        top1, top5 = accuracy(*acc_or_outputs, topk=(1, 5))
        metric = (top1 + top5) / 2
        metric = metric.item()
    elif d == 'HatefulMemes':
        metric = roc_auc(*acc_or_outputs)
    else:
        metric = acc_or_outputs

    return metric


@torch.no_grad()