Function eval

benchmark.py:20–105

(checkpoint_path, config_yaml='config/audiosep_base.yaml')

)

def eval(checkpoint_path, config_yaml='config/audiosep_base.yaml'):

    log_dir = 'eval_logs'
    os.makedirs(log_dir, exist_ok=True)

    device = "cuda"
    
    configs = parse_yaml(config_yaml)

    # AudioSet Evaluators
    audioset_evaluator = AudioSetEvaluator()
    # AudioCaps Evaluator
    audiocaps_evaluator = AudioCapsEvaluator()
    # VGGSound+ Evaluator
    vggsound_evaluator = VGGSoundEvaluator()
    # Clotho Evaluator
    clotho_evaluator = ClothoEvaluator()
    # MUSIC Evaluator
    music_evaluator = MUSICEvaluator()
    # ESC-50 Evaluator
    esc50_evaluator = ESC50Evaluator()
    
    # Load model
    query_encoder = CLAP_Encoder().eval()

    pl_model = load_ss_model(
        configs=configs,
        checkpoint_path=checkpoint_path,
        query_encoder=query_encoder
    ).to(device)

    print(f'-------  Start Evaluation  -------')

    # evaluation on Clotho
    SISDR, SDRi = clotho_evaluator(pl_model)
    msg_clotho = "Clotho Avg SDRi: {:.3f}, SISDR: {:.3f}".format(SDRi, SISDR)
    print(msg_clotho)
    
    # evaluation on VGGSound+ (YAN)
    SISDR, SDRi = vggsound_evaluator(pl_model)
    msg_vgg = "VGGSound Avg SDRi: {:.3f}, SISDR: {:.3f}".format(SDRi, SISDR)
    print(msg_vgg)
    
    # evaluation on MUSIC
    SISDR, SDRi = music_evaluator(pl_model)
    msg_music = "MUSIC Avg SDRi: {:.3f}, SISDR: {:.3f}".format(SDRi, SISDR)
    print(msg_music)

    # evaluation on ESC-50
    SISDR, SDRi = esc50_evaluator(pl_model)
    msg_esc50 = "ESC-50 Avg SDRi: {:.3f}, SISDR: {:.3f}".format(SDRi, SISDR)
    print(msg_esc50)

    # evaluation on AudioSet
    stats_dict = audioset_evaluator(pl_model=pl_model)
    median_sdris = {}
    median_sisdrs = {}