Function evaluate

single_eval.py:505–708

(gold, predict, db_dir, etype, kmaps, plug_value, keep_distinct, progress_bar_for_each_datapoint)

def evaluate(gold, predict, db_dir, etype, kmaps, plug_value, keep_distinct, progress_bar_for_each_datapoint):

    glist = []
    gseq_one = []
    l = gold
    if len(l.strip()) == 0:
        glist.append(gseq_one)
        gseq_one = []
    else:
        lstrip = l.strip().split('\t')
        gseq_one.append(lstrip)

    # include the last session
    # this was previously ignored in the SParC evaluation script
    # which might lead to slight differences in scores
    if len(gseq_one) != 0:
        glist.append(gseq_one)

    # spider formatting indicates that there is only one "single turn"
    # do not report "turn accuracy" for SPIDER
    include_turn_acc = len(glist) > 1

    l = predict
    plist = []
    pseq_one = []
    if len(l.strip()) == 0:
        plist.append(pseq_one)
        pseq_one = []
    else:
        pseq_one.append(l.strip().split('\t'))

    if len(pseq_one) != 0:
        plist.append(pseq_one)

    assert len(plist) == len(glist), "number of sessions must equal"

    evaluator = Evaluator()
    turns = ['turn 1', 'turn 2', 'turn 3', 'turn 4', 'turn > 4']
    levels = ['easy', 'medium', 'hard', 'extra', 'all', 'joint_all']

    partial_types = ['select', 'select(no AGG)', 'where', 'where(no OP)', 'group(no Having)',
                     'group', 'order', 'and/or', 'IUEN', 'keywords']
    entries = []
    scores = {}

    for turn in turns:
        scores[turn] = {'count': 0, 'exact': 0.}
        scores[turn]['exec'] = 0

    for level in levels:
        scores[level] = {'count': 0, 'partial': {}, 'exact': 0.}
        scores[level]['exec'] = 0
        for type_ in partial_types:
            scores[level]['partial'][type_] = {'acc': 0., 'rec': 0., 'f1': 0., 'acc_count': 0, 'rec_count': 0}

    debug = False
    if len(plist[0]) == 1:
        debug = True