Function load_dpr_wiki

utils/load_data_util.py:17–28

(file_path)

def load_dpr_wiki(file_path):
    title_set = set()

    with open(file_path, 'r', encoding='utf-8') as tsvfile:
        reader = csv.reader(tsvfile, delimiter='\t')
        headers = next(reader)  # Get the header row

        for row in tqdm(reader, desc="Loading TSV", unit=" rows"):
            # row: id  text  title
            title_set.add(_normalize(row[2]))

    return title_set


def load_tsv_file(file_path):