| 9 | |
| 10 | # .tsv -> .jsonl |
| 11 | def convert_collection(collection_tsv, collection_json): |
| 12 | with open(collection_tsv, 'r') as input, open(collection_json, 'w') as output: |
| 13 | reader = csv.reader(input, delimiter="\t") # passage_nums = 25700592 |
| 14 | for i, row in enumerate(tqdm(reader)): |
| 15 | # structure for each row: ['id', 'text', 'title'] |
| 16 | if i == 0: |
| 17 | continue |
| 18 | qid, text = row[0], row[1] |
| 19 | title = ' '.join(row[2].split(' [SEP] ')) |
| 20 | obj = {"id": qid, "contents": " ".join([title, text])} |
| 21 | output.write(json.dumps(obj, ensure_ascii=False) + '\n') |
| 22 | |
| 23 | |
| 24 | def gen_train_test_files(train_gold_path, dev_gold_path, train_path, dev_path, output_train, output_dev): |