(file_path)
| 29 | |
| 30 | |
| 31 | def load_tsv_file(file_path): |
| 32 | data = [] |
| 33 | document_dict = defaultdict(list) |
| 34 | documents = [] |
| 35 | |
| 36 | with open(file_path, 'r', encoding='utf-8') as tsvfile: |
| 37 | reader = csv.reader(tsvfile, delimiter='\t') |
| 38 | headers = next(reader) # Get the header row |
| 39 | |
| 40 | for row in tqdm(reader, desc="Loading TSV", unit=" rows"): |
| 41 | # row: id text title |
| 42 | data.append(row[1]) |
| 43 | document_dict[row[2]].append(row[1]) |
| 44 | |
| 45 | for title, texts in document_dict.items(): |
| 46 | documents.append(" ".join(texts)) |
| 47 | print(len(documents)) |
| 48 | |
| 49 | return documents |
| 50 | |
| 51 | |
| 52 | def save_dict_pickle(dict, path): |
nothing calls this directly
no outgoing calls
no test coverage detected