(input_file)
| 23 | |
| 24 | |
| 25 | def gen_mask_samples(input_file): |
| 26 | |
| 27 | print(f'Start masking...') |
| 28 | print(f'data: {input_file}') |
| 29 | |
| 30 | # Read and process the input data |
| 31 | input_file = input_file |
| 32 | samples = [] |
| 33 | with open(input_file, 'r') as f: |
| 34 | for line in f: |
| 35 | sample = json.loads(line) # Parse JSON text |
| 36 | query = sample.get('query', '') # Extract key to embed |
| 37 | entities = sample.get('ner_results', {}).get('entities', []) |
| 38 | masked_query = mask_query_by_entities(query, entities) |
| 39 | sample['masked_query'] = masked_query |
| 40 | samples.append(sample) |
| 41 | |
| 42 | output_file = re.sub(r"(\w+)\.txt$", r"masked_\1.txt", input_file) |
| 43 | with open(output_file, 'w') as f: |
| 44 | for sample in samples: |
| 45 | f.write(json.dumps(sample) + '\n') |
| 46 | print(f'Saved masked samples to: {output_file}') |
| 47 | |
| 48 | |
| 49 | def gen_embed_and_save(input_file, version, key, model='transformer'): |
no test coverage detected