(input_values,queue,queue_inforgather_train,queue_inforgather_evluate,file_train_path)
| 77 | |
| 78 | |
| 79 | def process_task4(input_values,queue,queue_inforgather_train,queue_inforgather_evluate,file_train_path): |
| 80 | # 执行任务，并返回结果列表 |
| 81 | train_samples = [] |
| 82 | dev_samples = [] |
| 83 | for index, row in input_values.iterrows(): |
| 84 | if row[0].strip() == "": |
| 85 | continue |
| 86 | if row[0].startswith("CAN"): |
| 87 | file_train_path = "/data_ssd/opendata/small/datasets_CAN" |
| 88 | elif row[0].startswith("SG"): |
| 89 | file_train_path = "/data_ssd/opendata/small/datasets_SG" |
| 90 | elif row[0].startswith("UK"): |
| 91 | file_train_path = "/data_ssd/opendata/small/datasets_UK" |
| 92 | elif row[0].startswith("USA"): |
| 93 | file_train_path = "/data_ssd/opendata/small/datasets_USA" |
| 94 | else: |
| 95 | print("nofind this file",row[0]) |
| 96 | file1_path = os.path.join(file_train_path,row[0]) |
| 97 | |
| 98 | if row[1].strip() == "": |
| 99 | continue |
| 100 | if row[1].startswith("CAN"): |
| 101 | file_train_path = "/data_ssd/opendata/small/datasets_CAN" |
| 102 | elif row[1].startswith("SG"): |
| 103 | file_train_path = "/data_ssd/opendata/small/datasets_SG" |
| 104 | elif row[1].startswith("UK"): |
| 105 | file_train_path = "/data_ssd/opendata/small/datasets_UK" |
| 106 | elif row[1].startswith("USA"): |
| 107 | file_train_path = "/data_ssd/opendata/small/datasets_USA" |
| 108 | else: |
| 109 | print("nofind this file",row[1]) |
| 110 | |
| 111 | file2_path = os.path.join(file_train_path,row[1]) |
| 112 | try: |
| 113 | sentence_text1 = analyze_column_values(file1_path,row[2]) |
| 114 | sentence_text2 = analyze_column_values(file2_path, row[3]) |
| 115 | except Exception as e: |
| 116 | continue |
| 117 | score = float(row[4]) # Normalize score to range 0 ... 1 |
| 118 | random_number = random.random() |
| 119 | flag = False |
| 120 | if random_number < 0.2: |
| 121 | flag= True |
| 122 | |
| 123 | if random_number > 0.8: |
| 124 | dev_samples.append(sentence_text1 + "#####" + sentence_text2 + "#####" + str(score)) |
| 125 | else: |
| 126 | train_samples.append(sentence_text1 + "#####" + sentence_text2) |
| 127 | |
| 128 | if flag: |
| 129 | shuffle_sentence1 = shuffle_sentence(sentence_text1) |
| 130 | shuffle_sentence2 = shuffle_sentence(sentence_text2) |
| 131 | random_number = random.random() |
| 132 | if random_number > 0.8: |
| 133 | dev_samples.append(shuffle_sentence1 + '#####' + shuffle_sentence2 + '#####' + str(score)) |
| 134 | else: |
| 135 | train_samples.append(shuffle_sentence1 + "#####" + shuffle_sentence2) |
| 136 | queue.put(1) |
nothing calls this directly
no test coverage detected