(train_file,dev_file,num_partitions)
| 159 | |
| 160 | |
| 161 | def processdate(train_file,dev_file,num_partitions): |
| 162 | train_input_list = [] |
| 163 | dev_input_list = [] |
| 164 | with open(train_file,"rb") as t_file: |
| 165 | train_string_list = pickle.load(t_file) |
| 166 | with open(dev_file,"rb") as d_file: |
| 167 | dev_string_list= pickle.load(d_file) |
| 168 | |
| 169 | train_input_list = map(train_process_single_ele,train_string_list) |
| 170 | dev_input_list = map(dev_process_single_ele,dev_string_list) |
| 171 | |
| 172 | train_input_list = [i for i in train_input_list if i !=""] |
| 173 | dev_input_list = [j for j in dev_input_list if j !=""] |
| 174 | |
| 175 | train_len = len(train_input_list) // num_partitions |
| 176 | return_train = train_input_list[0:train_len] |
| 177 | |
| 178 | dev_len = len(dev_input_list) // num_partitions |
| 179 | return_dev = dev_input_list[0:dev_len] |
| 180 | |
| 181 | return return_train,return_dev |
| 182 | |
| 183 | |
| 184 | def process_before_train(file_train_path,filecsv,filepath = "/data/lijiajun/deepjoin/pretrain_data_list/",name = "train_list.pkl",name2 = "evluate_list.pkl"): |
no outgoing calls
no test coverage detected