| 15 | |
| 16 | |
| 17 | def process_onedataset(dataset_file,model_name ='output/deepjoin_webtable_training-all-mpnet-base-v2-2023-10-18_19-54-27', |
| 18 | storepath = "/home/lijiajun/deepjoin/webtable/final_result/"): |
| 19 | |
| 20 | path,filename_dataset = os.path.split(dataset_file) |
| 21 | |
| 22 | model = SentenceTransformer(model_name) |
| 23 | os.makedirs(storepath,exist_ok=True) |
| 24 | storedata = [] |
| 25 | if os.path.isfile(dataset_file): |
| 26 | print("process data",dataset_file) |
| 27 | try: |
| 28 | #记载数据 |
| 29 | with open(dataset_file,"rb") as f: |
| 30 | data = pickle.load(f) |
| 31 | for ele in tqdm(data): |
| 32 | key,value = ele |
| 33 | sentence_embeddings = model.encode(value) |
| 34 | sentence_embeddings_np = np.array(sentence_embeddings) |
| 35 | tu1 = (key,sentence_embeddings_np) |
| 36 | storedata.append(tu1) |
| 37 | except Exception as e: |
| 38 | print(e) |
| 39 | storefilename = os.path.join(storepath,filename_dataset) |
| 40 | with open(storefilename,"wb") as f: |
| 41 | pickle.dump(storedata,f) |
| 42 | print("data process sucess",storefilename) |
| 43 | |
| 44 | |
| 45 | |