| 23 | import time |
| 24 | |
| 25 | class preprocessDataset: |
| 26 | |
| 27 | def __init__( |
| 28 | self, |
| 29 | data_path, |
| 30 | maxlen, |
| 31 | ): |
| 32 | self.data_path = data_path |
| 33 | #self.user_dict, self.item_dict, self.cate_dict, self.shop_dict, self.node_dict \ |
| 34 | #, self.product_dict, self.brand_dict, self.item_info = dict_list |
| 35 | self.neg_sample = 'LastInstance' # 'Random' or 'LastInstance' |
| 36 | |
| 37 | |
| 38 | self.source_buffer = [] |
| 39 | self.neg_hist_catch = {} |
| 40 | self.maxlen = maxlen |
| 41 | self.max_catch_num = 20 |
| 42 | #self.end_of_data = False |
| 43 | st = time.time() |
| 44 | print("Start loading dict...") |
| 45 | #hack here |
| 46 | data_path = '/disk3/w.wei/dien-new/' |
| 47 | self.item_info = json.load(open(data_path + '/item_info.json', 'r')) |
| 48 | print ('Finish loading item_info.json,length=',len(self.item_info.keys())) |
| 49 | self.user_dict = json.load(open(data_path + '/user_voc.json', 'r')) |
| 50 | print ('Finish loading user_voc.json,length=',len(self.user_dict.keys())) |
| 51 | self.item_dict = json.load(open(data_path + '/item_voc.json', 'r')) |
| 52 | print ('Finish loading item_voc.json,length=',len(self.item_dict.keys())) |
| 53 | self.cate_dict = json.load(open(data_path + '/cate_voc.json', 'r')) |
| 54 | print ('Finish loading cate_voc.json,length=',len(self.cate_dict.keys())) |
| 55 | self.shop_dict = json.load(open(data_path + '/shop_voc.json', 'r')) |
| 56 | print ('Finish loading shop_voc.json,length=',len(self.shop_dict.keys())) |
| 57 | self.node_dict = json.load(open(data_path + '/node_voc.json', 'r')) |
| 58 | print ('Finish loading node_voc.json,length=',len(self.node_dict.keys())) |
| 59 | self.product_dict = json.load(open(data_path + '/product_voc.json', 'r')) |
| 60 | print ('Finish loading product_voc.json,length=',len(self.product_dict.keys())) |
| 61 | self.brand_dict = json.load(open(data_path + '/brand_voc.json', 'r')) |
| 62 | print ('Finish loading brand_voc.json,length=',len(self.brand_dict.keys())) |
| 63 | print("Time for load dict=", time.time()-st) |
| 64 | #import pdb; pdb.set_trace() |
| 65 | self.all_items = self.item_info.keys() |
| 66 | self.num_items = len(self.all_items) |
| 67 | #generate random neg item to initialize last_item information |
| 68 | item_idx = int(random.random()*self.num_items) |
| 69 | neg_item = self.all_items[item_idx] |
| 70 | self.last_cate = self.map_cate(self.item_info[neg_item][0]) |
| 71 | self.last_shop = self.map_shop(self.item_info[neg_item][1]) |
| 72 | self.last_node = self.map_node(self.item_info[neg_item][2]) |
| 73 | self.last_product = self.map_product(self.item_info[neg_item][3]) |
| 74 | self.last_brand = self.map_brand(self.item_info[neg_item][4]) |
| 75 | self.last_item = self.map_item(self.all_items[item_idx])#map origin item to item_id |
| 76 | |
| 77 | def get_id_nums(self): |
| 78 | uid_n = len(self.user_dict.keys()) |
| 79 | item_n = len(self.item_dict.keys()) |
| 80 | cate_n = len(self.cate_dict.keys()) |
| 81 | shop_n = len(self.shop_dict.keys()) |
| 82 | node_n = len(self.node_dict.keys()) |