| 19 | |
| 20 | @time_it(freq=10) |
| 21 | def __init__(self, source, dict_list, |
| 22 | batch_size=128, |
| 23 | maxlen=2000, |
| 24 | skip_empty=False, |
| 25 | sort_by_length=True, |
| 26 | max_batch_size=20, |
| 27 | minlen=None, |
| 28 | parall=False |
| 29 | ): |
| 30 | self.source = open(source, 'r') |
| 31 | self.batch_shuffle = 1 |
| 32 | self.neg_sample = 'LastInstance' # 'Random' or 'LastInstance' |
| 33 | #self.user_dict = copy.copy(dict_list[0]) |
| 34 | #self.item_dict = copy.copy(dict_list[1]) |
| 35 | #self.cate_dict = copy.copy(dict_list[2]) |
| 36 | #self.shop_dict = copy.copy(dict_list[3]) |
| 37 | #self.node_dict = copy.copy(dict_list[4]) |
| 38 | #self.product_dict = copy.copy(dict_list[5]) |
| 39 | #self.brand_dict = copy.copy(dict_list[6]) |
| 40 | #self.item_info = copy.copy(dict_list[7]) |
| 41 | self.user_dict, self.item_dict, self.cate_dict, self.shop_dict, self.node_dict, self.product_dict, self.brand_dict, self.item_info = dict_list |
| 42 | # self.user_dict = json.load(open('user_voc.json', 'r')) |
| 43 | # self.item_dict = json.load(open('item_voc.json', 'r')) |
| 44 | # self.cate_dict = json.load(open('cate_voc.json', 'r')) |
| 45 | # self.shop_dict = json.load(open('shop_voc.json', 'r')) |
| 46 | # self.node_dict = json.load(open('node_voc.json', 'r')) |
| 47 | # self.product_dict = json.load(open('product_voc.json', 'r')) |
| 48 | # self.brand_dict = json.load(open('brand_voc.json', 'r')) |
| 49 | # self.item_info = json.load(open('item_info.json', 'r')) |
| 50 | self.all_items = self.item_info.keys() |
| 51 | self.num_items = len(self.all_items) |
| 52 | self.batch_size = batch_size |
| 53 | self.maxlen = maxlen |
| 54 | self.minlen = minlen |
| 55 | self.skip_empty = skip_empty |
| 56 | self.sort_by_length = sort_by_length |
| 57 | self.max_catch_num = 20 |
| 58 | |
| 59 | self.source_buffer = [] |
| 60 | #self.batch_size = batch_size * max_batch_size |
| 61 | self.batch_size = batch_size |
| 62 | self.neg_hist_catch = {} |
| 63 | self.end_of_data = False |
| 64 | #generate random neg item to initialize last_item information |
| 65 | item_idx = int(random.random()*self.num_items) |
| 66 | neg_item = self.all_items[item_idx] |
| 67 | self.last_cate = self.map_cate(self.item_info[neg_item][0]) |
| 68 | self.last_shop = self.map_shop(self.item_info[neg_item][1]) |
| 69 | self.last_node = self.map_node(self.item_info[neg_item][2]) |
| 70 | self.last_product = self.map_product(self.item_info[neg_item][3]) |
| 71 | self.last_brand = self.map_brand(self.item_info[neg_item][4]) |
| 72 | self.last_item = self.map_item(self.all_items[item_idx])#map origin item to item_id |
| 73 | |
| 74 | def get_id_nums(self): |
| 75 | uid_n = len(self.user_dict.keys()) |