(self, start_id, total_thread)
| 32 | return self |
| 33 | |
| 34 | def data_read(self, start_id, total_thread): |
| 35 | sample_id = start_id |
| 36 | while sample_id < self.data_file_num: |
| 37 | if len(self.queue) >= self.max_queue_size: |
| 38 | time.sleep(1) |
| 39 | continue |
| 40 | processed_data_path = self.data_path + self.data_file + "_" + str(sample_id) + '_processed.npz' |
| 41 | print('Start loading processed data...' + processed_data_path) |
| 42 | st = time.time() |
| 43 | data = np.load(processed_data_path) |
| 44 | source = data['source_array'] |
| 45 | uid_array = np.array(source)[:,0] |
| 46 | item_array = np.array(source)[:,1] |
| 47 | cate_array = np.array(source)[:,2] |
| 48 | shop_array = np.array(source)[:,3] |
| 49 | node_array = np.array(source)[:,4] |
| 50 | product_array = np.array(source)[:,5] |
| 51 | brand_array = np.array(source)[:,6] |
| 52 | |
| 53 | target = data['target_array'] |
| 54 | history_item = data['history_item_array'] |
| 55 | history_cate = data['history_cate_array'] |
| 56 | history_shop = data['history_shop_array'] |
| 57 | history_node = data['history_node_array'] |
| 58 | history_product = data['history_product_array'] |
| 59 | history_brand = data['history_brand_array'] |
| 60 | |
| 61 | neg_history_item = data['neg_history_item_array'] |
| 62 | neg_history_cate = data['neg_history_cate_array'] |
| 63 | neg_history_shop = data['neg_history_shop_array'] |
| 64 | neg_history_node = data['neg_history_node_array'] |
| 65 | neg_history_product = data['neg_history_product_array'] |
| 66 | neg_history_brand = data['neg_history_brand_array'] |
| 67 | print('Finish loading processed data id '+ str(sample_id) + ',Time cost = %.4f' % (time.time()-st)) |
| 68 | data_file = (uid_array,item_array,cate_array,shop_array,node_array,product_array,brand_array,\ |
| 69 | target, history_item,history_cate,history_shop, history_node,history_product,history_brand,\ |
| 70 | neg_history_item,neg_history_cate,neg_history_shop, neg_history_node,neg_history_product,neg_history_brand) |
| 71 | while self.help_count % total_thread != start_id: |
| 72 | time.sleep(1) |
| 73 | print('help_count=', self.help_count) |
| 74 | self.queue.append(data_file) |
| 75 | self.help_count += 1 |
| 76 | sample_id = sample_id + total_thread |
| 77 | |
| 78 | |
| 79 | def _batch_data(self, data, data_slice): |
nothing calls this directly
no outgoing calls
no test coverage detected