(self)
| 111 | neg_hist_product, neg_hist_brand ] |
| 112 | |
| 113 | def next(self): |
| 114 | previous_data_out = [] |
| 115 | data_file_read = 0 |
| 116 | batch_id = 0 |
| 117 | #print('in next func') |
| 118 | #import pdb; pdb.set_trace() |
| 119 | previous_line = 0 |
| 120 | while len(self.queue) < 2: |
| 121 | time.sleep(1) |
| 122 | print ('Now the queue has two data file loaded in!') |
| 123 | while data_file_read < self.data_file_num: |
| 124 | if len(self.queue) == 0: |
| 125 | time.sleep(1) |
| 126 | continue |
| 127 | data = self.queue.popleft() |
| 128 | file_line_num = data[0].shape[0] |
| 129 | start_ind = 0 |
| 130 | data_file_read = data_file_read + 1 |
| 131 | stime = time.time() |
| 132 | #print('start one file,time=', stime) |
| 133 | while start_ind <= file_line_num - self.batch_size: |
| 134 | |
| 135 | if previous_line != 0: |
| 136 | batch_left = self.batch_size - previous_line |
| 137 | else: |
| 138 | batch_left = self.batch_size |
| 139 | data_slice = slice(start_ind, start_ind + batch_left) |
| 140 | # slice the data from the list |
| 141 | data_out = self._batch_data(data, data_slice) #data_out is tuple |
| 142 | |
| 143 | if previous_line != 0: |
| 144 | #attach the data |
| 145 | for i in range(len(data_out)): |
| 146 | data_out[i] = np.concatenate( |
| 147 | [previous_data_out[i], data_out[i]], |
| 148 | axis=0 |
| 149 | ) |
| 150 | if self.batch_size != len(data_out[0]): |
| 151 | raise ValueError('batch fetched wrong!') |
| 152 | |
| 153 | start_ind = start_ind + batch_left |
| 154 | |
| 155 | previous_line = 0 |
| 156 | #print("start_ind ", start_ind) |
| 157 | yield data_out |
| 158 | if start_ind != file_line_num: |
| 159 | data_slice = slice(start_ind, file_line_num) |
| 160 | previous_data_out = self._batch_data(data, data_slice) |
| 161 | previous_line = file_line_num - start_ind |
| 162 | print("Left batch of size %d" %( previous_line)) |
| 163 | etime = time.time() |
| 164 | print('Consume one file takes time= %.4f' %(etime-stime)) |
| 165 | print ('drop last batch since it is not full batch size') |
| 166 | |
| 167 | |
| 168 | def test(): |
no test coverage detected