Method process2

DIEN/data_utils.py:206–421

(self, data_files, file_name)

    # 2nd method by using previous user's behavior to replace the current user's negative behavior data
    def process2(self, data_files, file_name):
        #try:
        #    ss = self.source_buffer.pop()
        #except IndexError:
            

        source = []
        target = []
        hist_item_list = []
        hist_cate_list = []
        hist_shop_list = []
        hist_node_list = []
        hist_product_list = []
        hist_brand_list = []
     
        neg_hist_item_list = []
        neg_hist_cate_list = []
        neg_hist_shop_list = []
        neg_hist_node_list = []
        neg_hist_product_list = []
        neg_hist_brand_list = []

        last_neg_hist = self.gen_neg_hist3(self.maxlen)
        
        st = time.time()
        #data_size = 20000
        #data_size = 500000 #around 9GB data for maxlength=100
        data_size = 50000 #around 9GB data for maxlength=1000
        line_count = 0
        file_count = 0
        for file_read in data_files:
            data_file = self.data_path + file_read
            print('Open file ', data_file)
            f = open(data_file, 'r')
            tmp_s = f.readline()
            while(tmp_s):
                line_count = line_count + 1
                
                ss = tmp_s.strip().split("^H")
                uid = self.map_user(ss[0])
                hist_item = map(self.map_item, ss[1].split('\003'))
                hist_cate = map(self.map_cate, ss[2].split('\003'))
                hist_shop = map(self.map_shop, ss[3].split('\003'))
                hist_node = map(self.map_node, ss[4].split('\003'))
                hist_product = map(self.map_product, ss[5].split('\003'))
                hist_brand = map(self.map_brand, ss[6].split('\003'))
                       
                pos_item = hist_item[-1] 
                pos_cate = hist_cate[-1] 
                pos_shop = hist_shop[-1] 
                pos_node = hist_node[-1] 
                pos_product = hist_product[-1] 
                pos_brand = hist_brand[-1]  
                if self.neg_sample == 'LastInstance':
                    #set item of last instance as neg sample
                    neg_item, neg_cate, neg_shop, neg_node, neg_product, neg_brand \
                    = self.last_item, self.last_cate, self.last_shop, self.last_node, \
                        self.last_product, self.last_brand