Method process

DIEN/data_utils.py:424–630

(self, data_files, file_name)

    # Process the data by read in the user information from raw_path file
    def process(self, data_files, file_name):
        #try:
        #    ss = self.source_buffer.pop()
        #except IndexError:
            

        source = []
        target = []
        hist_item_list = []
        hist_cate_list = []
        hist_shop_list = []
        hist_node_list = []
        hist_product_list = []
        hist_brand_list = []
     
        neg_hist_item_list = []
        neg_hist_cate_list = []
        neg_hist_shop_list = []
        neg_hist_node_list = []
        neg_hist_product_list = []
        neg_hist_brand_list = []

        st = time.time()
        data_size = 500000 #around 9GB data for maxlength=100
        #data_size = 250000 #around 9GB data for maxlength=1500
        line_count = 0
        file_count = 0
        for file_read in data_files:
            data_file = self.data_path + file_read
            print('Open file ', data_file)
            f = open(data_file, 'r')
            tmp_s = f.readline()
            while(tmp_s):
                line_count = line_count + 1
                
                ss = tmp_s.strip().split("^H")
                uid = self.map_user(ss[0])
                hist_item = map(self.map_item, ss[1].split('\003'))
                hist_cate = map(self.map_cate, ss[2].split('\003'))
                hist_shop = map(self.map_shop, ss[3].split('\003'))
                hist_node = map(self.map_node, ss[4].split('\003'))
                hist_product = map(self.map_product, ss[5].split('\003'))
                hist_brand = map(self.map_brand, ss[6].split('\003'))
                       
                pos_item = hist_item[-1] 
                pos_cate = hist_cate[-1] 
                pos_shop = hist_shop[-1] 
                pos_node = hist_node[-1] 
                pos_product = hist_product[-1] 
                pos_brand = hist_brand[-1]  
                if self.neg_sample == 'LastInstance':
                    #set item of last instance as neg sample
                    neg_item, neg_cate, neg_shop, neg_node, neg_product, neg_brand \
                    = self.last_item, self.last_cate, self.last_shop, self.last_node, \
                        self.last_product, self.last_brand
                elif self.neg_sample == 'Random': 
                    #generate random neg_item information for neg sample
                    item_idx = int(random.random()*self.num_items)