Method read_data_sets

tf_classify_demo/sample_data.py:98–150

读取文件，加载数据

(file_name)

    @staticmethod
    def read_data_sets(file_name):
        """
        读取文件，加载数据
        """
        instance = InputData()
        file_object = open(file_name, 'r')
        while True:
            line = file_object.readline(1024)
            if line:
                line = line.strip()
                if len(line) == 0:
                    continue
                split = line.split(' ')
                group_id = 0
                try:
                    group_id = int(split[0])
                except ValueError:
                    continue
                txt = ' '.join(split[1:])
                txt = txt.replace('None', '').strip()
                if len(txt) == 0:
                    continue

                vectors = {}
                seg_list = jieba.cut(txt)
                for seg in seg_list:
                    seg_unicode = seg.encode('utf-8')
                    if seg_unicode in instance.word_vector_dict:
                        word_id = instance.word_id_dict[seg_unicode]
                        if word_id in instance.maps.local_word_id_map:
                            local_word_id = instance.maps.local_word_id_map[word_id]
                            vectors[local_word_id] = instance.word_vector_dict[seg_unicode]
                        else:
                            local_word_id = instance.dim_info.max_word_id
                            instance.maps.local_word_id_map[word_id] = local_word_id
                            vectors[local_word_id] = instance.word_vector_dict[seg_unicode]
                            instance.dim_info.max_word_id = instance.dim_info.max_word_id + 1

                # 稀疏向量
                item = {'vectors':vectors,
                        'local_group_id':instance.maps.local_group_id_map[str(group_id)]}
                instance.data.append(item)
            else:
                break
        file_object.close()

        random.shuffle(instance.data)
        for _ in range(TEST_COUNT):
            instance.test_data.append(instance.data.pop())
        instance.dim_info.x_dim = instance.dim_info.max_word_id * instance.dim_info.vec_dim
        print("max_word_id=", instance.dim_info.max_word_id)
        print("x_dim=", instance.dim_info.x_dim)
        return instance

    def generate_xs(self, txt):
        """