Method write

modelscope/models/nlp/mglm/data_utils/datasets.py:355–377

given a generator of metrics for each of the data points X_i, write the metrics, text, and labels to a csv file

(self, writer_gen=None, path=None, skip_header=False)

        return {'text': x, 'length': len(x), 'label': y}

    def write(self, writer_gen=None, path=None, skip_header=False):
        """
        given a generator of metrics for each of the data points X_i,
            write the metrics, text, and labels to a csv file
        """
        if path is None:
            path = self.path + '.results'
        print('generating csv at ' + path)
        with open(path, 'w') as csvfile:
            c = csv.writer(csvfile, delimiter=self.delim)
            if writer_gen is not None:
                # if first item of generator is a header of what the metrics mean then write header to csv file
                if not skip_header:
                    header = (self.label_key, ) + tuple(
                        next(writer_gen)) + (self.text_key, )
                    c.writerow(header)
                for i, row in enumerate(writer_gen):
                    row = (self.Y[i], ) + tuple(row) + (self.X[i], )
                    c.writerow(row)
            else:
                c.writerow([self.label_key, self.text_key])
                for row in zip(self.Y, self.X):
                    c.writerow(row)


class json_dataset(data.Dataset):