| 13 | self.delimiter = delimiter |
| 14 | |
| 15 | def read(self, path): |
| 16 | logging.debug( |
| 17 | "Reading from %s using CSV format with configuration %s" |
| 18 | % (path, self.__dict__) |
| 19 | ) |
| 20 | # do not read the first line as header |
| 21 | read_options = pyarrow.csv.ReadOptions(autogenerate_column_names=True) |
| 22 | parse_options = pyarrow.csv.ParseOptions(delimiter=self.delimiter) |
| 23 | arr = pyarrow.csv.read_csv( |
| 24 | path, read_options=read_options, parse_options=parse_options |
| 25 | ) |
| 26 | logging.debug("Done reading from %s" % path) |
| 27 | return arr.to_pandas().to_numpy() |
| 28 | |
| 29 | def write(self, path, arr): |
| 30 | logging.debug( |