Method read

app/src/utilities/io/reader.py:43–71

(self, uri: str, max_samples: Optional[int] = -1,
             return_idx: bool = True)

        self._total_consumed = 0

    def read(self, uri: str, max_samples: Optional[int] = -1,
             return_idx: bool = True):
        n_samples = 0

        try:
            with self.__get_filehandle(uri) as fh:
                for idx, obj in enumerate(fh):
                    try:
                        record = self._obj_decoder.decode(obj)
                        if return_idx:
                            yield idx, record
                        else:
                            yield record
                    except Exception as e:
                        self._print(f"__SAMPLE_READ_ERROR__ {uri}/{idx}: "
                                    f"{e.__class__.__name__}: {e}")
                        continue

                    n_samples += 1

                    if n_samples >= max_samples > 0:
                        break
        except S3ReadError as e:
            raise e
        except LocalReadError:
            raise e
        except Exception as e:
            raise UnknownReadError(f"unknown __URI_READ_ERROR__ {uri}: "
                                   f"{e.__class__.__name__}: {e}")

    def __get_filehandle(self, uri: str):
        uri = urlparse(uri)