Class AstroClipDataset

astroclip/data/dataset.py:39–118

TODO: Short description of my dataset.

class AstroClipDataset(datasets.GeneratorBasedBuilder):
    """TODO: Short description of my dataset."""

    VERSION = datasets.Version("1.1.5")

    BUILDER_CONFIGS = [
        datasets.BuilderConfig(
            name="joint",
            version=VERSION,
            description="This part of the dataset covers examples from both specral and image domains",
        ),
    ]

    DEFAULT_CONFIG_NAME = "joint"

    def _info(self):
        if self.config.name == "joint":
            features = datasets.Features(
                {
                    "image": datasets.Array3D(shape=(152, 152, 3), dtype="float32"),
                    "spectrum": datasets.Array2D(shape=(7781, 1), dtype="float32"),
                    "redshift": datasets.Value("float32"),
                    "targetid": datasets.Value("int64"),
                }
            )
        else:
            raise NotImplementedError(
                "Only the joint configuration is implemented for now"
            )

        return datasets.DatasetInfo(
            description=_DESCRIPTION,
            features=features,
            homepage=_HOMEPAGE,
            license=_LICENSE,
            citation=_CITATION,
        )

    def _split_generators(self, dl_manager):
        urls = _URLS[self.config.name]
        dl_manager.download_config.storage_options["timeout"] = ClientTimeout(total=5000, connect=1000)
        data_dir = dl_manager.download_and_extract(urls)
        return [
            datasets.SplitGenerator(
                name=datasets.Split.TRAIN,
                gen_kwargs={
                    "filepath": data_dir,
                    "split": "train",
                },
            ),
            datasets.SplitGenerator(
                name=datasets.Split.TEST,
                gen_kwargs={"filepath": data_dir, "split": "test"},
            ),
        ]

    def _generate_examples(self, filepath, split):
        """Yields examples."""