Class ShardedFileTemplate

tensorflow_datasets/core/naming.py:487–703

Template to produce filenames for sharded datasets. Attributes: data_dir: the directory that contains the files for the shards. template: template of the sharded files, e.g. '${SPLIT}/data.${FILEFORMAT}-${SHARD_INDEX}'. dataset_name: the name of the dataset. split: the split

@dataclasses.dataclass()
class ShardedFileTemplate:
  """Template to produce filenames for sharded datasets.

  Attributes:
    data_dir: the directory that contains the files for the shards.
    template: template of the sharded files, e.g.
      '${SPLIT}/data.${FILEFORMAT}-${SHARD_INDEX}'.
    dataset_name: the name of the dataset.
    split: the split of the dataset.
    filetype_suffix: the filetype suffix to denote the type of file. For
      example, `tfrecord`.
  """

  data_dir: epath.Path
  template: str = DEFAULT_FILENAME_TEMPLATE
  dataset_name: str | None = None
  split: str | None = None
  filetype_suffix: str | None = None

  def __post_init__(self):
    self.data_dir = epath.Path(self.data_dir)
    if self.split is not None and not self.split:
      raise ValueError(f'Split must be a non-empty string: {self}')
    if self.split is not None and not any(
        char.isalnum() for char in self.split
    ):
      raise ValueError(
          'Split name should contain at least one alphanumeric character.'
          f' Given split name: {self.split}'
      )
    if self.filetype_suffix is not None and not self.filetype_suffix:
      raise ValueError(f'Filetype suffix must be a non-empty string: {self}')
    if not self.template:
      self.template = DEFAULT_FILENAME_TEMPLATE

  @functools.cached_property
  def regex(self) -> re.Pattern[str]:
    """Returns the regular expression for this template.

    Can be used to test whether a filename matches to this template.
    """
    return _regex_for_template(self.template)

  def parse_filename_info(self, filename: str) -> FilenameInfo | None:
    """Parses the filename using this template.

    Note that when the filename doesn't specify the dataset name, split, or
    filetype suffix, but this template does, then the value in the template will
    be used.

    Arguments:
      filename: the filename that should be parsed.

    Returns:
      the FilenameInfo corresponding to the given file if it could be parsed.
      None otherwise.
    """