Function _make_file_instructions

tensorflow_datasets/core/splits.py:569–601

Returns file instructions by applying the given instruction on the given splits. Args: split_infos: Dataset splits information instruction: `ReadInstruction` or `str` Returns: List of FileInstruction instances

(
    split_infos: Sequence[SplitInfo],
    instruction: SplitArg,
)

def _make_file_instructions(
    split_infos: Sequence[SplitInfo],
    instruction: SplitArg,
) -> list[shard_utils.FileInstruction]:
  """Returns file instructions by applying the given instruction on the given splits.

  Args:
    split_infos: Dataset splits information
    instruction: `ReadInstruction` or `str`

  Returns:
    List of FileInstruction instances
  """

  # TODO(epot): Should try to merge the instructions together as well as
  # performing additional validation. For example, should raise an error
  # if there is overlap between splits (`train[:50]+train[:25]`)
  # If there is a single shard, `train[:25]+train[50:75]` could be optimized
  # into a single `ds.take(25).skip(50-25).take(75-50)`

  absolute_instructions = _make_absolute_instructions(
      split_infos=split_infos, instruction=instruction
  )
  instructions = []
  info_per_split = {split_info.name: split_info for split_info in split_infos}
  for abs_instr in absolute_instructions:
    split_info = info_per_split[str(abs_instr.splitname)]
    instructions.extend(
        _file_instructions_for_split(
            instruction=abs_instr, split_info=split_info
        )
    )
  return instructions


class AbstractSplit(abc.ABC):