Class IndexFileReader

cdslib/core/data/index_reader.py:15–138

The class implements a simple reader for an index file like scp files. Given an index file (e.g., scp file, with or without the unique id), the class creates a indexed list of the file content. For example, if i-th line in the scp file contains "uid_abc abc.npy", - dset[i] re

class IndexFileReader:
    """
    The class implements a simple reader for an index file like scp files.

    Given an index file (e.g., scp file, with or without the unique id),
    the class creates a indexed list of the file content.

    For example, if i-th line in the scp file contains "uid_abc abc.npy",

    - dset[i] returns the array contained in abc.npy.
    - dset.get_uid(i) returns uid_abc.

    Currently, we support only npy files.

    Note:
        The class is not a proper scp file reader.
        If a typical scp reader is what you are after,
        please use kaldiio.

    """

    def __init__(self, index_filename: str, with_uid: bool, root_dir: str = None):
        """
        Args:
            index_filename:
                the index file that lists all the data, one per line.
            with_uid:
                If with_uid is True, each line is composed of
                `a_unique_id_of_the_data  filename_of_the_data`.
                If with_uid is False, each line is composed of
                `filename_of_the_data`.
            root_dir:
                The root folder for the files listed in the index_filename.
                For example, if root_dir = 'a/b' and index_filename[0] = 'c/d.npy',
                the file is at 'a/b/c/d.npy'.
                If None is given, use the folder of the index_filename as root_dir.
                If '' is given, no modification will be made.
        """
        self.index_filename = index_filename
        self.with_uid = with_uid
        self.root_dir = root_dir
        if self.root_dir is None:
            self.root_dir = os.path.dirname(self.index_filename)

        # read the index file
        self.uids, self.filenames = self._read_index_file(
            index_filename=self.index_filename,
            with_uid=self.with_uid,
            root_dir=self.root_dir,
        )

    def __len__(self):
        """Returns the number of files in the index file."""
        return len(self.filenames)

    def __getitem__(self, i: int) -> T.Any:
        """Returns the content of the i-th line in the index file."""