Function shuffle

python/paddle/reader/decorator.py:201–243

This API creates a decorated reader that outputs the shuffled data. The output data from the origin reader will be saved into a buffer, and then shuffle the data. The size of buffer is determined by argument buf_size. Args: reader(callable): the original reader whose data

(reader: _Reader[_T], buf_size: int)

def shuffle(reader: _Reader[_T], buf_size: int) -> _Reader[_T]:
    """
    This API creates a decorated reader that outputs the shuffled data.

    The output data from the origin reader will be saved into a buffer,
    and then shuffle the data. The size of buffer is determined by argument buf_size.

    Args:
        reader(callable): the original reader whose data will be shuffled.
        buf_size(int): the size of shuffled buffer.

    Returns:
        callable: a decorated reader.

    Examples:
        .. code-block:: pycon

            >>> # doctest: +SKIP('outputs are 0~4 unordered arrangement')
            >>> def reader():
            ...     for i in range(5):
            ...         yield i
            >>> shuffled_reader = paddle.reader.decorator.shuffle(reader, 3)
            >>> for e in shuffled_reader():
            ...     print(e)
            >>> # outputs are 0~4 unordered arrangement
    """

    def data_reader() -> Generator[_T, None, None]:
        buf = []
        for e in reader():
            buf.append(e)
            if len(buf) >= buf_size:
                random.shuffle(buf)
                for b in buf:
                    yield b
                buf = []

        if len(buf) > 0:
            random.shuffle(buf)
            for b in buf:
                yield b

    return data_reader


def chain(*readers: _Reader[_T]) -> _Reader[_T]: