Method build_streaming_iterator

espnet2/tasks/abs_task.py:2399–2447

Build DataLoader using iterable dataset

(
        cls,
        data_path_and_name_and_type,
        preprocess_fn,
        collate_fn,
        key_file: Optional[str] = None,
        batch_size: int = 1,
        dtype: Optional[Any] = np.float32,
        num_workers: int = 1,
        allow_variable_data_keys: bool = False,
        ngpu: int = 0,
        inference: bool = False,
        mode: Optional[str] = None,
        multi_task_dataset: bool = False,
    )

    @classmethod
    @typechecked
    def build_streaming_iterator(
        cls,
        data_path_and_name_and_type,
        preprocess_fn,
        collate_fn,
        key_file: Optional[str] = None,
        batch_size: int = 1,
        dtype: Optional[Any] = np.float32,
        num_workers: int = 1,
        allow_variable_data_keys: bool = False,
        ngpu: int = 0,
        inference: bool = False,
        mode: Optional[str] = None,
        multi_task_dataset: bool = False,
    ) -> DataLoader:
        """Build DataLoader using iterable dataset"""
        # For backward compatibility for pytorch DataLoader
        if collate_fn is not None:
            kwargs = dict(collate_fn=collate_fn)
        else:
            kwargs = {}

        if multi_task_dataset:
            dataset_class = ESPnetMultiTaskDataset
        else:
            dataset_class = IterableESPnetDataset
        dataset = dataset_class(
            data_path_and_name_and_type,
            float_dtype=dtype,
            preprocess=preprocess_fn,
            key_file=key_file,
        )

        if dataset.apply_utt2category:
            kwargs.update(batch_size=1)
        else:
            kwargs.update(batch_size=batch_size)

        cls.check_task_requirements(
            dataset, allow_variable_data_keys, train=False, inference=inference
        )

        return DataLoader(
            dataset=dataset,
            pin_memory=ngpu > 0,
            num_workers=num_workers,
            sampler=getattr(dataset, "example_list", None),
            **kwargs,
        )

    # ~~~~~~~~~ The methods below are mainly used for inference ~~~~~~~~~
    @classmethod