Method filter

python/ray/data/dataset.py:1551–1753

Filter out rows that don't satisfy the given predicate. You can use either a function or a callable class or an expression to perform the transformation. For functions, Ray Data uses stateless Ray tasks. For classes, Ray Data uses stateful Ray actors. For more inform

(
        self,
        fn: Optional[UserDefinedFunction[Dict[str, Any], bool]] = None,
        expr: Optional[Union[str, Expr]] = None,
        *,
        compute: Union[str, ComputeStrategy] = None,
        fn_args: Optional[Iterable[Any]] = None,
        fn_kwargs: Optional[Dict[str, Any]] = None,
        fn_constructor_args: Optional[Iterable[Any]] = None,
        fn_constructor_kwargs: Optional[Dict[str, Any]] = None,
        num_cpus: Optional[float] = None,
        num_gpus: Optional[float] = None,
        memory: Optional[float] = None,
        concurrency: Optional[Union[int, Tuple[int, int], Tuple[int, int, int]]] = None,
        ray_remote_args_fn: Optional[Callable[[], Dict[str, Any]]] = None,
        **ray_remote_args,
    )

    @PublicAPI(api_group=BT_API_GROUP)
    def filter(
        self,
        fn: Optional[UserDefinedFunction[Dict[str, Any], bool]] = None,
        expr: Optional[Union[str, Expr]] = None,
        *,
        compute: Union[str, ComputeStrategy] = None,
        fn_args: Optional[Iterable[Any]] = None,
        fn_kwargs: Optional[Dict[str, Any]] = None,
        fn_constructor_args: Optional[Iterable[Any]] = None,
        fn_constructor_kwargs: Optional[Dict[str, Any]] = None,
        num_cpus: Optional[float] = None,
        num_gpus: Optional[float] = None,
        memory: Optional[float] = None,
        concurrency: Optional[Union[int, Tuple[int, int], Tuple[int, int, int]]] = None,
        ray_remote_args_fn: Optional[Callable[[], Dict[str, Any]]] = None,
        **ray_remote_args,
    ) -> "Dataset":
        """Filter out rows that don't satisfy the given predicate.

        You can use either a function or a callable class or an expression to
        perform the transformation.
        For functions, Ray Data uses stateless Ray tasks. For classes, Ray Data uses
        stateful Ray actors. For more information, see
        :ref:`Stateful Transforms <stateful_transforms>`.

        .. tip::
           If you use the `expr` parameter with a predicate expression, Ray Data
           optimizes your filter with native Arrow interfaces.

        .. deprecated::
           String expressions are deprecated and will be removed in a future version.
           Use predicate expressions from `ray.data.expressions` instead.

        Examples:

            >>> import ray
            >>> from ray.data.expressions import col
            >>> ds = ray.data.range(100)
            >>> # String expressions (deprecated - will warn)
            >>> ds.filter(expr="id <= 4").take_all()
            [{'id': 0}, {'id': 1}, {'id': 2}, {'id': 3}, {'id': 4}]
            >>> # Using predicate expressions (preferred)
            >>> ds.filter(expr=(col("id") > 10) & (col("id") < 20)).take_all()
            [{'id': 11}, {'id': 12}, {'id': 13}, {'id': 14}, {'id': 15}, {'id': 16}, {'id': 17}, {'id': 18}, {'id': 19}]

        Time complexity: O(dataset size / parallelism)

        Args:
            fn: The predicate to apply to each row, or a class type
                that can be instantiated to create such a callable.
            expr: An expression that represents a predicate (boolean condition) for filtering.
                Can be either a string expression (deprecated) or a predicate expression
                from `ray.data.expressions`.
            compute: The compute strategy to use for the map operation.

                * If ``compute`` is not specified for a function, will use ``ray.data.TaskPoolStrategy()`` to launch concurrent tasks based on the available resources and number of input blocks.

                * Use ``ray.data.TaskPoolStrategy(size=n)`` to launch at most ``n`` concurrent Ray tasks.