Function test_drop_duplicates

dask/dataframe/dask_expr/tests/test_collection.py:1742–1784

(df, pdf, split_out)

@pytest.mark.parametrize("split_out", [1, True])
def test_drop_duplicates(df, pdf, split_out):
    with dask.config.set({"dataframe.shuffle.method": "tasks"}):
        assert_eq(
            df.drop_duplicates(split_out=split_out),
            pdf.drop_duplicates(),
        )
        assert_eq(
            df.drop_duplicates(ignore_index=True, split_out=split_out),
            pdf.drop_duplicates(ignore_index=True),
            check_index=split_out is not True,
        )
        assert_eq(
            df.drop_duplicates(subset=["y"], split_out=split_out),
            pdf.drop_duplicates(subset=["y"]),
        )
        assert_eq(
            df.drop_duplicates(subset=["y"], split_out=split_out, keep="last"),
            pdf.drop_duplicates(subset=["y"], keep="last"),
        )
        assert_eq(
            df.y.drop_duplicates(split_out=split_out),
            pdf.y.drop_duplicates(),
        )
        assert_eq(
            df.y.drop_duplicates(split_out=split_out, keep="last"),
            pdf.y.drop_duplicates(keep="last"),
        )
        actual = df.set_index("y").index.drop_duplicates(split_out=split_out)
        if split_out is True:
            actual = actual.compute().sort_values()  # shuffle is unordered
        assert_eq(
            actual,
            pdf.set_index("y").index.drop_duplicates(),
        )

    with pytest.raises(KeyError, match="'a'"):
        df.drop_duplicates(subset=["a"], split_out=split_out)

    with pytest.raises(TypeError, match="got an unexpected keyword argument"):
        df.x.drop_duplicates(subset=["a"], split_out=split_out)

    with pytest.raises(NotImplementedError, match="with keep=False"):
        df.x.drop_duplicates(keep=False)


def test_drop_duplicates_split_out(df, pdf):