(tmp_path: Path)
| 188 | |
| 189 | |
| 190 | def test_e2e_df_processing(tmp_path: Path): |
| 191 | def _test_df( |
| 192 | df: pd.DataFrame, |
| 193 | expected_types_pd13: SList, |
| 194 | ): |
| 195 | df_conv = df.convert_dtypes() |
| 196 | df_proc = process_df(df, copy=True) |
| 197 | |
| 198 | # check both df's have same nulls |
| 199 | pd.testing.assert_frame_equal(pd.isnull(df), pd.isnull(df_conv)) |
| 200 | pd.testing.assert_frame_equal(pd.isnull(df), pd.isnull(df_proc)) |
| 201 | |
| 202 | # check we can save and load processed file |
| 203 | df2 = save_load_arrow(tmp_path, df_proc) |
| 204 | pd.testing.assert_frame_equal(df_proc, df2) |
| 205 | |
| 206 | log.info(f"Using PD_VERSION={PD_VERSION}") |
| 207 | expected_types = expected_types_pd13 |
| 208 | |
| 209 | assert [str(x) for x in df2.dtypes] == expected_types |
| 210 | |
| 211 | # DF 1 |
| 212 | df = vd.data.cars() |
| 213 | _test_df(df, ["string", "Float64", "UInt8", "Float64", "UInt8", "UInt16", "Float64", "datetime64[ns]", "category"]) |
| 214 | |
| 215 | # DF 2 - float64/int64 downcasting for older pandas versions |
| 216 | df = pd.DataFrame( |
| 217 | dict( |
| 218 | int_col=[x for x in range(30)], |
| 219 | # NOTE - Pandas downcasting |
| 220 | # ensure large ints remain if can't be downcasted |
| 221 | int64_col=[x for x in range(29)] + [int(-1e10)], |
| 222 | # handle invalid in downcasting for older pandas versions |
| 223 | float64_int_col=[float(x) for x in range(29)] + [-1e2], # int |
| 224 | float64_int64_col=[float(x) for x in range(29)] + [-1e10], # bigint |
| 225 | float_col=[(x + 0.1) for x in range(30)], |
| 226 | ) |
| 227 | ) |
| 228 | _test_df( |
| 229 | df, |
| 230 | # convert_dtypes doesn't convert Float64 -> Int64 on Windows |
| 231 | ["UInt8", "Int64", "Int8", "Float64" if sys.platform == "win32" else "Int64", "Float64"], |
| 232 | ) |
| 233 | |
| 234 | # DF 3 - basic types |
| 235 | df = pd.DataFrame( |
| 236 | dict( |
| 237 | str_col=[str(x) for x in range(30)], |
| 238 | cat_col=["a" for x in range(30)], |
| 239 | int_col=[x for x in range(30)], |
| 240 | int64_col=[x for x in range(29)] + [int(-1e10)], |
| 241 | float_col=[(x + 0.1) for x in range(30)], |
| 242 | # store time as duration |
| 243 | time_col=[timedelta(seconds=x) for x in range(30)], |
| 244 | date_col=[datetime.utcnow() for x in range(30)], |
| 245 | obj_col=[(str(x), str(x)) for x in range(30)], |
| 246 | cat_obj_col=[("a", "b") for x in range(30)], |
| 247 | ) |
nothing calls this directly
no test coverage detected