Coerce dataframe to dtypes safely Operates in place Parameters ---------- df: Pandas DataFrame dtypes: dict like {'x': float}
(df, dtypes)
| 89 | |
| 90 | |
| 91 | def coerce_dtypes(df, dtypes): |
| 92 | """Coerce dataframe to dtypes safely |
| 93 | |
| 94 | Operates in place |
| 95 | |
| 96 | Parameters |
| 97 | ---------- |
| 98 | df: Pandas DataFrame |
| 99 | dtypes: dict like {'x': float} |
| 100 | """ |
| 101 | bad_dtypes = [] |
| 102 | bad_dates = [] |
| 103 | errors = [] |
| 104 | for c in df.columns: |
| 105 | if c in dtypes and df.dtypes[c] != dtypes[c]: |
| 106 | actual = df.dtypes[c] |
| 107 | desired = dtypes[c] |
| 108 | if is_float_dtype(actual) and is_integer_dtype(desired): |
| 109 | bad_dtypes.append((c, actual, desired)) |
| 110 | elif is_object_dtype(actual) and is_datetime64_any_dtype(desired): |
| 111 | # This can only occur when parse_dates is specified, but an |
| 112 | # invalid date is encountered. Pandas then silently falls back |
| 113 | # to object dtype. Since `object_array.astype(datetime)` will |
| 114 | # silently overflow, error here and report. |
| 115 | bad_dates.append(c) |
| 116 | else: |
| 117 | try: |
| 118 | df[c] = df[c].astype(dtypes[c]) |
| 119 | except Exception as e: |
| 120 | bad_dtypes.append((c, actual, desired)) |
| 121 | errors.append((c, e)) |
| 122 | |
| 123 | if bad_dtypes: |
| 124 | if errors: |
| 125 | ex = "\n".join( |
| 126 | f"- {c}\n {e!r}" for c, e in sorted(errors, key=lambda x: str(x[0])) |
| 127 | ) |
| 128 | exceptions = ( |
| 129 | "The following columns also raised exceptions on " |
| 130 | f"conversion:\n\n{ex}\n\n" |
| 131 | ) |
| 132 | extra = "" |
| 133 | else: |
| 134 | exceptions = "" |
| 135 | # All mismatches are int->float, also suggest `assume_missing=True` |
| 136 | extra = ( |
| 137 | "\n\nAlternatively, provide `assume_missing=True` " |
| 138 | "to interpret\n" |
| 139 | "all unspecified integer columns as floats." |
| 140 | ) |
| 141 | |
| 142 | bad_dtypes = sorted(bad_dtypes, key=lambda x: str(x[0])) |
| 143 | table = asciitable(["Column", "Found", "Expected"], bad_dtypes) |
| 144 | dtype_kw = "dtype={{{}}}".format( |
| 145 | ",\n ".join(f"{k!r}: '{v}'" for (k, v, _) in bad_dtypes) |
| 146 | ) |
| 147 | |
| 148 | dtype_msg = ( |
no test coverage detected