Coerce dataframe to dtypes safely Operates in place Parameters ---------- df: Pandas DataFrame dtypes: dict like {'x': float}
(df, dtypes)
| 90 | |
| 91 | |
| 92 | def coerce_dtypes(df, dtypes): |
| 93 | """Coerce dataframe to dtypes safely |
| 94 | |
| 95 | Operates in place |
| 96 | |
| 97 | Parameters |
| 98 | ---------- |
| 99 | df: Pandas DataFrame |
| 100 | dtypes: dict like {'x': float} |
| 101 | """ |
| 102 | bad_dtypes = [] |
| 103 | bad_dates = [] |
| 104 | errors = [] |
| 105 | for c in df.columns: |
| 106 | if c in dtypes and df.dtypes[c] != dtypes[c]: |
| 107 | actual = df.dtypes[c] |
| 108 | desired = dtypes[c] |
| 109 | if is_float_dtype(actual) and is_integer_dtype(desired): |
| 110 | bad_dtypes.append((c, actual, desired)) |
| 111 | elif is_object_dtype(actual) and is_datetime64_any_dtype(desired): |
| 112 | # This can only occur when parse_dates is specified, but an |
| 113 | # invalid date is encountered. Pandas then silently falls back |
| 114 | # to object dtype. Since `object_array.astype(datetime)` will |
| 115 | # silently overflow, error here and report. |
| 116 | bad_dates.append(c) |
| 117 | else: |
| 118 | try: |
| 119 | df[c] = df[c].astype(dtypes[c]) |
| 120 | except Exception as e: |
| 121 | bad_dtypes.append((c, actual, desired)) |
| 122 | errors.append((c, e)) |
| 123 | |
| 124 | if bad_dtypes: |
| 125 | if errors: |
| 126 | ex = "\n".join( |
| 127 | f"- {c}\n {e!r}" for c, e in sorted(errors, key=lambda x: str(x[0])) |
| 128 | ) |
| 129 | exceptions = ( |
| 130 | "The following columns also raised exceptions on " |
| 131 | "conversion:\n\n%s\n\n" |
| 132 | ) % ex |
| 133 | extra = "" |
| 134 | else: |
| 135 | exceptions = "" |
| 136 | # All mismatches are int->float, also suggest `assume_missing=True` |
| 137 | extra = ( |
| 138 | "\n\nAlternatively, provide `assume_missing=True` " |
| 139 | "to interpret\n" |
| 140 | "all unspecified integer columns as floats." |
| 141 | ) |
| 142 | |
| 143 | bad_dtypes = sorted(bad_dtypes, key=lambda x: str(x[0])) |
| 144 | table = asciitable(["Column", "Found", "Expected"], bad_dtypes) |
| 145 | dtype_kw = "dtype={%s}" % ",\n ".join( |
| 146 | f"{k!r}: '{v}'" for (k, v, _) in bad_dtypes |
| 147 | ) |
| 148 | |
| 149 | dtype_msg = ( |
no test coverage detected