Function process_df

python-client/src/datapane/common/df_processor.py:136–165

Processing steps needed before writing / after reading We only modify the dataframe to optimise size, rather than convert/infer types, e.g. no longer parsing dates from strings NOTE - this mutates the dataframe by default but returns it - use the returned copy!

(df: pd.DataFrame, copy: bool = False)

def process_df(df: pd.DataFrame, copy: bool = False) -> pd.DataFrame:
    """
    Processing steps needed before writing / after reading
    We only modify the dataframe to optimise size,
    rather than convert/infer types, e.g. no longer parsing dates from strings

    NOTE - this mutates the dataframe by default but returns it - use the returned copy!
    """
    if copy:
        df = df.copy(deep=True)

    convert_axis(df)

    # convert timedelta
    timedelta_to_str(df)

    # NOTE - pandas >= 1.3 handles downcasting of nullable values correctly
    df = df.convert_dtypes()
    downcast_numbers(df)

    # save timedeltas cols (unneeded whilst timedelta_to_str used)
    # td_col = df.select_dtypes("timedelta")
    # df[td_col.columns] = td_col
    obj_to_str(df)
    parse_categories(df)

    # convert all strings to use the arrow dtype
    str_to_arrow_str(df)

    return df


def to_df(value: Any) -> pd.DataFrame: