Detect and converts categories
(data: pd.DataFrame)
| 67 | |
| 68 | |
| 69 | def parse_categories(data: pd.DataFrame): |
| 70 | """Detect and converts categories""" |
| 71 | |
| 72 | def criteria(ser: pd.Series) -> bool: |
| 73 | """Decides whether to convert into categorical""" |
| 74 | nunique: int = ser.nunique() |
| 75 | |
| 76 | if nunique <= 20 and (nunique != ser.size): |
| 77 | # few unique values => make it a category regardless of the proportion |
| 78 | return True |
| 79 | |
| 80 | prop_unique = (nunique + 1) / (ser.size + 1) # + 1 for nan |
| 81 | |
| 82 | if prop_unique <= 0.05: |
| 83 | # a lot of redundant information => categories are more compact |
| 84 | return True |
| 85 | |
| 86 | return False |
| 87 | |
| 88 | def try_to_category(ser: pd.Series) -> pd.Series: |
| 89 | return ser.astype("category") if criteria(ser) else ser |
| 90 | |
| 91 | potential_cats = data.select_dtypes(["string", "object"]) |
| 92 | data[potential_cats.columns] = potential_cats.apply(try_to_category) |
| 93 | |
| 94 | |
| 95 | def obj_to_str(df: pd.DataFrame): |