Function preprocess_data

src/data_preprocessing.py:18–62

Preprocesses the dataset by handling missing values, scaling numerical features, and encoding categorical features. Args: df (pd.DataFrame): The dataset to preprocess. numerical_features (list): List of numerical feature column names. categorical_features (list)

(df, numerical_features, categorical_features, missing_strategy="mean", scale=True, encode=True)

def preprocess_data(df, numerical_features, categorical_features, missing_strategy="mean", scale=True, encode=True):
    """
    Preprocesses the dataset by handling missing values, scaling numerical features,
    and encoding categorical features.

    Args:
        df (pd.DataFrame): The dataset to preprocess.
        numerical_features (list): List of numerical feature column names.
        categorical_features (list): List of categorical feature column names.
        missing_strategy (str): Strategy for imputing missing values ('mean', 'median', or 'most_frequent').
        scale (bool): Whether to scale numerical features.
        encode (bool): Whether to encode categorical features.

    Returns:
        pd.DataFrame: The preprocessed dataset.
        ColumnTransformer: The fitted transformer for future use.
    """
    transformers = []

    # Handle numerical features
    if numerical_features:
        num_transformer = []
        if missing_strategy:
            num_transformer.append(("imputer", SimpleImputer(strategy=missing_strategy)))
        if scale:
            num_transformer.append(("scaler", StandardScaler()))
        transformers.append(("num", Pipeline(num_transformer), numerical_features))

    # Handle categorical features
    if categorical_features:
        cat_transformer = []
        if missing_strategy:
            cat_transformer.append(("imputer", SimpleImputer(strategy="most_frequent")))
        if encode:
            cat_transformer.append(("encoder", OneHotEncoder(handle_unknown="ignore")))
        transformers.append(("cat", Pipeline(cat_transformer), categorical_features))

    # Create a column transformer
    preprocessor = ColumnTransformer(transformers, remainder="passthrough")

    # Apply transformations
    processed_array = preprocessor.fit_transform(df)
    processed_df = pd.DataFrame(processed_array, columns=preprocessor.get_feature_names_out())

    return processed_df, preprocessor


def detect_outliers_iqr(df, numerical_features, threshold=1.5):