Function clean_ohlcv_df

crates/openquant/src/data_processing.rs:161–197

(
    df: &DataFrame,
    keep_last: bool,
)

}

pub fn clean_ohlcv_df(
    df: &DataFrame,
    keep_last: bool,
) -> Result<(DataFrame, DataQualityReport), String> {
    require_ohlcv_columns(df)?;

    if df.height() == 0 {
        let empty = sort_ohlcv_df(df)?;
        let report = DataQualityReport {
            row_count: 0,
            symbol_count: 0,
            duplicate_key_count: 0,
            gap_interval_count: 0,
            ts_min: None,
            ts_max: None,
            rows_removed_by_deduplication: 0,
        };
        return Ok((empty, report));
    }

    let sorted = sort_ohlcv_df(df)?;
    let before = sorted.height();

    let cleaned = sorted
        .unique_stable(
            Some(&["symbol".to_string(), "ts_us".to_string()]),
            if keep_last { UniqueKeepStrategy::Last } else { UniqueKeepStrategy::First },
            None,
        )
        .map_err(|e| format!("polars unique failed: {e}"))?;

    let removed = before.saturating_sub(cleaned.height());
    let mut report = quality_report_from_sorted_df(&cleaned, removed)?;
    report.duplicate_key_count = 0;

    Ok((cleaned, report))
}

pub fn align_calendar_df(df: &DataFrame, interval_seconds: i64) -> Result<DataFrame, String> {
    if interval_seconds <= 0 {