Skip to content

Preprocessing

rucola._preprocessing.winsorize_outliers(values)

Trim daily values above P_out = q_0.75 + 3·IQR per station.

Implements Eq. 1 from González-Rouco et al. (2001). The extreme value is replaced by P_out rather than removed, preserving the event while reducing its influence on non-resistant statistics. Requires station_id and value columns.

Source code in src/rucola/_preprocessing.py 
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
def winsorize_outliers(values: pl.DataFrame) -> pl.DataFrame:
    """Trim daily values above P_out = q_0.75 + 3·IQR per station.

    Implements Eq. 1 from González-Rouco et al. (2001). The extreme value is
    replaced by P_out rather than removed, preserving the event while reducing
    its influence on non-resistant statistics. Requires ``station_id`` and
    ``value`` columns.
    """
    thresholds = (
        values.group_by("station_id")
        .agg(
            pl.col("value").quantile(0.75).alias("q75"),
            (pl.col("value").quantile(0.75) - pl.col("value").quantile(0.25)).alias("iqr"),
        )
        .with_columns((pl.col("q75") + 3 * pl.col("iqr")).alias("p_out"))
        .select("station_id", "p_out")
    )
    return (
        values.join(thresholds, on="station_id", how="left")
        .with_columns(
            pl.when(pl.col("value") > pl.col("p_out")).then(pl.col("p_out")).otherwise(pl.col("value")).alias("value"),
        )
        .drop("p_out")
    )

rucola._preprocessing.compute_annual_totals(values, min_coverage=0.8)

Aggregate daily values to annual totals (e.g. precipitation).

Years with fewer than min_coverage × 365 valid observations are set to null. Returns one row per (station_id, year) with a date column set to Jan 1 of each year, making the result directly compatible with Rucola.

Source code in src/rucola/_preprocessing.py 
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
def compute_annual_totals(
    values: pl.DataFrame,
    min_coverage: float = 0.8,
) -> pl.DataFrame:
    """Aggregate daily values to annual totals (e.g. precipitation).

    Years with fewer than ``min_coverage × 365`` valid observations are set to
    null. Returns one row per (station_id, year) with a ``date`` column set to
    Jan 1 of each year, making the result directly compatible with ``Rucola``.
    """
    return (
        values.with_columns(pl.col("date").dt.year().alias("_year"))
        .group_by("station_id", "_year", "parameter")
        .agg(
            pl.col("value").count().alias("n_obs"),
            pl.col("value").sum().alias("value"),
        )
        .with_columns(
            pl.when(pl.col("n_obs") >= min_coverage * _DAYS_PER_YEAR)
            .then(pl.col("value"))
            .otherwise(None)
            .alias("value"),
            pl.date(pl.col("_year"), 1, 1).alias("date"),
        )
        .drop("_year", "n_obs")
        .sort("station_id", "date")
    )

rucola._preprocessing.compute_annual_means(values, min_coverage=0.8)

Aggregate daily values to annual means (e.g. temperature).

Years with fewer than min_coverage × 365 valid observations are set to null. Returns one row per (station_id, year) compatible with Rucola.

Source code in src/rucola/_preprocessing.py 
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
def compute_annual_means(
    values: pl.DataFrame,
    min_coverage: float = 0.8,
) -> pl.DataFrame:
    """Aggregate daily values to annual means (e.g. temperature).

    Years with fewer than ``min_coverage × 365`` valid observations are set to
    null. Returns one row per (station_id, year) compatible with ``Rucola``.
    """
    return (
        values.with_columns(pl.col("date").dt.year().alias("_year"))
        .group_by("station_id", "_year", "parameter")
        .agg(
            pl.col("value").count().alias("n_obs"),
            pl.col("value").mean().alias("value"),
        )
        .with_columns(
            pl.when(pl.col("n_obs") >= min_coverage * _DAYS_PER_YEAR)
            .then(pl.col("value"))
            .otherwise(None)
            .alias("value"),
            pl.date(pl.col("_year"), 1, 1).alias("date"),
        )
        .drop("_year", "n_obs")
        .sort("station_id", "date")
    )