Source code for livef1.data_processing.silver_functions_polars

"""
Polars implementation of :mod:`livef1.data_processing.silver_functions`.

Used when the polars dataframe backend is configured::

    livef1.configure(dataframe_backend="polars")
    session = livef1.get_session(2026, "British", "Race")
    session.generate()
    session.laps                        # polars.DataFrame

The functions take the session's polars bronze tables (built by
:mod:`livef1.data_processing.bronze_polars`) and reproduce the pandas silver
functions with Polars expressions. The bronze DataFrames are never mutated.
"""
import re
from datetime import timedelta

import numpy as np
import polars as pl
from scipy.interpolate import interp1d

from ..utils.constants import (
    interpolation_map,
    silver_cartel_col_order,
    silver_laps_col_order,
    silver_race_control_messages_col_order,
    FIA_CATEGORY_SCOPE_RULES,
    penalty_types,
)
from .bronze_polars import utc_to_datetime as _utc_to_datetime, str_to_duration as _str_to_duration

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def _select(df, columns=None):
    """Keep only ``columns`` of a bronze table (missing ones are ignored); categoricals become strings."""
    if df is None:
        return None
    if columns is not None:
        df = df.select([c for c in columns if c in df.columns])
    return df.with_columns(pl.col(pl.Categorical, pl.Enum).cast(pl.String))


def _dt_lit(value):
    """Session datetime (python datetime / numpy datetime64) -> polars datetime[ns] literal."""
    return pl.lit(value).cast(pl.Datetime("ns"))


def _gap_to_float(col):
    """
    GapToLeader / IntervalToPositionAhead parsing used by the laps state machine.

    null -> null (value absent, keep the previous one), "...LAP..." -> 0.0,
    "...L..." -> NaN (value present but explicitly reset to None), else float.
    """
    c = pl.col(col)
    return (
        pl.when(c.is_null()).then(None)
        .when(c.str.contains("LAP", literal=True)).then(0.0)
        .when(c.str.contains("L", literal=True)).then(float("nan"))
        .otherwise(c.cast(pl.Float64, strict=False))
        .alias(col)
    )


def _ensure_columns(df, cols, dtype=pl.String):
    missing = [c for c in cols if c not in df.columns]
    return df.with_columns([pl.lit(None, dtype).alias(c) for c in missing]) if missing else df


# ---------------------------------------------------------------------------
# Laps
# ---------------------------------------------------------------------------

def add_track_status(laps_df, df_trackstatus):
    start = laps_df["LapStartTime"].to_list()
    end = laps_df["LapStartTime"].shift(-1).to_list()

    # Missing lap ends are extrapolated from the previous lap end + lap interval (sequential, like pandas).
    for i in range(1, len(end)):
        if end[i] is None:
            prev, prev_start, curr_start = end[i - 1], start[i - 1], start[i]
            end[i] = None if None in (prev, prev_start, curr_start) else prev + (curr_start - prev_start)

    laps_df = laps_df.with_columns(pl.Series("LapEndTime", end, dtype=pl.Duration("us")))
    if laps_df["LapEndTime"].null_count() == laps_df.height:
        return laps_df.with_columns(pl.lit(None, pl.String).alias("TrackStatus"))

    # Outer merge on LapEndTime == timestamp, sorted, then LapNo backfilled:
    # every track status event belongs to the first lap ending at/after it.
    timeline = pl.concat([
        df_trackstatus.select(
            pl.col("timestamp").cast(pl.Duration("us")).alias("LapEndTime"),
            pl.lit(None, pl.Int64).alias("LapNo"),
            pl.col("Status").cast(pl.String),
            pl.lit(0).alias("_order"),
        ),
        laps_df.select(
            "LapEndTime",
            pl.col("LapNo").cast(pl.Int64),
            pl.lit(None, pl.String).alias("Status"),
            pl.lit(1).alias("_order"),
        ),
    ]).sort(["LapEndTime", "_order"], nulls_last=True, maintain_order=True)

    status_df = (
        timeline
        .with_columns(pl.col("LapNo").backward_fill())
        .drop_nulls("LapNo")
        .group_by("LapNo")
        .agg(pl.col("Status").drop_nulls().unique().sort().str.join(",").alias("TrackStatus"))
        .with_columns(pl.col("TrackStatus").replace("", "1"))
    )
    return laps_df.join(status_df, on="LapNo", how="left", maintain_order="left")


def delete_laps(laps_df, df_rcm):
    tokens = pl.col("Message").str.split(" ")
    df_rcm_del = (
        df_rcm
        .filter((pl.col("Category") == "Other") & (tokens.list.get(0, null_on_oob=True) == "CAR"))
        .select(
            "Message",
            tokens.list.get(1, null_on_oob=True).alias("deleted_driver"),
            tokens.list.get(3, null_on_oob=True).alias("deleted_type"),
            tokens.list.len().alias("_n_tokens"),
            tokens.alias("_tokens"),
        )
        .with_columns(
            pl.when(pl.col("deleted_type") == "TIME")
            .then(pl.col("_tokens").list.get(4, null_on_oob=True))
            .alias("deleted_time")
        )
    )

    # Drop deletions that were later reinstated (and the reinstatement message itself)
    reinstated = df_rcm_del.filter(
        pl.col("Message").str.contains("REINSTATED", literal=True) & (pl.col("deleted_type") == "TIME")
    ).select("deleted_driver", "deleted_time").unique()
    df_rcm_del = df_rcm_del.join(reinstated, on=["deleted_driver", "deleted_time"], how="anti", maintain_order="left")

    laps_df = laps_df.with_columns(pl.lit(False).alias("IsDeleted"))
    if df_rcm_del.height == 0:
        return laps_df

    deletions = (
        df_rcm_del
        .with_columns(
            pl.when(pl.col("_n_tokens") > 12)
            .then(
                pl.when(pl.col("deleted_type") == "LAP").then(pl.col("_tokens").list.get(12, null_on_oob=True))
                .when(pl.col("deleted_type") == "TIME").then(pl.col("_tokens").list.get(13, null_on_oob=True))
            )
            .cast(pl.Int64, strict=False)
            .alias("LapNo")
        )
        .drop_nulls("LapNo")
        .select(pl.col("deleted_driver").alias("DriverNo"), "LapNo", pl.col("Message").alias("DeletionMessage"))
        .unique(subset=["DriverNo", "LapNo"], keep="last", maintain_order=True)
    )
    if deletions.height == 0:
        return laps_df

    return (
        laps_df
        .join(deletions.with_columns(pl.lit(True).alias("_deleted_hit")), on=["DriverNo", "LapNo"], how="left", maintain_order="left")
        .with_columns(pl.col("_deleted_hit").fill_null(False).alias("IsDeleted"))
        .drop("_deleted_hit")
    )


def generate_laps_table(session, df_exp, df_rcm, df_tyre, df_track):

    df_exp = _select(df_exp, [
        "SessionKey", "DriverNo", "timestamp", "RacingNumber", "Stopped", "InPit", "PitOut", "LastLapTime_Value",
        "Sectors_0_Value", "Sectors_1_Value", "Sectors_2_Value",
        "Sectors_0_PreviousValue", "Sectors_1_PreviousValue", "Sectors_2_PreviousValue",
        "Speeds_I1_Value", "Speeds_I2_Value", "Speeds_FL_Value", "Speeds_ST_Value",
        "Position", "GapToLeader", "IntervalToPositionAhead_Value",
    ])
    df_rcm = _select(df_rcm, ["Category", "Message"])
    df_tyre = _select(df_tyre)
    df_track = _select(df_track)

    session_key = df_exp["SessionKey"][0]

    sector_value_cols = ["Sectors_0_Value", "Sectors_1_Value", "Sectors_2_Value"]
    sector_prev_cols = ["Sectors_0_PreviousValue", "Sectors_1_PreviousValue", "Sectors_2_PreviousValue"]
    speedTrap_cols = {
        "Speeds_I1_Value": "Speed_I1",
        "Speeds_I2_Value": "Speed_I2",
        "Speeds_FL_Value": "Speed_FL",
        "Speeds_ST_Value": "Speed_ST",
    }
    misc_cols = {"Position": "Position"}
    if session.type == "Race":
        misc_cols["GapToLeader"] = "GapToLeader"
        misc_cols["IntervalToPositionAhead_Value"] = "IntervalToPositionAhead"

    # Record columns and their dtypes (the pandas `cols`, minus the `None` keys)
    lap_schema = {
        "LapNo": pl.Int64,
        "LapTime": pl.Duration("us"),
        "PitIn": pl.Duration("us"),
        "PitOut": pl.Duration("us"),
        "Sector1_Time": pl.Duration("us"),
        "Sector2_Time": pl.Duration("us"),
        "Sector3_Time": pl.Duration("us"),
        **{v: pl.Float64 for v in speedTrap_cols.values()},
        **{v: (pl.String if k == "Position" else pl.Float64) for k, v in misc_cols.items()},
        "NoPits": pl.Int64,
        "LapStartTime": pl.Duration("us"),
    }
    cols = [c for c in lap_schema if c not in ("NoPits", "LapStartTime")]

    # Columns visited by the state machine, in the same order as the pandas row iteration
    loop_cols = ["InPit", "PitOut"] + sector_value_cols + sector_prev_cols + list(speedTrap_cols) + list(misc_cols)

    # Vectorised preprocessing: '' -> null, then parse timings / speeds / gaps once for all drivers
    df_test = (
        _ensure_columns(df_exp, ["LastLapTime_Value", "RacingNumber", "Stopped", "InPit", "PitOut"]
                        + sector_value_cols + sector_prev_cols + list(speedTrap_cols) + list(misc_cols))
        .select(["DriverNo", "timestamp", "LastLapTime_Value", "RacingNumber", "Stopped"] + loop_cols)
        .with_columns(pl.when(pl.col(pl.String) == "").then(None).otherwise(pl.col(pl.String)).name.keep())
        .filter(pl.col("RacingNumber").is_null())
        .with_columns(
            [_str_to_duration(c) for c in sector_value_cols + sector_prev_cols + ["LastLapTime_Value"]]
            + [pl.col(c).cast(pl.Float64, strict=False) for c in speedTrap_cols]
            + [_gap_to_float(c) for c in misc_cols if c != "Position"]
            + [pl.col("timestamp").cast(pl.Duration("us"))]
        )
    )

    def new_record(prev=None):
        if prev is None:
            record = {key: None for key in cols}
            record["LapNo"], record["NoPits"] = 1, 0
            return record
        record = {key: None for key in prev}
        record["LapNo"] = prev["LapNo"] + 1
        record["NoPits"] = prev["NoPits"]
        record["Position"] = prev["Position"]
        return record

    def close_lap(laps, record):
        if record["LapTime"] is None and None not in (record["Sector1_Time"], record["Sector2_Time"], record["Sector3_Time"]):
            record["LapTime"] = record["Sector1_Time"] + record["Sector2_Time"] + record["Sector3_Time"]
        laps.append(record)
        return new_record(record)

    all_laps = []

    for (driver_no,), df_driver in df_test.partition_by("DriverNo", as_dict=True, maintain_order=True).items():
        laps, record = [], new_record()
        last_record_ts = timedelta(seconds=0)

        for row in df_driver.iter_rows(named=True):
            ts = row["timestamp"]

            if row["Stopped"] == True:
                record = close_lap(laps, record)
                continue

            if row["LastLapTime_Value"] is not None:
                if row["Sectors_2_Value"] is not None:
                    record["LapTime"] = row["LastLapTime_Value"]
                elif row["Sectors_2_PreviousValue"] is not None and laps:
                    laps[-1]["LapTime"] = row["LastLapTime_Value"]

            for sc_key in loop_cols:
                sc_value = row[sc_key]
                if sc_value is None:
                    continue

                if sc_key in speedTrap_cols:
                    record[speedTrap_cols[sc_key]] = sc_value

                elif sc_key == "InPit":
                    if sc_value == 1:
                        record["PitIn"] = ts
                elif sc_key == "PitOut":
                    if sc_value == True:
                        record["PitOut"] = ts
                        record["NoPits"] += 1

                elif sc_key == "Position":
                    record["Position"] = sc_value
                elif sc_key in misc_cols:
                    record[misc_cols[sc_key]] = None if sc_value != sc_value else sc_value  # NaN -> explicit reset

                else:
                    _, sc_no, key_type = sc_key.split("_")
                    sector_col = f"Sector{int(sc_no) + 1}_Time"

                    if key_type == "Value":
                        if record[sector_col] is None:
                            record[sector_col] = sc_value
                            last_record_ts = ts
                            if sc_no == "2":
                                record = close_lap(laps, record)
                                record["LapStartTime"] = ts
                        elif sc_value == record[sector_col]:
                            pass
                        elif ts - last_record_ts > timedelta(seconds=10):
                            record = close_lap(laps, record)
                            record[sector_col] = sc_value
                            record["LapStartTime"] = ts - sc_value
                            last_record_ts = ts

                    elif key_type == "PreviousValue":
                        if sc_no != "2":
                            record[sector_col] = sc_value
                            last_record_ts = ts
                        elif laps:
                            laps[-1][sector_col] = sc_value
                            last_record_ts = ts

        if not laps:
            continue

        laps_df = (
            pl.DataFrame(laps, schema=lap_schema)
            .with_columns(pl.lit(driver_no).alias("DriverNo"))
            # Lap start = previous lap start + previous lap time, otherwise the recorded start
            .with_columns((pl.col("LapStartTime") + pl.col("LapTime")).shift(1).fill_null(pl.col("LapStartTime")).alias("LapStartTime"))
        )
        all_laps.append(add_track_status(laps_df, df_track))

    all_laps_df = pl.concat(all_laps, how="diagonal_relaxed")
    all_laps_df = all_laps_df.with_columns(
        (pl.col("LapStartTime").cast(pl.Duration("ns")) + _dt_lit(session.first_datetime)).fill_null(_dt_lit(session.session_start_datetime)).alias("LapStartDate")
    )
    if all_laps_df.height > 1:
        r0, r1 = all_laps_df.row(0, named=True), all_laps_df.row(1, named=True)
        fill = None if r1["LapStartTime"] is None else r1["LapStartTime"] - (r1["LapStartDate"] - r0["LapStartDate"])
        all_laps_df = all_laps_df.with_columns(pl.col("LapStartTime").fill_null(pl.lit(fill, pl.Duration("us"))))

    # Fill position data with timing data (first non-null Position per driver from the RacingNumber rows)
    first_pos = (
        df_exp.filter(pl.col("RacingNumber").is_not_null())
        .sort("timestamp", maintain_order=True)
        .group_by("DriverNo", maintain_order=True)
        .agg(pl.col("Position").drop_nulls().first().alias("_first_pos"))
    )
    all_laps_df = (
        all_laps_df.join(first_pos, on="DriverNo", how="left", maintain_order="left")
        .with_columns(
            pl.when((pl.col("LapNo") == 1) & pl.col("Position").is_null())
            .then(pl.col("_first_pos")).otherwise(pl.col("Position")).alias("Position")
        )
        .drop("_first_pos")
    )

    # Delete laps
    all_laps_df = delete_laps(all_laps_df, df_rcm)

    # Add session data
    all_laps_df = all_laps_df.with_columns(pl.lit(session_key).alias("SessionKey"))

    # Add pit data
    if "PitStopSeries" in session.topic_names_info:
        df_pit = _select(session.get_data("PitStopSeries", level="bronze")).select(
            pl.col("RacingNumber").alias("DriverNo"),
            "Utc",
            pl.col("PitStopTime").alias("PitStopDuration"),
            pl.col("PitLaneTime").alias("PitLaneDuration"),
            pl.col("Lap").cast(pl.String).alias("LapNo"),
        ).with_row_index("_pit_idx")
        df_pit = _utc_to_datetime(df_pit)

        # Where LapNo is missing or empty, infer from pit time vs lap start times
        missing = df_pit.filter(pl.col("LapNo").is_null() | (pl.col("LapNo").str.strip_chars() == ""))
        if missing.height:
            inferred = (
                missing.drop_nulls("Utc").select("_pit_idx", "DriverNo", "Utc")
                .join(all_laps_df.drop_nulls("LapStartDate").select("DriverNo", "LapStartDate", "LapNo"), on="DriverNo")
                .filter(pl.col("LapStartDate") <= pl.col("Utc"))
                .group_by("_pit_idx")
                .agg(pl.col("LapNo").get(pl.col("LapStartDate").arg_max()).cast(pl.String).alias("_lap"))
            )
            df_pit = (
                df_pit.join(inferred, on="_pit_idx", how="left", maintain_order="left")
                .with_columns(pl.coalesce("_lap", "LapNo").alias("LapNo"))
                .drop("_lap")
            )

        df_pit = df_pit.drop("Utc", "_pit_idx").with_columns(
            pl.col("LapNo").cast(pl.Float64, strict=False).fill_null(-1).cast(pl.Int64)
        )
        all_laps_df = all_laps_df.join(df_pit, on=["DriverNo", "LapNo"], how="left", maintain_order="left")

    # Add tyre data: union of laps and tyre events sorted by (DriverNo, LapEndTime), then forward-filled per driver
    tyre_cols = ["Compound", "New", "TyreAge"]
    # A lap without LapTime (e.g. lap 1) ends when the driver's next lap starts; without a next lap, use its own start
    all_laps_df = all_laps_df.sort(["DriverNo", "LapNo"], maintain_order=True).with_columns(
        (pl.col("LapStartTime") + pl.col("LapTime"))
        .fill_null(pl.col("LapStartTime").shift(-1).over("DriverNo"))
        .fill_null(pl.col("LapStartTime"))
        .alias("LapEndTime")
    )
    tyre = df_tyre.select(
        "DriverNo",
        pl.col("timestamp").cast(pl.Duration("us")).alias("LapEndTime"),
        "Compound", "New",
        pl.col("TotalLaps").cast(pl.Float64).alias("TyreAge"),
    )
    all_laps_df = (
        pl.concat([tyre.with_columns(pl.lit(0).alias("_order")), all_laps_df.with_columns(pl.lit(1).alias("_order"))], how="diagonal_relaxed")
        .sort(["DriverNo", "LapEndTime", "_order"], nulls_last=True, maintain_order=True)
        .with_columns(pl.col(tyre_cols).forward_fill().over("DriverNo"))
        .filter(pl.col("_order") == 1)
        .drop("_order")
        .drop_nulls("SessionKey")
    )

    all_laps_df = _ensure_columns(all_laps_df, [c for c in silver_laps_col_order if c != "Driver"])
    all_laps_df = all_laps_df.select([c for c in silver_laps_col_order if c != "Driver"]).sort(["DriverNo", "LapNo"], maintain_order=True)

    # Add driver data (Driver objects, same as the pandas table)
    return all_laps_df.with_columns(
        pl.Series("Driver", [session.drivers.get(d) for d in all_laps_df["DriverNo"]], dtype=pl.Object)
    )


# ---------------------------------------------------------------------------
# Car telemetry
# ---------------------------------------------------------------------------

def _interpolate_quadratic(x, y):
    """pandas ``interpolate(method="quadratic")``: scipy interp1d on the index, leading/trailing NaN kept."""
    y = y.astype(np.float64)
    valid = ~np.isnan(y)
    if valid.all() or valid.sum() < 3:
        return y
    out = y.copy()
    out[~valid] = interp1d(x[valid], y[valid], kind="quadratic", fill_value=np.nan, bounds_error=False)(x[~valid])
    first, last = np.argmax(valid), len(valid) - np.argmax(valid[::-1])
    out[:first] = np.nan
    out[last:] = np.nan
    return out


def interpolate_driver(df_driver):
    """Per-driver interpolation following ``interpolation_map`` (skips columns that are < 20% filled)."""
    exprs = []
    x = None
    for col, method in interpolation_map.items():
        if col not in df_driver.columns or df_driver[col].count() < df_driver.height * 0.2:
            continue
        c = pl.col(col)
        if method == "ffill":
            exprs.append(c.forward_fill())
        elif method == "linear":
            # pandas linear interpolation also fills trailing NaN with the last value
            exprs.append(c.interpolate().forward_fill())
        elif method == "index":
            exprs.append(c.interpolate_by("Utc").forward_fill())
        elif method in ("quadratic", "polynomial"):
            if x is None:
                x = df_driver["Utc"].cast(pl.Int64).to_numpy().astype(np.float64)
            values = _interpolate_quadratic(x, df_driver[col].cast(pl.Float64).to_numpy())
            exprs.append(pl.Series(col, values).cast(df_driver.schema[col]))
    return df_driver.with_columns(exprs) if exprs else df_driver


def add_distance_to_lap(df_driver, start_x, start_y, x_coeff, y_coeff):
    """Vectorised ``add_distance_to_lap`` applied to every lap of a driver at once."""
    dt = pl.col("timestamp").diff().dt.total_microseconds().fill_null(0) / 1e6
    lap_distance = (((pl.col("Speed") + pl.col("Speed").shift(1)) / 2 / 3.6) * dt).cum_sum()

    x0, y0 = pl.col("X").first(), pl.col("Y").first()
    direction = pl.when(((x0 - start_x) / x_coeff > 0) & ((y0 - start_y) / y_coeff > 0)).then(1).otherwise(-1)
    offset = direction * ((x0 - start_x) ** 2 + (y0 - start_y) ** 2).sqrt() / 10

    return df_driver.with_columns((offset + lap_distance).over("LapNo").alias("Distance"))


def add_track_status_telemetry(telemetry_df, df_track):
    return (
        telemetry_df
        .join(df_track.select("timestamp", pl.col("Status").cast(pl.String).alias("TrackStatus"))
              .unique("timestamp", keep="last", maintain_order=True),
              on="timestamp", how="left", maintain_order="left")
        .with_columns(pl.col("TrackStatus").forward_fill())
        .drop_nulls("SessionKey")
    )


def add_lineposition(telemetry_df, df_tmg):
    # Outer join + ffill on the timing rows == latest non-null timing Position at/before each sample
    positions = df_tmg.select("timestamp", "Position").drop_nulls("Position").sort("timestamp", maintain_order=True)
    return telemetry_df.sort("timestamp", maintain_order=True).join_asof(positions, on="timestamp", strategy="backward")


def add_interval_to_position_ahead(telemetry_df, df_tmg):
    # Same filter as pandas: nulls are kept (str.contains on NA -> False), laps behind and '' are dropped
    temp_tmg = (
        df_tmg.select("timestamp", "IntervalToPositionAhead_Value")
        .filter(
            ~pl.col("IntervalToPositionAhead_Value").str.contains("L", literal=True).fill_null(False)
            & (pl.col("IntervalToPositionAhead_Value") != "").fill_null(True)
        )
        .sort("timestamp", maintain_order=True)
    )
    return (
        telemetry_df.join_asof(temp_tmg, on="timestamp", strategy="backward")
        .with_columns(pl.col("IntervalToPositionAhead_Value").cast(pl.Float64).alias("IntervalToPositionAhead"))
        .drop("IntervalToPositionAhead_Value")
    )


def assign_regions(df_corners):
    """``np.select`` over track regions as a when/then chain (first match wins, default None)."""
    expr = pl.lit(None, pl.String)
    distance = pl.col("Distance")
    for row in reversed(list(df_corners.iter_rows(named=True))):
        if row["corner_end"] >= row["corner_start"]:
            cond = (distance >= row["corner_start"]) & (distance < row["corner_end"])
        else:
            cond = (distance >= row["corner_start"]) | (distance < row["corner_end"])
        expr = pl.when(cond).then(pl.lit(row["name"])).otherwise(expr)
    return expr.alias("TrackRegion")


[docs] def generate_car_telemetry_table(session, df_car, df_pos, df_tyre, laps, df_track, df_tmg, df_circuits): """ Polars twin of ``silver_functions.generate_car_telemetry_table``. ``laps`` must be the Polars laps table produced by :func:`generate_laps_table`. """ # Only the columns the silver table uses are converted (bronze `timestamp` is recomputed from Utc) df_car = _utc_to_datetime(_select(df_car, ["SessionKey", "DriverNo", "Utc", "RPM", "Speed", "GearNo", "Throttle", "Brake", "DRS"])) df_pos = _utc_to_datetime(_select(df_pos, ["DriverNo", "Utc", "Status", "X", "Y", "Z"])) df_tyre = _select(df_tyre, ["DriverNo", "timestamp", "Compound", "New", "TotalLaps"]).with_columns(pl.col("timestamp").cast(pl.Duration("ns"))) df_tmg = _select(df_tmg, ["DriverNo", "timestamp", "Position", "IntervalToPositionAhead_Value"]).with_columns(pl.col("timestamp").cast(pl.Duration("ns"))) df_track = _select(df_track, ["timestamp", "Status"]).with_columns(pl.col("timestamp").cast(pl.Duration("ns"))) df_circuits = _select(df_circuits) first_datetime = _dt_lit(session.first_datetime) circuit = session.meeting.circuit has_start = hasattr(circuit, "start_coordinates") tmg_has_interval = "IntervalToPositionAhead_Value" in df_tmg.columns tmg_by_driver = df_tmg.partition_by("DriverNo", as_dict=True) empty_tmg = df_tmg.clear() # Tyre events forward-filled per driver: an as-of join on them equals the pandas outer join + ffill tyre_by_driver = ( df_tyre.select("DriverNo", "timestamp", "Compound", "New", pl.col("TotalLaps").cast(pl.Float64).alias("TyreAge")) .sort(["DriverNo", "timestamp"], maintain_order=True) .with_columns(pl.col("Compound", "New", "TyreAge").forward_fill().over("DriverNo")) .partition_by("DriverNo", as_dict=True, include_key=False) ) empty_tyre = pl.DataFrame(schema={"timestamp": pl.Duration("ns"), "Compound": pl.String, "New": pl.String, "TyreAge": pl.Float64}) # Car and position data are joined driver by driver (in sorted DriverNo order, like the pandas # global sort) so only one driver's joined samples are in memory at a time. car_by_driver = df_car.with_columns(pl.lit(True).alias("_car")).partition_by("DriverNo", as_dict=True) pos_by_driver = df_pos.with_columns(pl.lit(True).alias("_pos")).partition_by("DriverNo", as_dict=True) del df_car, df_pos empty_car = pl.DataFrame(schema=next(iter(car_by_driver.values())).schema) if car_by_driver else None empty_pos = pl.DataFrame(schema=next(iter(pos_by_driver.values())).schema) if pos_by_driver else None last_car_status = None all_drivers_data = [] for key in sorted(set(car_by_driver) | set(pos_by_driver)): driver_no = key[0] df_driver = ( car_by_driver.pop(key, empty_car) .join(pos_by_driver.pop(key, empty_pos), on=["DriverNo", "Utc"], how="full", coalesce=True) .sort("Utc", maintain_order=True) .rename({"Status": "CarStatus"}) .with_columns( # pandas forward-fills CarStatus over all drivers at once, so it carries across drivers pl.col("CarStatus").forward_fill().fill_null(pl.lit(last_car_status, pl.String)), (pl.when(pl.col("_car")).then(pl.lit("car")).otherwise(pl.lit("")) + pl.when(pl.col("_pos")).then(pl.lit("position")).otherwise(pl.lit(""))).alias("tag"), ) .drop("_car", "_pos") ) if df_driver.height: last_car_status = df_driver["CarStatus"][-1] laps_driver = ( laps.filter(pl.col("DriverNo") == driver_no) .select("LapNo", "LapStartDate", (pl.col("LapStartDate") + pl.col("LapTime")).alias("lap_end_date")) ) df_driver = interpolate_driver(df_driver) # Lap number = lap with the latest start at/before the sample, restricted to the laps window df_driver = ( df_driver.join_asof( laps_driver.select("LapStartDate", pl.col("LapNo").cast(pl.Float64)) .drop_nulls("LapStartDate").sort("LapStartDate", maintain_order=True), left_on="Utc", right_on="LapStartDate", strategy="backward", ) .drop("LapStartDate") .filter(pl.col("Utc").is_between(laps_driver["LapStartDate"].min(), laps_driver["lap_end_date"].max())) .with_columns( pl.col("SessionKey").forward_fill().backward_fill(), (pl.col("Utc") - first_datetime).alias("timestamp"), ) ) if has_start: df_driver = add_distance_to_lap( df_driver, circuit.start_coordinates[0], circuit.start_coordinates[1], circuit.start_direction[0], circuit.start_direction[1], ) else: df_driver = df_driver.with_columns(pl.lit(None, pl.Float64).alias("Distance")) tmg_driver = tmg_by_driver.get((driver_no,), empty_tmg) df_driver = add_track_status_telemetry(df_driver, df_track) df_driver = add_lineposition(df_driver, tmg_driver) if tmg_has_interval: df_driver = add_interval_to_position_ahead(df_driver, tmg_driver) # Add tyre data df_driver = df_driver.join_asof(tyre_by_driver.get((driver_no,), empty_tyre), on="timestamp", strategy="backward") if df_circuits is not None and has_start: df_driver = df_driver.with_columns(assign_regions(df_circuits)) else: df_driver = df_driver.with_columns(pl.lit(None, pl.String).alias("TrackRegion")) all_drivers_data.append(df_driver.select([c for c in silver_cartel_col_order if c in df_driver.columns])) # rechunk=False: the per-driver chunks are referenced, not copied into a new buffer return pl.concat(all_drivers_data, how="diagonal_relaxed", rechunk=False)
# --------------------------------------------------------------------------- # Race control messages # --------------------------------------------------------------------------- def _is_valid_car_number(num_str, context_before, context_after): """True if a number next to CAR is not a lap / turn / timestamp / lap time.""" text = context_before + num_str + context_after if re.search(r'\d+\s*:\s*' + re.escape(num_str) + r'\s*:\s*\d+', text): return False if re.search(re.escape(num_str) + r'\s*:\s*\d+', context_after): return False if re.search(r'LAP\s+' + re.escape(num_str) + r'\b', text, re.IGNORECASE): return False if re.search(r'TURN\s+' + re.escape(num_str) + r'\b', text, re.IGNORECASE): return False if re.search(r'\d+\s*:\s*' + re.escape(num_str) + r'\s*\.\s*\d+', text): return False return True def parse_cars_from_message(message): """ Same parsing as the pandas pipeline. Kept as a Python UDF because the patterns rely on look-arounds, which the Polars (Rust) regex engine lacks. """ if message is None or not message: return None cars = [] message_upper = message.upper() # Case 1: "CAR 23 (ALB) ..." if re.search(r'\bCAR\s+', message_upper) and not re.search(r'\bCARS\s+', message_upper): for match in re.finditer(r'(?i)\bCAR\s+(\d+)', message): car_num = match.group(1) context_before = message[max(0, match.start() - 30):match.start()] context_after = message[match.end():min(len(message), match.end() + 30)] if _is_valid_car_number(car_num, context_before, context_after): cars.append(int(car_num)) # Case 2: "CARS 44 (HAM) AND 18 (STR) ..." elif re.search(r'\bCARS\s+', message_upper): cars_pattern = r'(?i)\bCARS\s+((?:\d+\s*\([^)]+\)(?:\s*,\s*|\s+AND\s+)?)+?)(?=\s+(?:NOTED|WILL|REVIEWED|NO|FIA|$))' cars_match = re.search(cars_pattern, message) if cars_match: cars.extend(int(n) for n in re.findall(r'(\d+)\s*\([^)]+\)', cars_match.group(1))) else: cars_match = re.search(r'(?i)\bCARS\s+((?:\d+\s*\([^)]+\)(?:\s*,\s*|\s+AND\s+)?)+)', message) if cars_match and len(cars_match.group(1)) < 200: cars.extend(int(n) for n in re.findall(r'(\d+)\s*\([^)]+\)', cars_match.group(1))) unique_cars = list(dict.fromkeys(cars)) return unique_cars if unique_cars else None def _category_scope_exprs(columns): """Vectorised ``parse_category_scope``: returns (Category, Scope, Status, info) expressions.""" message = pl.col("Message").str.to_uppercase() category, scope, flag = pl.col("Category"), pl.col("Scope"), pl.col("Flag") mode = pl.col("Mode") if "Mode" in columns else pl.lit(None, pl.String) status = pl.col("Status").cast(pl.String) if "Status" in columns else pl.lit(None, pl.String) null = pl.lit(None, pl.String) def last_after(expr, sep): return expr.str.split(sep).list.last().str.strip_chars() # Flag def clean_driver_flag(flag_name): return ( message.str.replace_all(flag_name, "", literal=True) .str.replace_all("WAVED", "", literal=True) .str.replace_all("FLAG", "", literal=True) .str.replace_all("FOR", "", literal=True) .str.strip_chars() ) flag_info = ( pl.when((scope == "Driver") & (flag == "BLUE")).then(clean_driver_flag("BLUE").str.split(" ").list.last()) .when((scope == "Driver") & (flag == "BLACK AND WHITE")).then(last_after(clean_driver_flag("BLACK AND WHITE"), "-")) .when(scope == "Sector").then(pl.lit("SECTOR ") + last_after(message, "SECTOR")) .when(scope == "Track").then(last_after(message, "-")) .otherwise(null) ) # Other: first matching FIA rule wins rules = [(cat, sc, kws) for cat, scopes in FIA_CATEGORY_SCOPE_RULES.items() for sc, kws in scopes.items()] penalty_type = null for pt in reversed(penalty_types): penalty_type = pl.when(message.str.contains(pt, literal=True)).then(pl.lit(pt)).otherwise(penalty_type) penalty_value = pl.when(penalty_type == "TIME PENALTY").then(message.str.extract(r'(\d{1,2})\s*SECOND', 1)).otherwise(null) penalty_status = pl.when(message.str.contains("PENALTY SERVED", literal=True)).then(pl.lit("Served")).otherwise(null) dash_info = pl.when(message.str.contains("-", literal=True)).then(last_after(message, "-")).otherwise(null) # No FIA rule matched the message other = {"Category": pl.lit("Other"), "Scope": pl.lit("Unclassified"), "Status": null, "info": null} for cat, sc, kws in reversed(rules): matched = message.str.contains_any(kws) values = { "Category": pl.lit(cat), "Scope": pl.lit(sc), "Status": penalty_status if cat == "Penalty" else status, "info": penalty_value if cat == "Penalty" else dash_info, } other = {k: pl.when(matched).then(values[k]).otherwise(other[k]) for k in other} branches = { "Category": (category, category, category, other["Category"], pl.lit("Other")), "Scope": (scope, last_after(message, " "), mode, other["Scope"], pl.lit("Unclassified")), "Status": (status, status, status, other["Status"], null), "info": (flag_info, null, status, other["info"], null), } exprs = [] for name, (flag_v, drs_v, sc_v, other_v, else_v) in branches.items(): exprs.append( pl.when(category == "Flag").then(flag_v) .when(category == "Drs").then(drs_v) .when(category == "SafetyCar").then(sc_v) .when(category == "Other").then(other_v) .otherwise(else_v) .alias(name) ) return exprs def generate_race_control_messages_table(session, rcm_df): rcm = _select(rcm_df) rcm = rcm.with_columns( pl.col("Message").map_elements(parse_cars_from_message, return_dtype=pl.List(pl.Int64), skip_nulls=True).alias("RacingNumber") ).with_columns(_category_scope_exprs(rcm.columns)) return rcm.select([c for c in silver_race_control_messages_col_order if c in rcm.columns])