"""Reference baselines that run on every PaMIR dataset without edits.
Most of the datasets ship ``object`` or ``bool`` feature columns. A model that
only selects numeric columns silently discards them; one that hands them
straight to an estimator raises. The baselines ordinal-encode non-numeric
columns against the levels seen in the **training** rows (an unseen level gets
its own code, ``-2``; a missing value ``-1``), so a row is scored without
reference to any other row being scored.
Each baseline comes in two forms with identical models:
* ``logistic_fit`` / ``gbdt_fit`` — ``fit_fn(X_train, y_train) -> scorer``, the
form the arrival-mode streaming protocol calls;
* ``logistic_baseline`` / ``gbdt_baseline`` — ``predict_fn(X_train, y_train,
X_test)``, for the i.i.d. protocol and the refresh mode.
from pamir import evaluate, fleet_summary, gbdt_fit
results = evaluate(gbdt_fit) # arrival mode, reference setting
fleet_summary(results)
The 0.3.0 baselines, which encoded against the levels of ``X_train`` and
``X_test`` together, are kept as ``logistic_baseline_v03`` / ``gbdt_baseline_v03``
to reproduce 0.3.0 numbers. :func:`encode_features` (the shared-level encoder)
is unchanged; it is legal in the i.i.d. protocol, where the model is handed the
rows it must score, but not in arrival mode, where it would let a row's code
depend on rows that arrived after it.
"""
from typing import Dict, List, Tuple
import numpy as np
import pandas as pd
def _non_numeric_columns(frame: pd.DataFrame) -> List[str]:
# Anything an estimator cannot take as a float needs an ordinal code first.
# Test against numeric-ness rather than a fixed dtype list so that
# pyarrow-backed strings (pandas >= 3 loads parquet text as ``str``, not
# ``object``), categoricals and the like are all caught. ``bool`` is
# numeric and casts to float directly, so it is intentionally left alone.
return [c for c in frame.columns
if not pd.api.types.is_numeric_dtype(frame[c])]
[docs]
def encode_features(
X_train: pd.DataFrame,
X_test: pd.DataFrame,
) -> Tuple[pd.DataFrame, pd.DataFrame]:
"""Ordinal-encode non-numeric columns against one shared level set.
Both frames are returned as float, so a level present in only one of them
still maps to the same code in both. Numeric missing values are left as
``NaN`` — that is information, and filling it is the caller's decision.
Parameters
----------
X_train, X_test : DataFrame
Feature frames with identical columns, in identical order.
Returns
-------
(train, test) : tuple of DataFrame
New frames; the inputs are not modified.
Raises
------
ValueError
If the two frames do not carry the same columns in the same order.
"""
if list(X_train.columns) != list(X_test.columns):
raise ValueError(
"X_train and X_test must carry the same column names in the same "
f"order; got {list(X_train.columns)[:5]}... and "
f"{list(X_test.columns)[:5]}..."
)
train, test = X_train.copy(), X_test.copy()
for col in _non_numeric_columns(train):
# Use the nullable string dtype so the level set is pure ``str`` (no
# mixed float/NA that would make ``sorted`` raise on pandas >= 3), and
# take levels from the non-missing values only. Missing entries then
# map to code -1, distinct from every real level.
tr = train[col].astype("string")
te = test[col].astype("string")
levels = pd.Index(sorted(set(tr.dropna()) | set(te.dropna())))
train[col] = pd.Categorical(tr, categories=levels).codes
test[col] = pd.Categorical(te, categories=levels).codes
return train.astype("float64"), test.astype("float64")
def _constant_scores(y_train: np.ndarray, n_test: int) -> np.ndarray:
"""Base-rate scores, for a training split that holds a single class.
Not an error: with one class there is nothing to discriminate on, so the
honest answer is a constant. The protocol scores it as AUC 0.5.
"""
return np.full(n_test, float(np.mean(y_train)) if len(y_train) else 0.5)
[docs]
def logistic_baseline_v03(
X_train: pd.DataFrame,
y_train: np.ndarray,
X_test: pd.DataFrame,
) -> np.ndarray:
"""0.3.0 logistic baseline (levels shared between X_train and X_test)."""
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
y_train = np.asarray(y_train)
if len(np.unique(y_train)) < 2:
return _constant_scores(y_train, len(X_test))
train, test = encode_features(X_train, X_test)
model = make_pipeline(
SimpleImputer(strategy="median"),
StandardScaler(),
LogisticRegression(max_iter=1000),
)
model.fit(train, y_train)
return model.predict_proba(test)[:, 1]
[docs]
def gbdt_baseline_v03(
X_train: pd.DataFrame,
y_train: np.ndarray,
X_test: pd.DataFrame,
max_iter: int = 150,
learning_rate: float = 0.1,
seed: int = 42,
) -> np.ndarray:
"""0.3.0 gradient-boosting baseline (levels shared between X_train and X_test)."""
from sklearn.ensemble import HistGradientBoostingClassifier
y_train = np.asarray(y_train)
if len(np.unique(y_train)) < 2:
return _constant_scores(y_train, len(X_test))
train, test = encode_features(X_train, X_test)
# scikit-learn's histogram binning raises on a column with a single distinct
# value; drop constant columns (they carry no signal) before fitting.
keep = train.columns[train.nunique(dropna=True) >= 2]
if len(keep) == 0:
return _constant_scores(y_train, len(X_test))
train, test = train[keep], test[keep]
model = HistGradientBoostingClassifier(
max_iter=max_iter, learning_rate=learning_rate, random_state=seed)
model.fit(train, y_train)
return model.predict_proba(test)[:, 1]
# ---------------------------------------------------------------------------
# 0.4.0 baselines: encoder fitted on the training rows only
# ---------------------------------------------------------------------------
[docs]
def fit_encoder(X_train: pd.DataFrame) -> Dict[str, pd.Index]:
"""Level sets of the non-numeric columns, taken from the training rows."""
return {col: pd.Index(sorted(set(X_train[col].astype("string").dropna())))
for col in _non_numeric_columns(X_train)}
[docs]
def apply_encoder(levels: Dict[str, pd.Index], X: pd.DataFrame) -> pd.DataFrame:
"""Ordinal codes for the encoded columns: -1 missing, -2 unseen level."""
out = X.copy()
for col, cats in levels.items():
s = X[col].astype("string")
codes = pd.Categorical(s, categories=cats).codes.astype("float64")
codes[(codes == -1) & s.notna().to_numpy()] = -2.0
out[col] = codes
return out.astype("float64")
def _constant_scorer(y_train: np.ndarray):
rate = float(np.mean(y_train)) if len(y_train) else 0.5
return lambda X: np.full(len(X), rate)
[docs]
def logistic_fit(X_train: pd.DataFrame, y_train: np.ndarray):
"""Median-imputed, standardized logistic regression; returns a scorer.
The cheap reference point: fast enough to run the whole fleet under the
default protocol parameters.
"""
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
y_train = np.asarray(y_train)
if len(np.unique(y_train)) < 2:
return _constant_scorer(y_train)
columns = list(X_train.columns)
levels = fit_encoder(X_train)
model = make_pipeline(
SimpleImputer(strategy="median"),
StandardScaler(),
LogisticRegression(max_iter=1000),
)
model.fit(apply_encoder(levels, X_train), y_train)
def scorer(X: pd.DataFrame) -> np.ndarray:
return model.predict_proba(apply_encoder(levels, X[columns]))[:, 1]
return scorer
[docs]
def gbdt_fit(X_train: pd.DataFrame, y_train: np.ndarray, max_iter: int = 150,
learning_rate: float = 0.1, seed: int = 42):
"""Histogram gradient boosting with native missing-value handling; returns a scorer.
Uses scikit-learn rather than XGBoost so it needs no optional dependency,
and so it cannot hit the OpenMP clash documented in the installation notes.
"""
from sklearn.ensemble import HistGradientBoostingClassifier
y_train = np.asarray(y_train)
if len(np.unique(y_train)) < 2:
return _constant_scorer(y_train)
levels = fit_encoder(X_train)
train = apply_encoder(levels, X_train)
# scikit-learn's histogram binning raises on a column with a single distinct
# value; drop constant columns (they carry no signal) before fitting.
keep = list(train.columns[train.nunique(dropna=True) >= 2])
if not keep:
return _constant_scorer(y_train)
model = HistGradientBoostingClassifier(
max_iter=max_iter, learning_rate=learning_rate, random_state=seed)
model.fit(train[keep], y_train)
columns = list(X_train.columns)
def scorer(X: pd.DataFrame) -> np.ndarray:
return model.predict_proba(apply_encoder(levels, X[columns])[keep])[:, 1]
return scorer
[docs]
def logistic_baseline(X_train: pd.DataFrame, y_train: np.ndarray,
X_test: pd.DataFrame) -> np.ndarray:
""":func:`logistic_fit` as a ``predict_fn``."""
return logistic_fit(X_train, y_train)(X_test)
[docs]
def gbdt_baseline(X_train: pd.DataFrame, y_train: np.ndarray, X_test: pd.DataFrame,
max_iter: int = 150, learning_rate: float = 0.1,
seed: int = 42) -> np.ndarray:
""":func:`gbdt_fit` as a ``predict_fn``."""
return gbdt_fit(X_train, y_train, max_iter=max_iter,
learning_rate=learning_rate, seed=seed)(X_test)