"""Reference baselines that run on all 19 datasets without edits.
Most of the datasets ship ``object`` or ``bool`` feature columns.
A model that only selects numeric columns silently discards them; one that
hands them straight to an estimator raises. :func:`encode_features` is the
small piece of glue in between, and the two baselines below use it.
from pamir import evaluate, fleet_summary, gbdt_baseline
results = evaluate(gbdt_baseline, lag=1000)
fleet_summary(results)
Encoding against ``X_train`` **and** ``X_test`` together is legal under both
protocols: each hands the model the features of the rows it must score, and
withholds only their labels. No target statistic crosses the boundary.
"""
from typing import List, Tuple
import numpy as np
import pandas as pd
def _non_numeric_columns(frame: pd.DataFrame) -> List[str]:
# Anything an estimator cannot take as a float needs an ordinal code first.
# Test against numeric-ness rather than a fixed dtype list so that
# pyarrow-backed strings (pandas >= 3 loads parquet text as ``str``, not
# ``object``), categoricals and the like are all caught. ``bool`` is
# numeric and casts to float directly, so it is intentionally left alone.
return [c for c in frame.columns
if not pd.api.types.is_numeric_dtype(frame[c])]
[docs]
def encode_features(
X_train: pd.DataFrame,
X_test: pd.DataFrame,
) -> Tuple[pd.DataFrame, pd.DataFrame]:
"""Ordinal-encode non-numeric columns against one shared level set.
Both frames are returned as float, so a level present in only one of them
still maps to the same code in both. Numeric missing values are left as
``NaN`` — that is information, and filling it is the caller's decision.
Parameters
----------
X_train, X_test : DataFrame
Feature frames with identical columns, in identical order.
Returns
-------
(train, test) : tuple of DataFrame
New frames; the inputs are not modified.
Raises
------
ValueError
If the two frames do not carry the same columns in the same order.
"""
if list(X_train.columns) != list(X_test.columns):
raise ValueError(
"X_train and X_test must carry the same column names in the same "
f"order; got {list(X_train.columns)[:5]}... and "
f"{list(X_test.columns)[:5]}..."
)
train, test = X_train.copy(), X_test.copy()
for col in _non_numeric_columns(train):
# Use the nullable string dtype so the level set is pure ``str`` (no
# mixed float/NA that would make ``sorted`` raise on pandas >= 3), and
# take levels from the non-missing values only. Missing entries then
# map to code -1, distinct from every real level.
tr = train[col].astype("string")
te = test[col].astype("string")
levels = pd.Index(sorted(set(tr.dropna()) | set(te.dropna())))
train[col] = pd.Categorical(tr, categories=levels).codes
test[col] = pd.Categorical(te, categories=levels).codes
return train.astype("float64"), test.astype("float64")
def _constant_scores(y_train: np.ndarray, n_test: int) -> np.ndarray:
"""Base-rate scores, for a training split that holds a single class.
Not an error: with one class there is nothing to discriminate on, so the
honest answer is a constant. The protocol scores it as AUC 0.5.
"""
return np.full(n_test, float(np.mean(y_train)) if len(y_train) else 0.5)
[docs]
def logistic_baseline(
X_train: pd.DataFrame,
y_train: np.ndarray,
X_test: pd.DataFrame,
) -> np.ndarray:
"""Median-imputed, standardized logistic regression.
The cheap reference point: fast enough to run the whole fleet under the
default protocol parameters.
"""
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
y_train = np.asarray(y_train)
if len(np.unique(y_train)) < 2:
return _constant_scores(y_train, len(X_test))
train, test = encode_features(X_train, X_test)
model = make_pipeline(
SimpleImputer(strategy="median"),
StandardScaler(),
LogisticRegression(max_iter=1000),
)
model.fit(train, y_train)
return model.predict_proba(test)[:, 1]
[docs]
def gbdt_baseline(
X_train: pd.DataFrame,
y_train: np.ndarray,
X_test: pd.DataFrame,
max_iter: int = 150,
learning_rate: float = 0.1,
seed: int = 42,
) -> np.ndarray:
"""Histogram gradient boosting, handling missing values natively.
Uses scikit-learn rather than XGBoost so it needs no optional dependency,
and so it cannot hit the OpenMP clash documented in the installation notes.
"""
from sklearn.ensemble import HistGradientBoostingClassifier
y_train = np.asarray(y_train)
if len(np.unique(y_train)) < 2:
return _constant_scores(y_train, len(X_test))
train, test = encode_features(X_train, X_test)
# scikit-learn's histogram binning raises on a column with a single distinct
# value; drop constant columns (they carry no signal) before fitting.
keep = train.columns[train.nunique(dropna=True) >= 2]
if len(keep) == 0:
return _constant_scores(y_train, len(X_test))
train, test = train[keep], test[keep]
model = HistGradientBoostingClassifier(
max_iter=max_iter, learning_rate=learning_rate, random_state=seed)
model.fit(train, y_train)
return model.predict_proba(test)[:, 1]