Source code for pamir.evaluate

"""The streaming evaluation protocol.

Simulates a lender scoring an arriving stream of applications whose outcomes
are revealed only after a maturation delay of ``lag`` stream positions.  Rows
are replayed in their stored order, a fixed random permutation of the source
(the sources carry no usable origination dates), so the stream has no calendar
drift.  See :mod:`pamir.evaluate_iid` for the conventional train/test split.

Two modes
---------
``mode="arrival"`` (default, the reference protocol since 0.4.0)
    The model is a ``fit_fn(X_train, y_train) -> scorer``.  A refit trains on
    the labels resolved at that step; the resulting ``scorer(X_rows)`` scores
    every row that arrives after the refit and before the next one, and a
    score, once given, is final.  The model therefore never sees a label that
    had not matured when a row arrived, nor the features of a row that had not
    arrived, and the delay ``lag`` binds for every row.  Scorers are called on
    batches of already-arrived rows for speed; a probe re-scores a random subset
    of each batch on its own, and a scorer whose scores depend on the other rows
    of the batch is flagged ``row_independent = False`` (the run then has no
    fleet mean).  ``batch_scoring=False`` calls the scorer one row at a time.
    A plain ``predict_fn(X_train, y_train, X_test)`` is accepted and wrapped
    (one ``predict_fn`` call per scoring batch).

``mode="refresh"`` (the 0.3.0 protocol, kept for reproducibility)
    The model is a ``predict_fn``.  Every refit re-scores all rows whose label
    is unresolved, *including rows that have not yet arrived*, whose features
    the model therefore sees in advance; a row is evaluated on the last score
    committed before its label resolves.  0.3.0 numbers are reproduced with
    ``mode="refresh", max_lag_frac=None`` and the ``*_v03`` baselines.

Delay cap
---------
A delay as long as the stream leaves no label to learn from: at ``lag=1000``
the 1,000-row datasets would never be scored.  The effective delay is
``min(lag, floor(max_lag_frac * n))`` with ``max_lag_frac=0.2`` by default, and
is reported per dataset as ``lag_effective``.

Every call into user code is counted and failures are kept — see
:mod:`pamir.failures`.
"""

import inspect
import warnings
from typing import Callable, Dict, List, Optional, Sequence

import numpy as np
import pandas as pd
from sklearn.metrics import roc_auc_score

from pamir.catalog import list_datasets
from pamir.failures import FailureLog
from pamir.loader import load_dataset
from pamir.summary import fleet_summary, format_fleet_summary

MODES = ("arrival", "refresh")

# Columns of the per-dataset frame returned by `evaluate`.  `refit_points` and
# the per-row arrays are deliberately absent: they are lists per row, which
# block `to_csv` and every other tabular export.  Read them from `evaluate_one`.
RESULT_COLUMNS = ["dataset", "mode", "n_rows", "n_defaults", "DR", "lag_effective",
                  "k_refit", "n_refits", "n_refits_ok", "n_calls", "n_failures",
                  "n_scored", "contract_ok", "row_independent", "auc_final"]

# Label budgets: rows are grouped by the number of resolved labels their scorer
# was trained on, [0, 100), [100, 300), ..., [10000, inf).
BUDGET_EDGES = (0, 100, 300, 1000, 3000, 10000)
BUDGET_COLUMNS = [f"auc_budget_{lo}" for lo in BUDGET_EDGES]

# Minimum scored rows and defaults before an AUC is meaningful.
_MIN_SCORED_ROWS = 10
_MIN_SCORED_DEFAULTS = 3

# Row-independence probe: rows re-scored per batch, and the tolerance.
_PROBE_ROWS = 64
_PROBE_RTOL, _PROBE_ATOL = 1e-6, 1e-9


def effective_lag(lag: int, n: int, max_lag_frac: Optional[float]) -> int:
    """The delay actually applied to a stream of ``n`` rows."""
    if max_lag_frac is None:
        return int(lag)
    return int(max(1, min(int(lag), int(np.floor(max_lag_frac * n)))))


[docs] def from_predict_fn(predict_fn: Callable) -> Callable: """Turn ``predict_fn(X_train, y_train, X_test)`` into a ``fit_fn``. The returned ``fit_fn`` stores the training data; its scorer calls ``predict_fn`` once per scoring batch, so a model is refitted for every batch it scores. Correct, but slower than a native ``fit_fn``. """ def fit_fn(X_train, y_train): def scorer(X_rows): return predict_fn(X_train, y_train, X_rows) return scorer fit_fn._pamir_wrapped = True fit_fn.__name__ = f"from_predict_fn({getattr(predict_fn, '__name__', 'model')})" return fit_fn
def _n_required_positional(fn: Callable) -> Optional[int]: try: params = inspect.signature(fn).parameters.values() except (TypeError, ValueError): return None kinds = (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) return sum(1 for p in params if p.kind in kinds and p.default is inspect.Parameter.empty) def _as_fit_fn(model: Callable) -> Callable: """Accept a fit_fn as is; wrap a three-argument predict_fn.""" if getattr(model, "_pamir_wrapped", False): return model if _n_required_positional(model) == 3: return from_predict_fn(model) return model def _refit_due(cum_defaults: int, since_refit: int, n_attempts: int, min_defaults: int, k_refit: int) -> bool: """First refit at ``min_defaults`` resolved defaults, then every ``k_refit``.""" if cum_defaults == min_defaults and n_attempts == 0: return True return cum_defaults >= min_defaults and since_refit >= k_refit def _safe_auc(y: np.ndarray, preds: np.ndarray) -> Optional[float]: """ROC AUC over scored rows, or None when too few to be meaningful.""" valid = ~np.isnan(preds) n_pos = y[valid].sum() if (valid.sum() <= _MIN_SCORED_ROWS or n_pos < _MIN_SCORED_DEFAULTS or n_pos == valid.sum()): return None return float(roc_auc_score(y[valid], preds[valid])) def _refit_point(t: int, resolved_end: int, cum_defaults: int, y: np.ndarray, preds: np.ndarray, ok: bool) -> Dict: """Record cumulative AUC over everything resolved (and scored) so far.""" point = {"pos": t + 1, "resolved_n": resolved_end, "cum_defaults": cum_defaults, "ok": bool(ok)} auc = _safe_auc(y[:resolved_end], preds[:resolved_end]) if auc is not None: point["cum_auc"] = auc return point def auc_by_budget(y: np.ndarray, preds: np.ndarray, train_n: np.ndarray, edges: Sequence[int] = BUDGET_EDGES) -> Dict[str, Optional[float]]: """AUC of the rows scored by models trained on a given number of labels. ``train_n[i]`` is the number of resolved labels behind row ``i``'s score (``-1`` for unscored rows). Bins with too few rows or defaults are None. """ out: Dict[str, Optional[float]] = {} uppers = list(edges[1:]) + [np.inf] for lo, hi in zip(edges, uppers): mask = (train_n >= lo) & (train_n < hi) out[f"auc_budget_{lo}"] = _safe_auc(y[mask], preds[mask]) if mask.any() else None return out # --------------------------------------------------------------------------- # refresh mode (0.3.0) # --------------------------------------------------------------------------- def _run_refresh(predict_fn, X, y, lag, k_refit, min_defaults, log): n = len(y) preds = np.full(n, np.nan, dtype=np.float64) cum_defaults = 0 since_refit = 0 refit_points: List[Dict] = [] def score_and_commit(X_ctx, y_ctx, X_rem, start): """Call the model and commit its scores, validating the contract.""" scores = np.asarray(predict_fn(X_ctx, y_ctx, X_rem), dtype=np.float64) if scores.shape != (len(X_rem),): raise ValueError( f"predict_fn returned scores of shape {scores.shape} for " f"{len(X_rem)} rows; expected ({len(X_rem)},).") preds[start:] = scores return True for t in range(n): resolved_end = max(0, t + 1 - lag) if t >= lag and y[t - lag] == 1: cum_defaults += 1 since_refit += 1 if not (_refit_due(cum_defaults, since_refit, len(refit_points), min_defaults, k_refit) and resolved_end > 0 and t + 1 < n): continue if resolved_end - cum_defaults < min_defaults: continue ok = log.call(score_and_commit, X.iloc[:resolved_end], y[:resolved_end], X.iloc[resolved_end:], resolved_end) is not None since_refit = 0 refit_points.append(_refit_point(t, resolved_end, cum_defaults, y, preds, ok)) return preds, refit_points, None, None # --------------------------------------------------------------------------- # arrival mode # --------------------------------------------------------------------------- def _run_arrival(fit_fn, X, y, lag, k_refit, min_defaults, log, batch_scoring, probe, seed=0): n = len(y) wrapped = getattr(fit_fn, "_pamir_wrapped", False) preds = np.full(n, np.nan, dtype=np.float64) train_n = np.full(n, -1, dtype=np.int64) rng = np.random.RandomState(seed) state = {"scorer": None, "scorer_n": 0, "pending": None, "row_independent": True} cum_defaults = 0 since_refit = 0 refit_points: List[Dict] = [] def fit(X_ctx, y_ctx): scorer = fit_fn(X_ctx, y_ctx) if not callable(scorer): raise TypeError( "in arrival mode the model must be fit_fn(X_train, y_train) returning " f"a callable scorer(X_rows); got {type(scorer).__name__}. A " "three-argument predict_fn is accepted too (see from_predict_fn).") return scorer def score(start, stop): X_rows = X.iloc[start:stop] scores = np.asarray(state["scorer"](X_rows), dtype=np.float64) if scores.shape != (len(X_rows),): raise ValueError( f"scorer returned scores of shape {scores.shape} for " f"{len(X_rows)} rows; expected ({len(X_rows)},).") if probe and len(X_rows) >= 2: k = min(_PROBE_ROWS, max(1, len(X_rows) // 2)) sub = np.sort(rng.choice(len(X_rows), size=k, replace=False)) again = np.asarray(state["scorer"](X_rows.iloc[sub]), dtype=np.float64) if again.shape != (k,) or not np.allclose( again, scores[sub], rtol=_PROBE_RTOL, atol=_PROBE_ATOL, equal_nan=True): state["row_independent"] = False preds[start:stop] = scores train_n[start:stop] = state["scorer_n"] return True def flush(stop): """Score rows [pending, stop) with the scorer they arrived under.""" start = state["pending"] if state["scorer"] is None or start is None or stop <= start: return if batch_scoring: log.call(score, start, stop) else: for i in range(start, stop): log.call(score, i, i + 1) n_attempts = 0 for t in range(n): # Within step t: row t arrives (scored by the current scorer); then the # label of row t - lag resolves; then a refit may fire, and the new # scorer applies from row t + 1 on. resolved_end = max(0, t + 1 - lag) if t >= lag and y[t - lag] == 1: cum_defaults += 1 since_refit += 1 if not (_refit_due(cum_defaults, since_refit, n_attempts, min_defaults, k_refit) and resolved_end > 0 and t + 1 < n): continue if resolved_end - cum_defaults < min_defaults: continue flush(t + 1) n_attempts += 1 if wrapped: # nothing of the user's runs at fit time; scoring calls are counted new = fit(X.iloc[:resolved_end], y[:resolved_end]) else: new = log.call(fit, X.iloc[:resolved_end], y[:resolved_end]) since_refit = 0 if new is not None: state["scorer"], state["scorer_n"] = new, resolved_end state["pending"] = t + 1 refit_points.append(_refit_point(t, resolved_end, cum_defaults, y, preds, new is not None)) flush(n) return preds, refit_points, train_n, state["row_independent"] # --------------------------------------------------------------------------- # public API # ---------------------------------------------------------------------------
[docs] def evaluate_one( model: Callable, dataset_id: str, lag: int = 1000, k_refit: int = 10, max_n: Optional[int] = 20000, min_defaults: int = 3, on_error: str = "warn", mode: str = "arrival", max_lag_frac: Optional[float] = 0.2, batch_scoring: bool = True, probe: bool = True, strict: bool = True, ) -> Dict: """Run the streaming protocol on one dataset. Parameters ---------- model : callable ``mode="arrival"``: ``fit_fn(X_train, y_train) -> scorer`` with ``scorer(X_rows) -> np.ndarray`` (higher = more likely to default); a three-argument ``predict_fn`` is wrapped with :func:`from_predict_fn`. ``mode="refresh"``: ``predict_fn(X_train, y_train, X_test)``, where ``X_test`` is the remaining stream. dataset_id : str PaMIR dataset id. lag : int Label-maturation delay: the outcome of the row at position *t* is revealed at position *t + lag*. Capped by ``max_lag_frac``. k_refit : int Refit after every *k_refit* newly resolved defaults. This changes the score as well as the cost, so report it. max_n : int or None Truncate the stream to at most this many rows. min_defaults : int Resolved defaults (and non-defaults) required before the first refit. on_error : {"warn", "raise", "ignore"} What to do when the model raises. See :mod:`pamir.failures`. mode : {"arrival", "refresh"} See the module docstring. max_lag_frac : float or None Cap on the delay as a fraction of the stream length; None disables it. batch_scoring, probe : bool Arrival mode only: score arrived rows in batches, and probe each batch for row independence (see the module docstring). strict : bool Refuse a table that does not match its data contract (default True). Returns ------- dict with the keys of ``RESULT_COLUMNS``, plus ``lag``, ``errors``, ``refit_points`` (list of dicts with pos, resolved_n, cum_defaults, ok, cum_auc), and in arrival mode the ``auc_budget_*`` keys and ``train_n`` (per-row label budget, -1 where unscored). """ if mode not in MODES: raise ValueError(f"mode must be one of {MODES}, got {mode!r}.") X, y, meta = load_dataset(dataset_id, max_rows=max_n, strict=strict) n = len(y) lag_eff = effective_lag(lag, n, max_lag_frac) log = FailureLog(on_error, context=dataset_id) if mode == "refresh": preds, refit_points, train_n, row_independent = _run_refresh( model, X, y, lag_eff, k_refit, min_defaults, log) else: preds, refit_points, train_n, row_independent = _run_arrival( _as_fit_fn(model), X, y, lag_eff, k_refit, min_defaults, log, batch_scoring=batch_scoring, probe=probe) log.warn_if_failed() result = { "dataset": dataset_id, "mode": mode, "n_rows": n, "n_defaults": int(y.sum()), "DR": float(y.mean()), "lag": int(lag), "lag_effective": lag_eff, "k_refit": int(k_refit), "n_refits": len(refit_points), "n_refits_ok": sum(1 for p in refit_points if p["ok"]), **log.as_dict(), "n_scored": int(np.isfinite(preds).sum()), "contract_ok": meta["contract_ok"], "row_independent": row_independent, "auc_final": _safe_auc(y, preds), "refit_points": refit_points, } if train_n is not None: result.update(auc_by_budget(y, preds, train_n)) result["train_n"] = train_n.tolist() return result
[docs] def evaluate( model: Callable, datasets: Optional[Sequence[str]] = None, lag: int = 1000, k_refit: int = 10, max_n: Optional[int] = 20000, min_defaults: int = 3, verbose: bool = True, on_error: str = "warn", mode: str = "arrival", max_lag_frac: Optional[float] = 0.2, batch_scoring: bool = True, probe: bool = True, strict: bool = True, ) -> pd.DataFrame: """Run the streaming protocol across the PaMIR fleet. Parameters are those of :func:`evaluate_one`; ``datasets`` is a subset of dataset ids (default: every dataset in the catalogue). Returns ------- DataFrame with one row per dataset: the columns in ``RESULT_COLUMNS`` and, in arrival mode, ``BUDGET_COLUMNS``. Pass it to :func:`pamir.fleet_summary` for the headline numbers — a mean taken by hand over this frame silently excludes the datasets the model failed on. """ ds_ids = list(datasets) if datasets else list_datasets() rows = [] for ds in ds_ids: if verbose: print(f" {ds}...", end=" ", flush=True) result = evaluate_one(model, ds, lag=lag, k_refit=k_refit, max_n=max_n, min_defaults=min_defaults, on_error=on_error, mode=mode, max_lag_frac=max_lag_frac, batch_scoring=batch_scoring, probe=probe, strict=strict) if verbose: print(_format_dataset_line(result)) rows.append(result) columns = RESULT_COLUMNS + (BUDGET_COLUMNS if mode == "arrival" else []) df = pd.DataFrame(rows)[columns] summary = fleet_summary(df) if verbose and len(rows) > 1: print(format_fleet_summary(summary)) if on_error == "warn": _warn_no_headline(summary) return df
def _warn_no_headline(summary: Dict) -> None: if not summary["complete"]: warnings.warn( f"the model did not score {summary['n_datasets'] - summary['n_scored']} " f"of {summary['n_datasets']} datasets " f"({', '.join(summary['failed_datasets'])}). There is no fleet mean " "for a partial run — see pamir.fleet_summary.", UserWarning, stacklevel=3) elif not summary["contract_ok"]: warnings.warn( "every dataset was scored, but these tables do not match their data " f"contract: {', '.join(summary['contract_failed_datasets'])}. There is " "no fleet mean for a run on non-benchmark tables — see pamir.contract.", UserWarning, stacklevel=3) elif not summary.get("row_independent", True): warnings.warn( "the scorer's output depended on the other rows of a scoring batch on " f"{', '.join(summary['row_dependent_datasets'])}; in arrival mode that " "lets a row's score use rows that arrived after it. There is no fleet " "mean — fix the scorer, or pass batch_scoring=False.", UserWarning, stacklevel=3) def _format_dataset_line(result: Dict) -> str: auc = result["auc_final"] head = f"AUC={auc:.4f}" if auc is not None else "NO AUC" tail = f"({result['n_refits']} refits, lag {result['lag_effective']}" if result["n_failures"]: tail += f", {result['n_failures']}/{result['n_calls']} calls FAILED" if result.get("row_independent") is False: tail += ", ROW-DEPENDENT SCORER" if result.get("contract_ok") is False: tail += ", CONTRACT MISMATCH" return f"{head} {tail})"