Source code for pamir.summary

"""Fleet-level summary of a results frame, with coverage as a first-class field.

The headline number of a benchmark run is only defined when the model scored
every dataset it was given.  ``fleet_summary`` therefore reports ``auc_mean``
only for a complete run, and puts the partial average in a separately named
field so it cannot be mistaken for one.
"""

from typing import Dict, List, Optional

import pandas as pd

# The AUC column is named by the protocol that produced the frame.
AUC_COLUMNS = ("auc_final", "auc_mean")


def _auc_column(results: pd.DataFrame) -> str:
    for col in AUC_COLUMNS:
        if col in results.columns:
            return col
    raise ValueError(
        f"no auc column found: expected one of {AUC_COLUMNS}, "
        f"got {list(results.columns)}. Pass a frame returned by "
        "pamir.evaluate or pamir.evaluate_iid."
    )


[docs] def fleet_summary(results: pd.DataFrame) -> Dict[str, Optional[float]]: """Summarize a fleet run, withholding the mean when coverage is incomplete. Parameters ---------- results : DataFrame As returned by :func:`pamir.evaluate` or :func:`pamir.evaluate_iid`. Returns ------- dict with keys: ``n_datasets``, ``n_scored``, ``coverage``, ``complete`` How much of the fleet the model actually scored. ``auc_mean``, ``gini_mean`` The headline numbers — ``None`` unless every dataset was scored and, when the frame carries ``contract_ok``, every table matched its data contract. ``auc_mean_scored_only`` The average over the datasets that did score. Diagnostic only: it is an average over a subset the model chose by failing, so it is not comparable across models. ``n_failures``, ``failed_datasets`` What went wrong and where. ``contract_ok``, ``contract_failed_datasets`` Whether every table was the benchmark table (see :mod:`pamir.contract`). ``row_independent``, ``row_dependent_datasets`` Arrival mode: whether every scorer scored rows independently of the other rows in its batch. ``auc_by_budget`` Arrival mode: per label-budget bin, the mean AUC over the datasets that reach it and their number (``{}`` for other protocols). """ auc_col = _auc_column(results) scored = results[auc_col].notna() n_datasets = len(results) n_scored = int(scored.sum()) complete = n_datasets > 0 and n_scored == n_datasets partial_mean = float(results.loc[scored, auc_col].mean()) if n_scored else None has_names = "dataset" in results.columns failed: List[str] = results.loc[~scored, "dataset"].tolist() if has_names else [] contract_ok, contract_failed = True, [] if "contract_ok" in results.columns: ok = results["contract_ok"].fillna(False).astype(bool) contract_ok = bool(ok.all()) contract_failed = results.loc[~ok, "dataset"].tolist() if has_names else [] # Arrival mode records whether the scorer scored rows independently; refresh # and i.i.d. rows carry None (not applicable), which does not block. row_independent, row_dependent = True, [] if "row_independent" in results.columns: dependent = results["row_independent"].map( lambda v: v is not None and not pd.isna(v) and not bool(v)).astype(bool) row_independent = not bool(dependent.any()) row_dependent = results.loc[dependent, "dataset"].tolist() if has_names else [] headline = complete and contract_ok and row_independent # Label-budget curve (arrival mode): per budget bin, the mean AUC over the # datasets that reach that bin, with their count. Bins differ in which # datasets they contain, so the curve is descriptive, not a headline. budget = {} for col in [c for c in results.columns if c.startswith("auc_budget_")]: vals = pd.to_numeric(results[col], errors="coerce").dropna() budget[col] = {"mean": float(vals.mean()) if len(vals) else None, "n_datasets": int(len(vals))} return { "n_datasets": n_datasets, "n_scored": n_scored, "coverage": n_scored / n_datasets if n_datasets else 0.0, "complete": complete, "auc_mean": partial_mean if headline else None, "gini_mean": 2 * partial_mean - 1 if headline and partial_mean else None, "auc_mean_scored_only": partial_mean, "n_failures": (int(results["n_failures"].sum()) if "n_failures" in results.columns else None), "failed_datasets": failed, "contract_ok": contract_ok, "contract_failed_datasets": contract_failed, "row_independent": row_independent, "row_dependent_datasets": row_dependent, "auc_by_budget": budget, }
def format_fleet_summary(summary: Dict) -> str: """One or two lines of human-readable summary, for ``verbose=True``.""" if summary["complete"] and summary["auc_mean"] is not None: return (f"\n Fleet mean AUC: {summary['auc_mean']:.4f} " f"Gini: {summary['gini_mean']:.3f} " f"({summary['n_datasets']} datasets, all scored)") if summary["complete"] and not summary.get("contract_ok", True): names = ", ".join(summary.get("contract_failed_datasets") or []) or "—" return (f"\n NO FLEET MEAN: every dataset was scored, but these tables do " f"not match their data contract: {names}.") if summary["complete"]: names = ", ".join(summary.get("row_dependent_datasets") or []) or "—" return (f"\n NO FLEET MEAN: every dataset was scored, but the scorer was " f"not row-independent on: {names}.") failed = ", ".join(summary["failed_datasets"]) or "—" partial = summary["auc_mean_scored_only"] partial_text = f"{partial:.4f}" if partial is not None else "n/a" return ( f"\n NO FLEET MEAN: {summary['n_scored']} of {summary['n_datasets']} " f"datasets scored (failed: {failed})." f"\n Mean over the scored subset only: {partial_text} — not comparable " "with a model that scored the whole fleet." )