Source code for pamir.loader
"""Load a PaMIR dataset as (X, y, meta).
PaMIR does not ship the datasets; the harmonized parquet lives in a local
cache and is fetched from the dataset's original source on first use (see
:mod:`pamir.download`).
"""
from typing import Dict, Optional, Tuple
import numpy as np
import pandas as pd
from pamir.catalog import dataset_info
[docs]
def load_dataset(
dataset_id: str,
max_rows: Optional[int] = None,
auto_download: bool = True,
strict: bool = True,
) -> Tuple[pd.DataFrame, np.ndarray, Dict]:
"""Load a single dataset from the local cache.
Parameters
----------
dataset_id : str
One of the PaMIR dataset ids (see ``pamir.list_datasets()``).
max_rows : int, optional
Truncate to at most this many rows (in stream order).
auto_download : bool
If the dataset is not cached, fetch and harmonize it from source
(default True). Set False to require an explicit ``pamir.download()``.
strict : bool
The cached table is checked against its contract in ``expected.json``
on every load (a cache built by an older release, or from a drifted
source snapshot, is caught here). If True (default), a mismatch raises
:class:`pamir.contract.ContractError`; if False, the table is returned
with ``meta["contract_ok"] = False``.
Returns
-------
X : DataFrame
Feature columns exactly as stored: ``int64``, ``float64``, ``bool`` and
``object`` all occur, and most of the datasets carry at least one
non-numeric column. Nothing is encoded or imputed for you — see
:func:`pamir.encode_features` for the glue most models need.
y : ndarray of int
Binary target (1 = default, 0 = non-default).
meta : dict
Dataset metadata from the catalog, plus ``n_rows``, ``n_defaults``,
``n_features`` (after truncation), ``contract_ok``,
``contract_issues`` and ``snapshot_verified`` (True when the cache
records the raw-file digest and it equals the pinned one). The
catalog's ``notes`` key is present only on the datasets that required
harmonization, so read it with ``.get()``.
"""
from pamir.contract import EXPECTED, ContractError, check_table, read_record
from pamir.download import cache_dir, download
info = dataset_info(dataset_id)
pq = cache_dir() / f"{dataset_id}.parquet"
if not pq.exists():
if not auto_download:
raise FileNotFoundError(
f"'{dataset_id}' is not cached at {pq}. "
f"Run `pamir.download('{dataset_id}')` (needs pamir-credit[data]; "
f"Kaggle sources also need Kaggle credentials)."
)
pq = download(dataset_id, strict=strict)
df = pd.read_parquet(pq)
if "__target__" not in df.columns:
raise ValueError(f"No __target__ column in {pq}")
record = read_record(pq)
issues = check_table(dataset_id, df, raw_sha256=record.get("raw_sha256"))
if issues and strict:
raise ContractError(
dataset_id, issues,
hint=(f"The cached table at {pq} is not the benchmark table. Rebuild it "
f"with pamir.download('{dataset_id}', force=True), or pass "
"strict=False to load it anyway (flagged contract_ok=False)."))
pinned_raw = EXPECTED.get(dataset_id, {}).get("raw_sha256")
y = df["__target__"].to_numpy().astype(int)
X = df.drop(columns=["__target__"])
if max_rows is not None:
X = X.iloc[:max_rows]
y = y[:max_rows]
meta = dict(info)
meta["n_rows"] = len(y)
meta["n_defaults"] = int(y.sum())
meta["n_features"] = X.shape[1]
meta["contract_ok"] = not issues
meta["contract_issues"] = issues
meta["snapshot_verified"] = bool(pinned_raw) and record.get("raw_sha256") == pinned_raw
return X, y, meta