Source code for pamir.download

"""Fetch a dataset from its original source and harmonize it locally.

PaMIR ships code and a recipe, not data.  ``download(id)`` fetches the raw
file from the snapshot of the dataset's source that the catalog pins, into a
local cache; harmonizes it (see :mod:`pamir.harmonize`); checks the result
against the data contract in ``expected.json`` (see :mod:`pamir.contract`);
and only then writes the parquet.  A table that does not match the contract
raises :class:`pamir.contract.ContractError` and is not cached, unless the
caller passes ``strict=False``.  ``load_dataset`` calls this automatically on
first use.

Sources and their requirements:

- ``kaggle``     — needs ``kagglehub``; kagglehub 1.0 fetches public datasets
  anonymously, older versions need Kaggle credentials (``KAGGLE_USERNAME`` +
  ``KAGGLE_KEY``, or ``~/.kaggle/kaggle.json``).  The catalog pins a dataset
  version.
- ``uci_zip``    — public download, no credentials.
- ``github_raw`` / ``github_zip`` — public download, no credentials.
- ``hf``         — needs ``huggingface_hub``; the catalog pins a revision where
  one is recorded.

Install the optional fetch dependencies with ``pip install pamir-credit[data]``.
"""

import io
import os
import warnings
import zipfile
from pathlib import Path
from typing import Dict, List, Optional
from urllib.request import Request, urlopen

import pandas as pd

from pamir.catalog import dataset_info, fetched_from, open_datasets
from pamir.contract import (EXPECTED as _EXPECTED, ContractError, check_table,
                            file_sha256, write_table)
from pamir.harmonize import harmonize


[docs] def cache_dir() -> Path: """Directory where fetched raw files and harmonized parquet are cached.""" env = os.environ.get("PAMIR_CACHE") if env: base = Path(env) else: try: from platformdirs import user_cache_dir base = Path(user_cache_dir("pamir")) except Exception: base = Path.home() / ".cache" / "pamir" base.mkdir(parents=True, exist_ok=True) return base
def _http_get(url: str) -> bytes: req = Request(url, headers={"User-Agent": "Mozilla/5.0 (compatible; pamir-credit)"}) with urlopen(req, timeout=120) as r: # noqa: S310 (trusted, https) return r.read() def _extract_from_zip(blob: bytes, fname: str, _depth: int = 0): """Return the bytes of ``fname`` from a zip, recursing one level into nested zips (the PAKDD archive nests the modeling data inside another zip).""" with zipfile.ZipFile(io.BytesIO(blob)) as z: member = next((m for m in z.namelist() if m.endswith(fname)), None) if member is not None: return z.read(member) if _depth == 0: for m in z.namelist(): if m.lower().endswith(".zip"): got = _extract_from_zip(z.read(m), fname, _depth + 1) if got is not None: return got return None def _fetch_raw(spec: Dict, raw_dir: Path) -> Path: """Fetch the raw source file into ``raw_dir`` and return its path.""" dl = spec["download"] kind, loc, fname = dl["kind"], dl["locator"], dl["file"] raw_dir.mkdir(parents=True, exist_ok=True) target = raw_dir / fname if kind == "github_raw": target.write_bytes(_http_get(loc)) return target if kind in ("uci_zip", "github_zip"): data = _extract_from_zip(_http_get(loc), fname) if data is None: raise FileNotFoundError(f"{fname} not found in zip for '{spec['id']}'") target.write_bytes(data) return target if kind == "kaggle_competition": try: import kagglehub except ImportError as e: raise ImportError( "kaggle competition source needs `kagglehub` — pip install pamir-credit[data]" ) from e # Single-file download (path=) — avoids pulling the whole ~700MB # competition, and kagglehub gives a clear message if the rules were not # accepted (a bare 401 otherwise). Rules must be accepted once at # https://www.kaggle.com/competitions/<comp>/rules try: got = Path(kagglehub.competition_download(loc, path=fname)) except TypeError: # older kagglehub without path= got = Path(kagglehub.competition_download(loc)) if got.is_file(): return got hit = next((p for p in got.rglob("*") if p.name == fname), None) if hit is None: raise FileNotFoundError( f"{fname} not found in competition '{loc}' (under {got})") return hit if kind == "kaggle": try: import kagglehub except ImportError as e: raise ImportError( "kaggle source needs `kagglehub` — pip install pamir-credit[data]" ) from e handle = kaggle_handle(dl) path = Path(kagglehub.dataset_download(handle)) hit = next((p for p in path.rglob("*") if p.name == fname), None) if hit is None: raise FileNotFoundError( f"{fname} not found in Kaggle dataset '{handle}' for '{spec['id']}'") return hit if kind == "hf": try: from huggingface_hub import hf_hub_download except ImportError as e: raise ImportError( "hf source needs `huggingface_hub` — pip install pamir-credit[data]" ) from e return Path(hf_hub_download(repo_id=loc, filename=fname, repo_type="dataset", revision=dl.get("revision"))) raise ValueError(f"unknown download.kind {kind!r} for '{spec['id']}'") def kaggle_handle(dl: Dict) -> str: """The kagglehub handle for a download spec, pinned when a version is set.""" version = dl.get("version") return f"{dl['locator']}/versions/{int(version)}" if version else dl["locator"] def _validate(dataset_id: str, df: pd.DataFrame, raw_sha256: Optional[str] = None) -> List[str]: """Compare a harmonized table with expected.json; return the mismatches. Kept for backward compatibility; see :func:`pamir.contract.check_table`. """ return check_table(dataset_id, df, raw_sha256=raw_sha256)
[docs] def download(dataset_id: str, force: bool = False, quiet: bool = False, strict: bool = True) -> Path: """Fetch and harmonize one dataset; return the path to the cached parquet. Parameters ---------- dataset_id : str A PaMIR dataset id (see :func:`pamir.list_datasets`). force : bool Re-fetch and re-harmonize even if a cached parquet exists. quiet : bool Suppress progress messages. Contract mismatches are never silent: with ``strict=False`` they are emitted as warnings regardless of ``quiet``. strict : bool If True (default), a rebuilt table that does not match its contract in ``expected.json`` raises :class:`pamir.contract.ContractError` and nothing is cached. If False, the table is cached with its provenance record marked ``contract_ok: false``; evaluations on it carry ``contract_ok = False`` and get no fleet mean. Returns ------- Path to ``<cache>/<dataset_id>.parquet``. """ from pamir._version import __version__ spec = dataset_info(dataset_id) out = cache_dir() / f"{dataset_id}.parquet" if out.exists() and not force: return out def say(msg): if not quiet: print(f"[pamir] {dataset_id}: {msg}", flush=True) say(f"fetching from {fetched_from(spec)} …") raw = _fetch_raw(spec, cache_dir() / "raw" / dataset_id) raw_sha = file_sha256(raw) say("harmonizing …") df = harmonize(raw, spec) issues = check_table(dataset_id, df, raw_sha256=raw_sha) if issues and strict: raise ContractError( dataset_id, issues, hint=("Nothing was cached. The pinned source snapshot may be " "unavailable or the harmonizer may behave differently in this " "environment; pass strict=False to cache the table anyway " "(its results are then flagged contract_ok=False).")) for issue in issues: warnings.warn(f"[pamir] {dataset_id}: contract mismatch: {issue}", UserWarning, stacklevel=2) write_table(df, out, { "dataset": dataset_id, "pamir_version": __version__, "contract_ok": not issues, "issues": issues, "raw_sha256": raw_sha, "fetched_from": fetched_from(spec), }) say(f"cached {len(df):,} rows, {df.shape[1] - 1} features -> {out}") return out
def download_open(force: bool = False, quiet: bool = False, strict: bool = True) -> list: """Download every dataset whose source needs no credentials. A zero-setup starter set (UCI / GitHub / HuggingFace) — no Kaggle account required. Returns the list of cached parquet paths. """ paths = [] for ds in open_datasets(): paths.append(download(ds, force=force, quiet=quiet, strict=strict)) return paths