Source code for pamir.catalog
"""Dataset catalog: metadata for every PaMIR dataset.
Two provenance fields answer different questions and are kept apart:
``source`` says where the data originate (an institution, a paper, a
competition), and ``fetched_from`` says what :func:`pamir.download` actually
retrieves. ``fetched_from`` is computed from the ``download`` spec rather than
stored, so it cannot drift from what the downloader does.
"""
import json
from pathlib import Path
from typing import Dict, List, Optional
import pandas as pd
_CATALOG_PATH = Path(__file__).parent / "data" / "catalog.json"
_catalog = None
def _load_raw() -> List[Dict]:
global _catalog
if _catalog is None:
_catalog = json.loads(_CATALOG_PATH.read_text(encoding="utf-8"))
return _catalog
[docs]
def load_catalog() -> pd.DataFrame:
"""Return a DataFrame with one row per dataset.
Columns: id, name, rows, features, defaults, DR, source, fetched_from,
license, geography, product, target_definition, and the recipe fields.
"""
rows = [dict(e, fetched_from=fetched_from(e)) for e in _load_raw()]
return pd.DataFrame(rows).set_index("id")
def fetched_from(entry: Dict) -> str:
"""Human-readable description of what the downloader fetches for an entry."""
dl = entry.get("download", {})
kind, loc = dl.get("kind"), dl.get("locator", "")
if kind == "kaggle":
version = dl.get("version")
return f"Kaggle dataset {loc}" + (f" (version {version})" if version else "")
if kind == "kaggle_competition":
return f"Kaggle competition {loc}"
if kind == "hf":
revision = dl.get("revision")
return f"Hugging Face dataset {loc}" + (f" (revision {revision[:12]})" if revision else "")
if kind == "uci_zip":
return f"UCI archive {loc}"
if kind in ("github_raw", "github_zip"):
return f"GitHub file {loc}"
return f"{kind} {loc}"
[docs]
def list_datasets() -> List[str]:
"""Return sorted list of dataset ids."""
return sorted(e["id"] for e in _load_raw())
# Download sources that require credentials to fetch.
_CREDENTIAL_KINDS = {"kaggle", "kaggle_competition"}
def _needs_credentials(entry: Dict) -> bool:
return entry.get("download", {}).get("kind") in _CREDENTIAL_KINDS
def open_datasets() -> List[str]:
"""Ids whose source needs **no credentials** to download.
These come from UCI / GitHub / HuggingFace and work on a
fresh install with no setup; the rest are on Kaggle, which kagglehub 1.0
fetches anonymously and older kagglehub versions only with Kaggle
credentials (see ``pamir.download``).
"""
return sorted(e["id"] for e in _load_raw() if not _needs_credentials(e))
[docs]
def dataset_info(dataset_id: str) -> Dict:
"""Return full metadata dict for one dataset.
Raises KeyError if the id is not in the catalog.
"""
for e in _load_raw():
if e["id"] == dataset_id:
d = dict(e)
d["needs_credentials"] = _needs_credentials(e)
d["fetched_from"] = fetched_from(e)
return d
raise KeyError(
f"Unknown dataset '{dataset_id}'. "
f"Available: {', '.join(list_datasets())}"
)