From fd555c1dffb9c64d5e7d91168c106e9472d35cd7 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 17:44:25 -0700 Subject: [PATCH 01/41] feat(dynacell/eval): add GT-only CP feature selection select_features pools GT with one model's prediction, so the kept CP subset moves with the model being evaluated. select_gt_features applies the same variance + |r|>0.9 criteria to GT rows alone; it is the selector the per-target CP reference is fit with. select_features stays for cross_condition_probe, now sharing the pruning helper. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/feature_select.py | 64 +++++++++++++++---- .../evaluation/feature_select_test.py | 50 +++++++++++++++ 2 files changed, 101 insertions(+), 13 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/feature_select.py b/applications/dynacell/src/dynacell/evaluation/feature_select.py index 22a60a9af..a2b339a54 100644 --- a/applications/dynacell/src/dynacell/evaluation/feature_select.py +++ b/applications/dynacell/src/dynacell/evaluation/feature_select.py @@ -16,8 +16,9 @@ import numpy as np # Defaults shared with downstream consumers (e.g. the cp_selected_feature_mask -# JSON sidecar emitted by the evaluation pipeline). Keep `select_features`, -# `variance_threshold`, and `correlation_threshold` keyword defaults aligned +# JSON sidecar emitted by the evaluation pipeline, and the CP reference). Keep +# `select_features`, `select_gt_features`, `variance_threshold`, and +# `correlation_threshold` keyword defaults aligned # with these constants so the sidecar cannot drift from the actual call. DEFAULT_FREQ_CUT = 0.05 DEFAULT_UNIQUE_CUT = 0.01 @@ -160,6 +161,53 @@ def correlation_threshold( return keep +def _prune_mask( + X: np.ndarray, + freq_cut: float, + unique_cut: float, + corr_threshold: float, +) -> np.ndarray: + """Variance pruning then correlation pruning on the survivors; return the keep-mask.""" + mask_var = variance_threshold(X, freq_cut=freq_cut, unique_cut=unique_cut) + mask_corr_local = correlation_threshold(X[:, mask_var], threshold=corr_threshold, method="pearson") + keep_mask = np.zeros(X.shape[1], dtype=bool) + keep_mask[np.flatnonzero(mask_var)[mask_corr_local]] = True + return keep_mask + + +def select_gt_features( + gt: np.ndarray, + freq_cut: float = DEFAULT_FREQ_CUT, + unique_cut: float = DEFAULT_UNIQUE_CUT, + corr_threshold: float = DEFAULT_CORR_THRESHOLD, +) -> np.ndarray: + """Select features on ground-truth cells alone, never on predictions. + + Same criteria as :func:`select_features`, but the fit set holds only GT + rows, so the kept feature subset is a property of the target and cannot + move with the model being evaluated. Used to build the per-target CP + reference (:mod:`dynacell.evaluation.cp_reference`). + + Parameters + ---------- + gt : np.ndarray + Shape ``(n_gt, n_features)``, float64. Rows pooled across every + dataset the reference is fit on. + freq_cut : float, optional + Forwarded to :func:`variance_threshold`. Defaults to ``0.05``. + unique_cut : float, optional + Forwarded to :func:`variance_threshold`. Defaults to ``0.01``. + corr_threshold : float, optional + Forwarded to :func:`correlation_threshold`. Defaults to ``0.9``. + + Returns + ------- + np.ndarray + Boolean keep-mask of shape ``(n_features,)``. + """ + return _prune_mask(gt, freq_cut=freq_cut, unique_cut=unique_cut, corr_threshold=corr_threshold) + + def select_features( gt: np.ndarray, pred: np.ndarray, @@ -203,16 +251,6 @@ def select_features( if gt.shape[1] != pred.shape[1]: raise ValueError(f"feature dim mismatch: gt.shape[1]={gt.shape[1]} vs pred.shape[1]={pred.shape[1]}.") - n_features = gt.shape[1] pooled = np.vstack([gt, pred]) - - mask_var = variance_threshold(pooled, freq_cut=freq_cut, unique_cut=unique_cut) - pooled2 = pooled[:, mask_var] - - mask_corr_local = correlation_threshold(pooled2, threshold=corr_threshold, method="pearson") - - keep_mask = np.zeros(n_features, dtype=bool) - var_indices = np.flatnonzero(mask_var) - keep_mask[var_indices[mask_corr_local]] = True - + keep_mask = _prune_mask(pooled, freq_cut=freq_cut, unique_cut=unique_cut, corr_threshold=corr_threshold) return gt[:, keep_mask], pred[:, keep_mask], keep_mask diff --git a/applications/dynacell/src/dynacell/evaluation/feature_select_test.py b/applications/dynacell/src/dynacell/evaluation/feature_select_test.py index 66e45ba4c..873e9138b 100644 --- a/applications/dynacell/src/dynacell/evaluation/feature_select_test.py +++ b/applications/dynacell/src/dynacell/evaluation/feature_select_test.py @@ -8,6 +8,7 @@ from dynacell.evaluation.feature_select import ( correlation_threshold, select_features, + select_gt_features, variance_threshold, ) @@ -158,3 +159,52 @@ def test_select_features_returns_aligned_filtered_arrays( assert gt_f.shape[0] == 30 assert pred_f.shape[0] == 25 assert gt_f.shape[1] == pred_f.shape[1] == int(keep_mask.sum()) + + +def test_select_gt_features_drops_low_variance_and_correlated( + rng: np.random.Generator, +) -> None: + """GT-only selection drops a constant, a near-constant, and one of a correlated pair.""" + n = 200 + a = rng.standard_normal(n) + near_constant = np.zeros(n) + near_constant[0] = 1.0 + gt = np.column_stack( + [ + np.zeros(n), # 0: constant -> variance drop + near_constant, # 1: one dominant level -> variance drop + a, # 2: A + a + 1e-4 * rng.standard_normal(n), # 3: A + eps -> correlation drop (one of 2/3) + rng.standard_normal(n), # 4: independent + rng.standard_normal(n), # 5: independent + ] + ) + + keep = select_gt_features(gt) + + assert keep.dtype == bool and keep.shape == (6,) + assert not keep[0] and not keep[1] + assert keep[2] ^ keep[3] + assert keep[4] and keep[5] + + +def test_select_gt_features_is_deterministic_and_ignores_predictions( + rng: np.random.Generator, +) -> None: + """The mask is a function of GT alone: repeated calls agree, and no pred enters. + + ``select_features`` pools GT with a prediction, so two models get two masks; the + GT-only selector is what makes the CP feature space model-independent. + """ + gt = rng.standard_normal((150, 10)) + gt[:, 7] = gt[:, 2] * 3.0 + 1e-6 * rng.standard_normal(150) + + first = select_gt_features(gt) + second = select_gt_features(gt.copy()) + + np.testing.assert_array_equal(first, second) + assert not (first[2] and first[7]) + # A prediction that decorrelates column 7 would flip the pooled mask; GT-only cannot see it. + pred = rng.standard_normal((150, 10)) * 50.0 + _, _, pooled_mask = select_features(gt, pred) + assert not np.array_equal(pooled_mask, first) From da1f1b4f7068a2442e858bae040bf147ad197460 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 17:44:45 -0700 Subject: [PATCH 02/41] fix(dynacell/eval): score CP metrics in one shared per-target GT space CP (GLCM+) KID/FID/cosine were computed in a model-dependent space: each eval dir chose its own feature subset from pooled GT+pred cells, then z-scored pred and GT each by its OWN mean/std. Every model got a different mask (er/ipsc: fnet3d_paper kept 11 features, fcmae_vscyto3d_pretrained 10), and any per-feature offset or scale error -- e.g. uniform over-smoothing -- was standardized away. Every eval now loads the target's CP reference (cp_reference.py): a keep-mask chosen on GT cells only, pooled over iPSC + A549 mock/denv/zikv, plus one mean/std fit on that GT. Both are applied to pred AND GT, at dataset level and per row. The linear probe gets the masked, unscaled arrays (it scales per fold). The per-FOV drop of target-all-zero columns existed only to keep the per-side z-score finite, so it is gone. The dim-mismatch StaleCacheError moves into CPReference. - feature_metrics.cp.reference_path (null -> DATA_ROOT/cp_reference/ __3d.json, shared by lite and full evals). - Missing reference -> FileNotFoundError naming the build command; reference of another CP recipe (version/norm/GLCM) -> StaleCacheError. Both are checked before any model load. - The reference's content hash is stamped in metrics_provenance.json, and _final_metrics_cache_valid refuses a cache stamped with another hash, or with none, so old CP values cannot be silently reused. - cp_selected_feature_mask.json now records the reference path + hash. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/README.md | 1 + .../dynacell/evaluation/_configs/eval.yaml | 5 + .../src/dynacell/evaluation/cp_reference.py | 329 ++++++++++++++++++ .../dynacell/evaluation/cp_reference_test.py | 173 +++++++++ .../evaluation/instance_metrics_test.py | 3 +- .../dynacell/src/dynacell/evaluation/paths.py | 37 ++ .../src/dynacell/evaluation/pipeline.py | 164 +++++---- .../src/dynacell/evaluation/pipeline_cache.py | 40 ++- .../src/dynacell/evaluation/provenance.py | 31 +- .../dynacell/evaluation/provenance_test.py | 30 +- applications/dynacell/tests/_eval_fixtures.py | 30 ++ .../tests/test_evaluation_pipeline.py | 11 +- .../test_evaluation_pipeline_parallel.py | 30 +- .../tests/test_evaluation_timepoint_guard.py | 59 +++- .../dynacell/tests/test_pipeline_cache.py | 4 +- 15 files changed, 838 insertions(+), 109 deletions(-) create mode 100644 applications/dynacell/src/dynacell/evaluation/cp_reference.py create mode 100644 applications/dynacell/src/dynacell/evaluation/cp_reference_test.py diff --git a/applications/dynacell/src/dynacell/evaluation/README.md b/applications/dynacell/src/dynacell/evaluation/README.md index 61fb39d14..d0c966f10 100644 --- a/applications/dynacell/src/dynacell/evaluation/README.md +++ b/applications/dynacell/src/dynacell/evaluation/README.md @@ -24,6 +24,7 @@ End-to-end evaluation pipeline for virtual staining predictions against fluoresc - `io.gt_path` — fluorescence ground truth (channel: `io.gt_channel_name`) - `io.cell_segmentation_path` — *optional* precomputed cell segmentation HCS OME-Zarr. Required when `compute_feature_metrics=true` or when building CP/DINOv3/DynaCLR/CELL-DINO cache entries. Position layout must match GT/pred 1:1. - `io.gt_cache_dir`, `io.pred_cache_dir` — *optional* artifact cache directories; must be distinct. See [Caches](#caches). +- `feature_metrics.cp.reference_path` — the target's CP (GLCM+) reference, required when `compute_feature_metrics=true`. `null` (default) resolves to `DATA_ROOT/cp_reference/__3d.json` for full and lite evals alike. The reference holds a feature mask chosen on GT cells only, pooled over iPSC + A549 mock/denv/zikv, and one shared GT mean/std. Every eval applies both to pred AND GT, so all models are scored in one CP space. Build it from existing GT caches with `tools/build_cp_reference.py --target `. Its hash is stamped in `metrics_provenance.json`, and a rebuilt reference invalidates the final-metrics cache. ## Quick start diff --git a/applications/dynacell/src/dynacell/evaluation/_configs/eval.yaml b/applications/dynacell/src/dynacell/evaluation/_configs/eval.yaml index dc4c2c2d7..d327084ff 100644 --- a/applications/dynacell/src/dynacell/evaluation/_configs/eval.yaml +++ b/applications/dynacell/src/dynacell/evaluation/_configs/eval.yaml @@ -141,6 +141,11 @@ feature_metrics: enabled: true levels: 32 distances: [1] + # Per-target CP reference: GT-only feature mask + one shared mean/std, applied to + # BOTH pred and GT so every model is scored in the same CP space. null -> registry + # default DATA_ROOT/cp_reference/__3d.json (lite evals use it too). + # Build with applications/dynacell/tools/build_cp_reference.py --target . + reference_path: null # In-focus slab for the 2D deep-feature crops + per-cell similarity. When # enabled, the max-Z projection is restricted to a band centered on the phase # focus plane (read from focus_slice zattrs; write via precompute-gt build.focus) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py new file mode 100644 index 000000000..d101bc3f0 --- /dev/null +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -0,0 +1,329 @@ +"""Per-target CP (GLCM+) reference: one shared feature space for every eval of a target. + +The CP track used to be scored in a model-dependent space. Each eval dir pooled +its own GT and prediction cells to choose a feature subset, then z-scored the +prediction and the GT each by its *own* mean/std. That gives every model a +different feature set. It also erases any per-feature offset or scale difference +between prediction and GT, which is the signature of over-smoothing, so that +error could not register in CP KID/FID/cosine at all. + +A CP reference fixes both. It is fit once per target on **GT cells only**, +pooled across the target's benchmark test sets (iPSC + A549 mock/denv/zikv; +HEK and the movie sets are excluded from the fit). It holds: + +* a feature keep-mask chosen by :func:`~dynacell.evaluation.feature_select.select_gt_features` + with the pipeline's criteria constants, and +* one per-feature mean/std over the kept features of that same pooled GT. + +Every eval of the target applies the same mask and the same scaler to BOTH +prediction and GT, at dataset level and per row. The artifact is JSON and is +built by ``applications/dynacell/tools/build_cp_reference.py`` from existing GT +CP caches. + +CP regionprops are always computed on the full GT/prediction volume. The +2D-track evals score 3D prediction stores, and the deep-feature focus slab never +touches CP. So every current eval runs in the ``3d`` CP space, and +:data:`CP_REFERENCE_DIMENSION` is the only dimension the registry holds. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import numpy as np +from omegaconf import DictConfig, OmegaConf + +from dynacell.evaluation.cache import StaleCacheError +from dynacell.evaluation.feature_select import ( + DEFAULT_CORR_THRESHOLD, + DEFAULT_FREQ_CUT, + DEFAULT_UNIQUE_CUT, + select_gt_features, +) +from dynacell.evaluation.paths import cp_reference_path + +#: Schema of the reference JSON. Bump on any change to its keys or their meaning. +CP_REFERENCE_SCHEMA = 1 + +#: The CP feature space every current eval runs in (CP regionprops are 3D). +CP_REFERENCE_DIMENSION = "3d" + +#: Keys excluded from the content hash: the hash itself, and the build timestamp, +#: so rebuilding from identical GT caches yields an identical hash and does not +#: needlessly invalidate every final-metrics cache stamped with it. +_UNHASHED_KEYS = frozenset({"sha256", "created_at"}) + + +def _build_command(target_name: str, dimension: str) -> str: + """Return the command that builds the registry reference for one target.""" + return ( + "uv run --no-sync python applications/dynacell/tools/build_cp_reference.py " + f"--target {target_name} --dimension {dimension}" + ) + + +def payload_sha256(payload: dict[str, Any]) -> str: + """Return the content hash of a reference payload. + + Parameters + ---------- + payload : dict + Reference JSON payload. ``sha256`` and ``created_at`` are ignored. + + Returns + ------- + str + Hex sha256 over the canonical (sorted-key, compact) JSON encoding. + """ + body = {k: v for k, v in payload.items() if k not in _UNHASHED_KEYS} + return hashlib.sha256(json.dumps(body, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def gt_matrix_sha256(gt: np.ndarray) -> str: + """Return a sha256 over a stacked GT cell matrix (shape + float64 C-order bytes). + + Recorded in the reference so a later GT re-cache that changes any fitted + value, or the cell count, is detectable against the artifact. + """ + arr = np.ascontiguousarray(gt, dtype=np.float64) + digest = hashlib.sha256(f"{arr.shape[0]}x{arr.shape[1]}:".encode()) + digest.update(arr.tobytes()) + return digest.hexdigest() + + +def fit_cp_reference( + gt: np.ndarray, + *, + target_name: str, + dimension: str, + feature_names: tuple[str, ...], + cp_identity: dict[str, Any], + datasets: list[dict[str, Any]], +) -> dict[str, Any]: + """Fit a CP reference on pooled GT cells and return its JSON payload. + + Parameters + ---------- + gt : np.ndarray + Shape ``(n_cells, n_features)``: every finite GT cell of every fit + dataset, stacked in ``datasets`` order. + target_name : str + Eval ``target_name`` (``nucleus``/``membrane``/``er``/``mitochondria``). + dimension : str + CP feature-space dimension, :data:`CP_REFERENCE_DIMENSION`. + feature_names : tuple of str + Column names of ``gt``, from ``active_cp_feature_names``. + cp_identity : dict + Dataset-independent CP recipe identity + (:func:`~dynacell.evaluation.pipeline_cache.cp_recipe_identity`). + datasets : list of dict + Per-dataset fit inputs (dataset ref, GT cache paths read, ``n_cells``). + + Returns + ------- + dict + JSON-serializable payload, including its own content ``sha256``. + + Raises + ------ + ValueError + If ``gt`` does not match ``feature_names``, holds a non-finite value, or + selection keeps a zero-variance feature. + """ + if gt.ndim != 2 or gt.shape[1] != len(feature_names): + raise ValueError(f"GT matrix shape {gt.shape} does not match {len(feature_names)} CP feature names") + if not np.isfinite(gt).all(): + raise ValueError("GT matrix holds non-finite values; drop them before fitting") + n_cells = sum(int(d["n_cells"]) for d in datasets) + if n_cells != gt.shape[0]: + raise ValueError(f"datasets report {n_cells} cells but the GT matrix has {gt.shape[0]} rows") + + gt = np.asarray(gt, dtype=np.float64) + keep_mask = select_gt_features( + gt, freq_cut=DEFAULT_FREQ_CUT, unique_cut=DEFAULT_UNIQUE_CUT, corr_threshold=DEFAULT_CORR_THRESHOLD + ) + kept = gt[:, keep_mask] + mean = kept.mean(axis=0) + std = kept.std(axis=0) + if not (std > 0).all(): + raise ValueError(f"selection kept zero-variance features: {np.array(feature_names)[keep_mask][std == 0]}") + + payload: dict[str, Any] = { + "schema": CP_REFERENCE_SCHEMA, + "target_name": target_name, + "dimension": dimension, + "feature_names": list(feature_names), + "keep_mask": [bool(b) for b in keep_mask], + "kept_feature_names": [n for n, k in zip(feature_names, keep_mask, strict=True) if k], + "mean": [float(v) for v in mean], + "std": [float(v) for v in std], + "criteria": { + "freq_cut": DEFAULT_FREQ_CUT, + "unique_cut": DEFAULT_UNIQUE_CUT, + "corr_threshold": DEFAULT_CORR_THRESHOLD, + "fit_on": "gt_only", + }, + "cp_identity": cp_identity, + "fit": { + "datasets": datasets, + "n_cells": int(gt.shape[0]), + "gt_matrix_sha256": gt_matrix_sha256(gt), + }, + "created_at": datetime.now(UTC).isoformat(), + } + payload["sha256"] = payload_sha256(payload) + return payload + + +def write_cp_reference(payload: dict[str, Any], path: Path) -> None: + """Write a reference payload to ``path`` (parents created).""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=1) + "\n") + + +@dataclass(frozen=True) +class CPReference: + """A loaded, hash-verified CP reference. + + Picklable (plain arrays + scalars) so it can be shipped to spawn workers. + """ + + path: str + sha256: str + target_name: str + dimension: str + feature_names: tuple[str, ...] + keep_mask: np.ndarray + mean: np.ndarray + std: np.ndarray + cp_identity: dict[str, Any] + + def _check_width(self, x: np.ndarray) -> None: + """Raise when a CP matrix does not have the reference's column count.""" + if x.ndim != 2 or x.shape[1] != len(self.feature_names): + raise StaleCacheError( + f"CP feature dimension mismatch: matrix has shape {x.shape}, the CP reference " + f"{self.path} has {len(self.feature_names)} features. A CP feature cache was built " + "with a different recipe (e.g. a GLCM toggle or a CP_FEATURE_VERSION change without " + "a cache rebuild). Rebuild with force_recompute.gt_cp=true and/or pred_cp=true " + "(or force_recompute.all=true)." + ) + + def select(self, x: np.ndarray) -> np.ndarray: + """Return the kept columns of a raw CP matrix, unscaled.""" + self._check_width(x) + return x[:, self.keep_mask] + + def transform(self, x: np.ndarray) -> np.ndarray: + """Return the kept columns of a raw CP matrix, standardized by the shared GT scaler.""" + return (self.select(x) - self.mean) / self.std + + +def load_cp_reference(path: Path, *, target_name: str, dimension: str) -> CPReference: + """Load and hash-verify a CP reference. + + Parameters + ---------- + path : pathlib.Path + Reference JSON. + target_name : str + The eval's ``target_name``; the reference must have been fit for it. + dimension : str + The eval's CP feature-space dimension. + + Returns + ------- + CPReference + The verified reference. + + Raises + ------ + FileNotFoundError + If ``path`` does not exist; the message names the build command. + ValueError + If the stored hash disagrees with the content, or the reference was fit + for another target, dimension, or schema. + """ + if not path.is_file(): + raise FileNotFoundError( + f"CP reference {path} not found. CP (GLCM+) feature metrics are scored in a " + f"per-target GT feature space; build it with:\n {_build_command(target_name, dimension)}\n" + "(or point feature_metrics.cp.reference_path at an existing reference)." + ) + payload = json.loads(path.read_text()) + if payload["sha256"] != payload_sha256(payload): + raise ValueError(f"CP reference {path} content does not match its recorded sha256 (edited by hand?)") + if payload["schema"] != CP_REFERENCE_SCHEMA: + raise ValueError(f"CP reference {path} has schema {payload['schema']}, expected {CP_REFERENCE_SCHEMA}") + if payload["target_name"] != target_name or payload["dimension"] != dimension: + raise ValueError( + f"CP reference {path} was fit for ({payload['target_name']!r}, {payload['dimension']!r}), " + f"but this eval is ({target_name!r}, {dimension!r})" + ) + return CPReference( + path=str(path), + sha256=payload["sha256"], + target_name=payload["target_name"], + dimension=payload["dimension"], + feature_names=tuple(payload["feature_names"]), + keep_mask=np.asarray(payload["keep_mask"], dtype=bool), + mean=np.asarray(payload["mean"], dtype=np.float64), + std=np.asarray(payload["std"], dtype=np.float64), + cp_identity=payload["cp_identity"], + ) + + +def resolve_cp_reference_path(config: DictConfig) -> Path: + """Return ``feature_metrics.cp.reference_path``, or the registry default when null. + + The registry default lives under ``DATA_ROOT`` for full and lite evals alike, + so a lite eval scores in exactly the full benchmark's feature space. + """ + override = OmegaConf.select(config, "feature_metrics.cp.reference_path", default=None) + if override is not None: + return Path(override) + return cp_reference_path(config.target_name, CP_REFERENCE_DIMENSION) + + +def load_checked_cp_reference( + config: DictConfig, *, cp_identity: dict[str, Any], feature_names: tuple[str, ...] +) -> CPReference: + """Load the eval's CP reference and refuse one built for a different CP recipe. + + Parameters + ---------- + config : DictConfig + Eval config (``target_name``, ``feature_metrics.cp.reference_path``). + cp_identity : dict + The eval's own CP recipe identity. + feature_names : tuple of str + The eval's CP column names. + + Returns + ------- + CPReference + The verified reference. + + Raises + ------ + FileNotFoundError + If the reference does not exist. + StaleCacheError + If the reference's CP recipe identity or feature names differ from the eval's. + """ + ref = load_cp_reference( + resolve_cp_reference_path(config), target_name=config.target_name, dimension=CP_REFERENCE_DIMENSION + ) + if ref.cp_identity != cp_identity or ref.feature_names != feature_names: + raise StaleCacheError( + f"CP reference {ref.path} was built for a different CP recipe.\n" + f" reference identity: {ref.cp_identity}\n eval identity: {cp_identity}\n" + f" reference features: {len(ref.feature_names)}; eval features: {len(feature_names)}\n" + f"Rebuild it from caches of the current recipe:\n {_build_command(ref.target_name, ref.dimension)}" + ) + return ref diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py new file mode 100644 index 000000000..254635aca --- /dev/null +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -0,0 +1,173 @@ +"""Tests for the per-target CP reference and the pipeline's use of it.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest +from omegaconf import OmegaConf + +from dynacell.evaluation.cp_reference import ( + CP_REFERENCE_DIMENSION, + fit_cp_reference, + load_cp_reference, + payload_sha256, + resolve_cp_reference_path, + write_cp_reference, +) +from dynacell.evaluation.feature_metrics import compute_feature_similarity_pairwise +from dynacell.evaluation.feature_select import select_features +from dynacell.evaluation.paths import DATA_ROOT, LITE_DATA_ROOT +from dynacell.evaluation.pipeline import _BackboneLists, _extend_backbone, _stage_cp_dataset_inputs + +_N_FEATURES = 8 +_NAMES = tuple(f"f{i}" for i in range(_N_FEATURES)) + + +def _gt(n: int = 300, seed: int = 0) -> np.ndarray: + """Synthetic GT CP matrix: 6 independent columns, one near-duplicate, one constant.""" + rng = np.random.default_rng(seed) + x = rng.standard_normal((n, _N_FEATURES)) * np.arange(1, _N_FEATURES + 1) + 10.0 + x[:, 6] = x[:, 0] * 2.0 + 1e-6 * rng.standard_normal(n) # |r| ~ 1 with f0 + x[:, 7] = 3.0 # constant + return x + + +def _reference(tmp_path: Path, gt: np.ndarray): + """Fit, write and load a reference on ``gt``.""" + payload = fit_cp_reference( + gt, + target_name="er", + dimension=CP_REFERENCE_DIMENSION, + feature_names=_NAMES, + cp_identity={"cp_feature_version": "test"}, + datasets=[{"dataset": "synthetic", "n_cells": gt.shape[0]}], + ) + path = tmp_path / "er__3d.json" + write_cp_reference(payload, path) + return load_cp_reference(path, target_name="er", dimension=CP_REFERENCE_DIMENSION), payload + + +def _old_per_side_zscore(pred: np.ndarray, target: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """The retired transform: each side standardized by its own mean/std.""" + return ( + (pred - pred.mean(axis=0)) / (pred.std(axis=0) + 1e-8), + (target - target.mean(axis=0)) / (target.std(axis=0) + 1e-8), + ) + + +def _lists(pred: np.ndarray, gt: np.ndarray) -> _BackboneLists: + """One-FOV ``_BackboneLists`` holding ``(pred, gt)`` as two timepoints.""" + bb = _BackboneLists() + half = pred.shape[0] // 2 + _extend_backbone(bb, pred[:half], gt[:half], "A/1/0", 0) + _extend_backbone(bb, pred[half:], gt[half:], "A/1/0", 1) + return bb + + +def test_fit_drops_constant_and_correlated_and_records_provenance(tmp_path: Path) -> None: + gt = _gt() + ref, payload = _reference(tmp_path, gt) + assert not ref.keep_mask[7] + assert ref.keep_mask[0] ^ ref.keep_mask[6] + np.testing.assert_allclose(ref.mean, gt[:, ref.keep_mask].mean(axis=0)) + np.testing.assert_allclose(ref.std, gt[:, ref.keep_mask].std(axis=0)) + assert payload["fit"]["n_cells"] == gt.shape[0] + assert payload["criteria"]["fit_on"] == "gt_only" + assert len(payload["fit"]["gt_matrix_sha256"]) == 64 + # Transformed GT is exactly standardized by the shared scaler. + z = ref.transform(gt) + np.testing.assert_allclose(z.mean(axis=0), 0.0, atol=1e-10) + np.testing.assert_allclose(z.std(axis=0), 1.0, atol=1e-10) + + +def test_hash_ignores_build_time_and_detects_edits(tmp_path: Path) -> None: + """Identical inputs hash identically; a hand edit fails the load.""" + (tmp_path / "rebuild").mkdir() + _, first = _reference(tmp_path, _gt()) + _, second = _reference(tmp_path / "rebuild", _gt()) + assert first["sha256"] == second["sha256"] == payload_sha256({**first, "created_at": "another time"}) + + path = tmp_path / "er__3d.json" + edited = json.loads(path.read_text()) + edited["std"][0] *= 2.0 + path.write_text(json.dumps(edited)) + with pytest.raises(ValueError, match="does not match its recorded sha256"): + load_cp_reference(path, target_name="er", dimension=CP_REFERENCE_DIMENSION) + + +def test_load_refuses_missing_and_other_target(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError, match="build_cp_reference.py --target er --dimension 3d"): + load_cp_reference(tmp_path / "absent.json", target_name="er", dimension=CP_REFERENCE_DIMENSION) + _reference(tmp_path, _gt()) + with pytest.raises(ValueError, match="was fit for"): + load_cp_reference(tmp_path / "er__3d.json", target_name="nucleus", dimension=CP_REFERENCE_DIMENSION) + + +def test_registry_path_is_shared_by_lite_and_full() -> None: + """``reference_path: null`` resolves under DATA_ROOT, never LITE_DATA_ROOT.""" + config = OmegaConf.create({"target_name": "mitochondria", "feature_metrics": {"cp": {"reference_path": None}}}) + path = resolve_cp_reference_path(config) + assert path == DATA_ROOT / "cp_reference" / "mitochondria__3d.json" + assert LITE_DATA_ROOT not in path.parents + config.feature_metrics.cp.reference_path = "/x/ref.json" + assert resolve_cp_reference_path(config) == Path("/x/ref.json") + + +def test_two_models_share_one_mask(tmp_path: Path) -> None: + """Two different predictions against one GT are scored on the identical feature subset. + + The retired pooled GT+pred selection gives these two models different masks, + which is the model-dependence the reference removes. + """ + gt = _gt() + ref, _ = _reference(tmp_path, gt) + rng = np.random.default_rng(1) + pred_a = gt + 0.1 * rng.standard_normal(gt.shape) + pred_b = gt.copy() + pred_b[:, 6] = rng.standard_normal(gt.shape[0]) * 50.0 # decorrelates f6 from f0 in the pool + pred_b[:, 7] = rng.standard_normal(gt.shape[0]) # a constant GT column that varies in pred + + masks = [] + for i, pred in enumerate((pred_a, pred_b)): + save_dir = tmp_path / f"model_{i}" + save_dir.mkdir() + staged = _stage_cp_dataset_inputs(_lists(pred, gt), ref, save_dir) + sidecar = json.loads((save_dir / "cp_selected_feature_mask.json").read_text()) + assert sidecar["reference_sha256"] == ref.sha256 + assert staged[1].shape[1] == staged[2].shape[1] == int(ref.keep_mask.sum()) + # Probe inputs are masked but unscaled. + np.testing.assert_array_equal(staged[4], gt[:, ref.keep_mask]) + masks.append(sidecar["keep_mask"]) + assert masks[0] == masks[1] == [bool(b) for b in ref.keep_mask] + + old_a = select_features(gt, pred_a)[2] + old_b = select_features(gt, pred_b)[2] + assert not np.array_equal(old_a, old_b) + + +def test_over_smoothing_registers_in_the_shared_space(tmp_path: Path) -> None: + """``pred = 0.5 * GT + c`` (compressed range + offset) now scores clearly off GT. + + The retired per-side z-score maps any per-feature affine distortion of GT back + onto GT exactly, so it reported a perfect match (cosine 1, KID ~0). The shared + GT scaler keeps the distortion, so the same prediction is now penalized. + """ + gt = _gt(n=400) + ref, _ = _reference(tmp_path, gt) + pred = 0.5 * gt + 7.0 + + old_p, old_t = _old_per_side_zscore(ref.select(pred), ref.select(gt)) + old = compute_feature_similarity_pairwise(old_p, old_t, "CP", compute_fid=False) + new = compute_feature_similarity_pairwise(ref.transform(pred), ref.transform(gt), "CP", compute_fid=False) + + # KID of GT against itself: the estimator's floor for "indistinguishable" (not exactly 0). + floor = compute_feature_similarity_pairwise(ref.transform(gt), ref.transform(gt), "CP", compute_fid=False) + + assert old["CP_Median_Cosine_Similarity"] == pytest.approx(1.0, abs=1e-6) + assert old["CP_KID"] == pytest.approx(floor["CP_KID"], abs=1e-4) + assert new["CP_Median_Cosine_Similarity"] < 0.9 + assert new["CP_KID"] > 1.0 + assert new["CP_KID"] > 100 * abs(floor["CP_KID"]) diff --git a/applications/dynacell/src/dynacell/evaluation/instance_metrics_test.py b/applications/dynacell/src/dynacell/evaluation/instance_metrics_test.py index febe24dcf..75948ea72 100644 --- a/applications/dynacell/src/dynacell/evaluation/instance_metrics_test.py +++ b/applications/dynacell/src/dynacell/evaluation/instance_metrics_test.py @@ -136,6 +136,7 @@ def test_instance_ap_is_written_to_mask_metrics_csv(tmp_path): row = instance_average_precision(lab, lab) config = OmegaConf.create( { + "compute_feature_metrics": False, "save": { "save_dir": str(tmp_path), "mask_csv_filename": "mask_metrics.csv", @@ -144,7 +145,7 @@ def test_instance_ap_is_written_to_mask_metrics_csv(tmp_path): "pixel_metrics_filename": "pixel_metrics.npy", "feature_csv_filename": "feature_metrics.csv", "feature_metrics_filename": "feature_metrics.npy", - } + }, } ) save_metrics(config, mask_metrics=[{"FOV": "0/0/0", "Timepoint": 0, **row}]) diff --git a/applications/dynacell/src/dynacell/evaluation/paths.py b/applications/dynacell/src/dynacell/evaluation/paths.py index cfe0acc01..8f02509e4 100644 --- a/applications/dynacell/src/dynacell/evaluation/paths.py +++ b/applications/dynacell/src/dynacell/evaluation/paths.py @@ -960,6 +960,43 @@ def pred_cache_dir( return Path(data_root) / test_set / subroot / organelle / model / train_set / leaf +#: Eval ``target_name`` values that have a CP reference (one per benchmark target). +_CP_REFERENCE_TARGETS: frozenset[str] = frozenset({"nucleus", "membrane", "er", "mitochondria"}) + + +def cp_reference_path(target_name: str, dimension: str, data_root: str | Path = DATA_ROOT) -> Path: + """Return the registry path of a target's CP (GLCM+) reference. + + ``DATA_ROOT/cp_reference/__.json``. Keyed by the eval + ``target_name`` (``mitochondria``, not the grammar's ``mito``), since that is + what an eval config carries. Lite evals use this same ``DATA_ROOT`` path, never + ``LITE_DATA_ROOT``: the reference is fit on the full test sets so lite and full + score in one feature space. See :mod:`dynacell.evaluation.cp_reference`. + + Parameters + ---------- + target_name : str + Eval target (``nucleus``/``membrane``/``er``/``mitochondria``). + dimension : str + CP feature-space dimension (``3d``). + data_root : str or pathlib.Path, optional + Registry root. Defaults to :data:`DATA_ROOT`. + + Returns + ------- + pathlib.Path + The reference JSON path. + + Raises + ------ + ValueError + If ``target_name`` has no CP reference. + """ + if target_name not in _CP_REFERENCE_TARGETS: + raise ValueError(f"no CP reference for target {target_name!r}; expected one of {sorted(_CP_REFERENCE_TARGETS)}") + return Path(data_root) / "cp_reference" / f"{target_name}__{dimension}.json" + + def metrics_repo_dir( organelle: str, model: str, diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline.py b/applications/dynacell/src/dynacell/evaluation/pipeline.py index 0b083ff6f..29c86711c 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline.py @@ -17,20 +17,16 @@ from tqdm import tqdm from dynacell.evaluation._ref_hook import apply_dataset_ref -from dynacell.evaluation.cache import FeatureKind, StaleCacheError +from dynacell.evaluation.cache import FeatureKind +from dynacell.evaluation.cp_reference import CPReference, load_checked_cp_reference from dynacell.evaluation.cross_condition_probe import run_for_group as _cross_condition_run_for_group from dynacell.evaluation.feature_metrics import ( compute_feature_similarity, compute_feature_similarity_pairwise, ) -from dynacell.evaluation.feature_select import ( - DEFAULT_CORR_THRESHOLD, - DEFAULT_FREQ_CUT, - DEFAULT_UNIQUE_CUT, - select_features, -) from dynacell.evaluation.linear_probe import indistinguishability, paired_auroc from dynacell.evaluation.metrics import ( + CP_FEATURE_VERSION, active_cp_feature_names, ascupy, build_crops, @@ -45,6 +41,7 @@ ) from dynacell.evaluation.model_loader import EvalModels, init_cache_contexts, load_eval_models from dynacell.evaluation.pipeline_cache import ( + cp_recipe_identity, cpdino_infer_kwargs, flush_manifest, fov_cp_features, @@ -59,6 +56,7 @@ seg_spacing, ) from dynacell.evaluation.provenance import ( + PROVENANCE_FILENAME, check_cubic_pin, metrics_provenance_matches, write_metrics_provenance, @@ -101,34 +99,23 @@ def _build_focus_slabs_map(config: DictConfig, gt_positions) -> dict[str, list[s } -def _zscore_per_side(pred: np.ndarray, target: np.ndarray) -> tuple[np.ndarray, np.ndarray]: - """Per-side z-score: separate (mean, std) computed for each matrix.""" - pred_z = (pred - pred.mean(axis=0)) / (pred.std(axis=0) + 1e-8) - target_z = (target - target.mean(axis=0)) / (target.std(axis=0) + 1e-8) - return pred_z, target_z - - -def _cp_dropzero_zscore(pred_raw: np.ndarray, target_raw: np.ndarray) -> tuple[np.ndarray, np.ndarray]: - """Per-(FOV, timepoint) CP cleanup: drop target-zero columns then z-score. +def _eval_cp_reference(config: DictConfig) -> CPReference: + """Load the CP reference this eval scores in, refusing one of a different CP recipe. - Returns ``(np.empty(...), np.empty(...))`` when all columns drop, so - the caller can short-circuit and emit a NaN row. + The recipe identity and column names come from the same ``feature_metrics.cp.*`` + config and :data:`CP_FEATURE_VERSION` that key the CP feature caches, so a + reference built from caches of another recipe cannot be applied to this run's + features. See :mod:`dynacell.evaluation.cp_reference`. """ - if pred_raw.ndim == 2 and target_raw.ndim == 2 and pred_raw.shape[1] and target_raw.shape[1]: - if pred_raw.shape[1] != target_raw.shape[1]: - raise StaleCacheError( - f"CP feature dimension mismatch: pred has {pred_raw.shape[1]} columns, " - f"GT has {target_raw.shape[1]}. A CP feature cache was built with a different " - "recipe (e.g. a GLCM toggle or a CP_FEATURE_VERSION change without a cache " - "rebuild). Rebuild with force_recompute.gt_cp=true and/or pred_cp=true " - "(or force_recompute.all=true)." - ) - non_zero_cols = ~np.all(target_raw == 0, axis=0) - pred_mat = pred_raw[:, non_zero_cols] - target_mat = target_raw[:, non_zero_cols] - if pred_mat.size == 0: - return pred_mat, target_mat - return _zscore_per_side(pred_mat, target_mat) + norm_node = OmegaConf.select(config, "feature_metrics.cp.norm", default=None) + glcm_node = OmegaConf.select(config, "feature_metrics.cp.glcm", default=None) + norm = OmegaConf.to_container(norm_node, resolve=True) if norm_node is not None else {} + glcm = OmegaConf.to_container(glcm_node, resolve=True) if glcm_node is not None else {} + return load_checked_cp_reference( + config, + cp_identity=cp_recipe_identity(CP_FEATURE_VERSION, norm, glcm), + feature_names=tuple(active_cp_feature_names(bool(glcm.get("enabled", False)))), + ) def _real_vs_pred_probe( @@ -350,6 +337,43 @@ def _extend_backbone( bb.gt_ts.append(t_arr) +def _stage_cp_dataset_inputs( + cp: _BackboneLists, cp_reference: CPReference, save_dir: Path +) -> tuple[str, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Stage the dataset-level CP inputs in the reference feature space. + + KID/FID/cosine get the reference-masked features standardized by the shared GT + scaler (identical for pred and GT); the linear probe gets the reference-masked + raw features, since it fits its own per-fold scaler. Writes + ``cp_selected_feature_mask.json`` recording which reference was applied. + + Returns + ------- + tuple + ``("CP", pred_metric, target_metric, pred_probe, target_probe, pred_fovs, target_fovs)``. + """ + pred_cp_raw = np.concatenate(cp.pred_feats, axis=0) + target_cp_raw = np.concatenate(cp.gt_feats, axis=0) + mask_payload = { + "reference_path": cp_reference.path, + "reference_sha256": cp_reference.sha256, + "feature_names": list(cp_reference.feature_names), + "keep_mask": [bool(b) for b in cp_reference.keep_mask], + "n_kept": int(cp_reference.keep_mask.sum()), + "n_total": int(cp_reference.keep_mask.size), + } + (save_dir / "cp_selected_feature_mask.json").write_text(json.dumps(mask_payload, indent=2)) + return ( + "CP", + cp_reference.transform(pred_cp_raw), + cp_reference.transform(target_cp_raw), + cp_reference.select(pred_cp_raw), + cp_reference.select(target_cp_raw), + np.concatenate(cp.pred_fovs, axis=0), + np.concatenate(cp.gt_fovs, axis=0), + ) + + @dataclass class FovResult: """Output of ``_process_one_fov``: everything one FOV contributes to the run. @@ -557,11 +581,15 @@ def _process_one_fov( celldino_feature_extractor, morphem_feature_extractor, microssim_sim, + cp_reference: CPReference | None, predict_cached=None, target_cached=None, ) -> FovResult: """Compute everything one FOV contributes to the eval and return a FovResult. + ``cp_reference`` is the target's CP reference (``None`` only when + ``compute_feature_metrics=false``); per-timepoint CP metrics are scored in it. + No side effects on shared parent state (no segmentation_results plate writes, no manifest flush). The parent aggregator handles those — see ``_aggregate_fov_result``. Used by both the serial and process FOV-loop @@ -915,8 +943,10 @@ def _nuclei_seeds(segment_fn, **infer) -> np.ndarray | None: pred_celldino = pred_per_t["celldino"][t] if pred_per_t["celldino"] is not None else None pred_morphem = pred_per_t["morphem"][t] if pred_per_t["morphem"] is not None else None pred_cp, gt_cp_t = drop_paired_nonfinite_rows(pred_cp, gt_cp_per_t[t]) + # Same reference mask + shared GT scaler as the dataset-level block, so a + # per-row CP value is comparable across models and FOVs. if pred_cp.size and gt_cp_t.size: - pred_cp_z, gt_cp_z = _cp_dropzero_zscore(pred_cp, gt_cp_t) + pred_cp_z, gt_cp_z = cp_reference.transform(pred_cp), cp_reference.transform(gt_cp_t) else: pred_cp_z, gt_cp_z = pred_cp, gt_cp_t # Prefixes come from _COLUMN_PREFIX, not literals: these per-timepoint @@ -1167,6 +1197,7 @@ def _worker_run_fov( pos_name: str, cuda_empty_every_n: int, microssim_sim, + cp_reference: CPReference | None, ) -> FovResult: """Worker entry point: process one FOV by name and return FovResult. @@ -1178,7 +1209,8 @@ def _worker_run_fov( ``microssim_sim`` is the leaf-level fitted MicroMS3IM (or ``None`` when ``compute_microssim=false``); shipped per submission rather than via worker state because the parent fits it after the position list is - finalized and before any worker pool spawns. + finalized and before any worker pool spawns. ``cp_reference`` is shipped the + same way so every worker scores CP in the parent's verified reference. """ _worker_setup(config) state = _WORKER_STATE @@ -1221,6 +1253,7 @@ def _worker_run_fov( state["celldino"], state["morphem"], microssim_sim, + cp_reference, ) # Worker-side manifest flush so interrupted runs preserve progress even @@ -1261,6 +1294,9 @@ def evaluate_predictions(config: DictConfig, *, models: EvalModels | None = None OmegaConf.resolve(config) reset_timings() _validate_instance_ap_config(config) + # Before any model load or cache write: a missing or foreign-recipe CP reference + # must fail the run up front, not after the per-FOV loop. + cp_reference = _eval_cp_reference(config) if config.compute_feature_metrics else None use_gpu = bool(getattr(config, "use_gpu", True)) @@ -1607,6 +1643,7 @@ def _aggregate(result: FovResult) -> None: celldino_feature_extractor, morphem_feature_extractor, microssim_sim, + cp_reference, predict_cached=cached_pair[0], target_cached=cached_pair[1], ) @@ -1645,6 +1682,7 @@ def _aggregate(result: FovResult) -> None: pos_name, runtime.cuda_empty_cache_every_n_timepoints, microssim_sim, + cp_reference, ): pos_name for pos_name in pos_names_in_order } @@ -1682,45 +1720,15 @@ def _aggregate(result: FovResult) -> None: # Stage per-prefix inputs: (pred_for_metric, target_for_metric, # pred_for_probe, target_for_probe, pred_fovs, target_fovs). - # CP gets pruning + z-score; the pre-prune CP arrays feed the - # linear probe so MADScaler can normalize per-fold. + # CP is scored in the target's reference space (GT-only mask + shared GT + # scaler); the masked-but-unscaled CP arrays feed the linear probe so + # MADScaler can normalize per-fold. prefix_inputs: list[tuple[str, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]] = [] - cp = parent_lists["cp"] - if cp.pred_feats: - pred_cp_raw = np.concatenate(cp.pred_feats, axis=0) - target_cp_raw = np.concatenate(cp.gt_feats, axis=0) - target_cp_filtered, pred_cp_filtered, cp_keep_mask = select_features(target_cp_raw, pred_cp_raw) - cp_glcm_enabled = bool(OmegaConf.select(config, "feature_metrics.cp.glcm.enabled", default=False)) - mask_payload = { - "feature_names": list(active_cp_feature_names(cp_glcm_enabled)), - "keep_mask": [bool(b) for b in cp_keep_mask], - "n_kept": int(cp_keep_mask.sum()), - "n_total": int(cp_keep_mask.size), - "criteria": { - "freq_cut": DEFAULT_FREQ_CUT, - "unique_cut": DEFAULT_UNIQUE_CUT, - "corr_threshold": DEFAULT_CORR_THRESHOLD, - }, - } - (save_dir / "cp_selected_feature_mask.json").write_text(json.dumps(mask_payload, indent=2)) - if pred_cp_filtered.size and target_cp_filtered.size: - pred_cp_z, target_cp_z = _zscore_per_side(pred_cp_filtered, target_cp_filtered) - else: - pred_cp_z, target_cp_z = pred_cp_filtered, target_cp_filtered - prefix_inputs.append( - ( - "CP", - pred_cp_z, - target_cp_z, - pred_cp_filtered, - target_cp_filtered, - np.concatenate(cp.pred_fovs, axis=0), - np.concatenate(cp.gt_fovs, axis=0), - ) - ) + if parent_lists["cp"].pred_feats: + prefix_inputs.append(_stage_cp_dataset_inputs(parent_lists["cp"], cp_reference, save_dir)) - for key in deep_kinds: # cp is handled above (pruning + z-score) + for key in deep_kinds: # cp is handled above (reference space) bb = parent_lists[key] if bb.pred_feats: pred_arr = np.concatenate(bb.pred_feats, axis=0) @@ -1815,7 +1823,12 @@ def save_metrics(config: DictConfig, pixel_metrics=None, mask_metrics=None, feat # whether the cache is comparable instead of assuming it is. Stamping first meant a # crash partway through the loop above left a fresh stamp certifying a previous # run's rows -- the exact misattribution the sidecar exists to make detectable. - write_metrics_provenance(save_dir) + write_metrics_provenance(save_dir, cp_reference_sha256=_current_cp_reference_sha256(config)) + + +def _current_cp_reference_sha256(config: DictConfig) -> str | None: + """Content hash of the CP reference this config scores in; ``None`` without feature metrics.""" + return _eval_cp_reference(config).sha256 if config.compute_feature_metrics else None #: Pixel columns that only a dual-scaling run writes. Their absence marks a pixel @@ -1841,7 +1854,12 @@ def _final_metrics_cache_valid(config: DictConfig) -> bool: # Metric values are not comparable across cubic versions — FSC/FRC/Spectral_PCC # move (see dynacell.evaluation.provenance). A cache with no stamp, or one # stamped with a different cubic, is not reusable regardless of its columns. - if not metrics_provenance_matches(save_dir): + # Likewise for the CP reference: CP KID/FID/cosine were scored in the reference + # stamped beside them, so a rebuilt (or never-stamped) reference forces a recompute. + # An unstamped dir has nothing to reuse, so it needs no reference to be rejected. + if not (save_dir / PROVENANCE_FILENAME).is_file(): + return False + if not metrics_provenance_matches(save_dir, cp_reference_sha256=_current_cp_reference_sha256(config)): return False pixel_ok = (save_dir / config.save.pixel_metrics_filename).exists() mask_path = save_dir / config.save.mask_metrics_filename diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline_cache.py b/applications/dynacell/src/dynacell/evaluation/pipeline_cache.py index 097781698..0bd53ff73 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline_cache.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline_cache.py @@ -980,16 +980,46 @@ def _cp_identity(ctx: _CacheContext) -> dict[str, Any]: ``diff_artifact_params`` compares the current identity's keys, so dropping them is sufficient (any stale stored value is simply ignored). """ - glcm = ctx.cp_glcm or {} - norm = ctx.cp_norm or {} + return { + "spacing": ctx.spacing, + **cp_recipe_identity(ctx.cp_feature_version, ctx.cp_norm, ctx.cp_glcm), + **ctx.source_tag, + } + + +def cp_recipe_identity( + cp_feature_version: str | None, cp_norm: dict[str, Any], cp_glcm: dict[str, Any] +) -> dict[str, Any]: + """Return the dataset-independent part of the CP feature identity. + + Everything in :func:`_cp_identity` except ``spacing`` (which differs between + iPSC, A549 and HEK) and the side tag: the recipe that decides what the CP + columns *mean*. The per-target CP reference records it, and an eval refuses a + reference whose recipe differs from its own (see + :mod:`dynacell.evaluation.cp_reference`). + + Parameters + ---------- + cp_feature_version : str or None + :data:`~dynacell.evaluation.metrics.CP_FEATURE_VERSION` for the run. + cp_norm : dict + Resolved ``feature_metrics.cp.norm`` (empty = defaults). + cp_glcm : dict + Resolved ``feature_metrics.cp.glcm`` (empty = GLCM off). + + Returns + ------- + dict + Flat scalar/list identity, stable across YAML/JSON round-trips. + """ + glcm = cp_glcm or {} + norm = cp_norm or {} glcm_enabled = bool(glcm.get("enabled", False)) identity: dict[str, Any] = { - "spacing": ctx.spacing, - "cp_feature_version": ctx.cp_feature_version, + "cp_feature_version": cp_feature_version, "cp_glcm_enabled": glcm_enabled, "cp_norm_p_lo": float(norm.get("p_lo", 1.0)), "cp_norm_p_hi": float(norm.get("p_hi", 99.0)), - **ctx.source_tag, } if glcm_enabled: identity["cp_glcm_levels"] = int(glcm.get("levels", 32)) diff --git a/applications/dynacell/src/dynacell/evaluation/provenance.py b/applications/dynacell/src/dynacell/evaluation/provenance.py index d282bfae8..d0a35a367 100644 --- a/applications/dynacell/src/dynacell/evaluation/provenance.py +++ b/applications/dynacell/src/dynacell/evaluation/provenance.py @@ -28,6 +28,12 @@ * :func:`write_metrics_provenance` stamps the versions beside the metrics, and :func:`metrics_provenance_matches` lets the final-metrics cache gate refuse a cache built by a different ``cubic``. + +The same sidecar stamps the content hash of the CP reference the CP feature +metrics were scored in (:mod:`dynacell.evaluation.cp_reference`). CP KID/FID/ +cosine move whenever the reference is rebuilt, so a cache stamped with another +reference -- or with none, i.e. written before the shared CP space existed -- is +refused the same way. """ import json @@ -83,20 +89,23 @@ def check_cubic_pin() -> None: ) -def write_metrics_provenance(save_dir: Path) -> None: +def write_metrics_provenance(save_dir: Path, *, cp_reference_sha256: str | None) -> None: """Write the numeric-provenance sidecar into ``save_dir``. Parameters ---------- save_dir : pathlib.Path Directory that receives the metric CSV/NPY files. + cp_reference_sha256 : str or None + Content hash of the CP reference the feature metrics were scored in; + ``None`` when the run computed no feature metrics. """ - payload = {"versions": installed_versions()} + payload = {"versions": installed_versions(), "cp_reference_sha256": cp_reference_sha256} (save_dir / PROVENANCE_FILENAME).write_text(json.dumps(payload, indent=1, sort_keys=True) + "\n") -def metrics_provenance_matches(save_dir: Path) -> bool: - """Return True when ``save_dir``'s metrics were built by the running ``cubic``. +def metrics_provenance_matches(save_dir: Path, *, cp_reference_sha256: str | None) -> bool: + """Return True when ``save_dir``'s metrics were built by the running ``cubic`` and CP reference. A missing sidecar returns False. Every cache written before this stamp existed predates the fix and is exactly the ambiguous case that has to be @@ -108,14 +117,22 @@ def metrics_provenance_matches(save_dir: Path) -> bool: ---------- save_dir : pathlib.Path Directory holding the metric CSV/NPY files. + cp_reference_sha256 : str or None + Content hash of the CP reference the current run would score in + (``None`` when it computes no feature metrics). A sidecar written before + the CP reference existed carries no such key and never matches. Returns ------- bool - True when the recorded ``cubic`` version equals the installed one. + True when the recorded ``cubic`` version equals the installed one and + the recorded CP reference hash equals ``cp_reference_sha256``. """ path = save_dir / PROVENANCE_FILENAME if not path.is_file(): return False - recorded = json.loads(path.read_text()).get("versions", {}).get("cubic") - return recorded == version("cubic") + payload = json.loads(path.read_text()) + recorded = payload.get("versions", {}).get("cubic") + if recorded != version("cubic"): + return False + return "cp_reference_sha256" in payload and payload["cp_reference_sha256"] == cp_reference_sha256 diff --git a/applications/dynacell/src/dynacell/evaluation/provenance_test.py b/applications/dynacell/src/dynacell/evaluation/provenance_test.py index 69efedb4c..246fc06c4 100644 --- a/applications/dynacell/src/dynacell/evaluation/provenance_test.py +++ b/applications/dynacell/src/dynacell/evaluation/provenance_test.py @@ -38,10 +38,30 @@ def test_installed_versions_records_cubic(): def test_roundtrip_matches_running_environment(tmp_path): - write_metrics_provenance(tmp_path) + write_metrics_provenance(tmp_path, cp_reference_sha256="abc") payload = json.loads((tmp_path / PROVENANCE_FILENAME).read_text()) assert payload["versions"]["cubic"] == version("cubic") - assert metrics_provenance_matches(tmp_path) + assert payload["cp_reference_sha256"] == "abc" + assert metrics_provenance_matches(tmp_path, cp_reference_sha256="abc") + + +def test_feature_less_stamp_matches_only_a_feature_less_run(tmp_path): + """A run without feature metrics stamps ``None``; a CP-scoring run must not reuse it.""" + write_metrics_provenance(tmp_path, cp_reference_sha256=None) + assert metrics_provenance_matches(tmp_path, cp_reference_sha256=None) + assert not metrics_provenance_matches(tmp_path, cp_reference_sha256="abc") + + +def test_other_cp_reference_is_not_a_match(tmp_path): + """CP values scored in another reference are not reusable after a rebuild.""" + write_metrics_provenance(tmp_path, cp_reference_sha256="old") + assert not metrics_provenance_matches(tmp_path, cp_reference_sha256="new") + + +def test_stamp_predating_cp_reference_is_not_a_match(tmp_path): + """A sidecar written before the CP reference existed carries no hash and never matches.""" + (tmp_path / PROVENANCE_FILENAME).write_text(json.dumps({"versions": {"cubic": version("cubic")}})) + assert not metrics_provenance_matches(tmp_path, cp_reference_sha256=None) def test_missing_sidecar_is_not_a_match(tmp_path): @@ -50,16 +70,16 @@ def test_missing_sidecar_is_not_a_match(tmp_path): Every cache written before the stamp existed is exactly the ambiguous 0.8.0a2-or-0.9.0a1 case that has to be recomputed. """ - assert not metrics_provenance_matches(tmp_path) + assert not metrics_provenance_matches(tmp_path, cp_reference_sha256=None) def test_foreign_cubic_version_is_not_a_match(tmp_path): - write_metrics_provenance(tmp_path) + write_metrics_provenance(tmp_path, cp_reference_sha256=None) path = tmp_path / PROVENANCE_FILENAME payload = json.loads(path.read_text()) payload["versions"]["cubic"] = "0.8.0a2" path.write_text(json.dumps(payload)) - assert not metrics_provenance_matches(tmp_path) + assert not metrics_provenance_matches(tmp_path, cp_reference_sha256=None) def test_check_cubic_pin_accepts_the_declared_version(): diff --git a/applications/dynacell/tests/_eval_fixtures.py b/applications/dynacell/tests/_eval_fixtures.py index da4e4d913..3a190d4ad 100644 --- a/applications/dynacell/tests/_eval_fixtures.py +++ b/applications/dynacell/tests/_eval_fixtures.py @@ -207,6 +207,36 @@ def build_fixture(root: Path): return pred_path, gt_path, gt_cache_dir, pred_cache_dir +def make_cp_reference(config, path: Path, seed: int = 0) -> str: + """Fit a real CP reference matching ``config``'s CP recipe and point ``config`` at it. + + The GT matrix is synthetic but the reference is built by the production + :func:`~dynacell.evaluation.cp_reference.fit_cp_reference`, with the identity and + column names ``evaluate_predictions`` derives from the same config. Sets + ``config.feature_metrics.cp.reference_path`` and returns the reference's sha256. + """ + from dynacell.evaluation.cp_reference import CP_REFERENCE_DIMENSION, fit_cp_reference, write_cp_reference + from dynacell.evaluation.metrics import CP_FEATURE_VERSION, active_cp_feature_names + from dynacell.evaluation.pipeline_cache import cp_recipe_identity + + cp = OmegaConf.select(config, "feature_metrics.cp", default=None) + norm = OmegaConf.to_container(cp.norm, resolve=True) if cp is not None and "norm" in cp else {} + glcm = OmegaConf.to_container(cp.glcm, resolve=True) if cp is not None and "glcm" in cp else {} + names = tuple(active_cp_feature_names(bool(glcm.get("enabled", False)))) + gt = np.random.default_rng(seed).standard_normal((64, len(names))) + payload = fit_cp_reference( + gt, + target_name=config.target_name, + dimension=CP_REFERENCE_DIMENSION, + feature_names=names, + cp_identity=cp_recipe_identity(CP_FEATURE_VERSION, norm, glcm), + datasets=[{"dataset": "synthetic", "n_cells": gt.shape[0]}], + ) + write_cp_reference(payload, path) + OmegaConf.update(config, "feature_metrics.cp.reference_path", str(path), merge=True) + return payload["sha256"] + + def read_position_arrays(plate_path: Path) -> dict[str, np.ndarray]: """Return ``{pos_name: data}`` for every position in an HCS plate.""" out: dict[str, np.ndarray] = {} diff --git a/applications/dynacell/tests/test_evaluation_pipeline.py b/applications/dynacell/tests/test_evaluation_pipeline.py index f2977261c..a700681b0 100644 --- a/applications/dynacell/tests/test_evaluation_pipeline.py +++ b/applications/dynacell/tests/test_evaluation_pipeline.py @@ -44,11 +44,10 @@ def _import_pipeline_with_stubs(monkeypatch): feature_metrics_module.compute_feature_similarity_pairwise = lambda *args, **kwargs: {} feature_select_module = types.ModuleType("dynacell.evaluation.feature_select") - feature_select_module.select_features = lambda gt, pred, **kw: ( - gt, - pred, - np.ones(gt.shape[1] if gt is not None and gt.ndim >= 2 else 0, dtype=bool), - ) + # ``cross_condition_probe`` imports ``select_features``; ``cp_reference`` imports + # ``select_gt_features``. Neither is exercised by these cache-reuse tests. + feature_select_module.select_features = lambda gt, pred, **kw: (gt, pred, np.ones(gt.shape[1], dtype=bool)) + feature_select_module.select_gt_features = lambda gt, **kw: np.ones(gt.shape[1], dtype=bool) feature_select_module.DEFAULT_FREQ_CUT = 0.05 feature_select_module.DEFAULT_UNIQUE_CUT = 0.01 feature_select_module.DEFAULT_CORR_THRESHOLD = 0.9 @@ -119,7 +118,7 @@ def test_evaluate_model_reuses_cache_without_feature_metrics( _write_metrics(tmp_path / config.save.mask_metrics_filename, expected_mask_metrics) # A reusable cache is one this code could have written, which includes the # numeric-provenance stamp save_metrics emits. - write_metrics_provenance(tmp_path) + write_metrics_provenance(tmp_path, cp_reference_sha256=None) def fail_if_recomputed(_config): raise AssertionError("evaluate_predictions should not run when cache is valid") diff --git a/applications/dynacell/tests/test_evaluation_pipeline_parallel.py b/applications/dynacell/tests/test_evaluation_pipeline_parallel.py index 1b4fc9164..c0b8e526d 100644 --- a/applications/dynacell/tests/test_evaluation_pipeline_parallel.py +++ b/applications/dynacell/tests/test_evaluation_pipeline_parallel.py @@ -135,20 +135,34 @@ def test_fov_result_pickle_handles_empty_backbones(): assert restored.celldino.gt_feats == [] -def test_cp_dropzero_zscore_raises_on_dim_mismatch(): - """Mismatched pred/GT CP dims raise an actionable StaleCacheError, not IndexError. +def test_cp_reference_raises_on_dim_mismatch(tmp_path): + """A CP matrix of another width raises an actionable StaleCacheError, not IndexError. A stale CP cache built with a different recipe (e.g. a GLCM toggle or a - CP_FEATURE_VERSION change without a rebuild) used to crash _cp_dropzero_zscore - with a cryptic boolean-index IndexError; it must instead name the remedy. + CP_FEATURE_VERSION change without a rebuild) would otherwise crash the + boolean-mask indexing with a cryptic IndexError; it must instead name the remedy. """ - pipeline = _live_pipeline_module() from dynacell.evaluation.cache import StaleCacheError + from dynacell.evaluation.cp_reference import ( + CP_REFERENCE_DIMENSION, + fit_cp_reference, + load_cp_reference, + write_cp_reference, + ) - pred = np.ones((4, 58), dtype=np.float32) - gt = np.ones((4, 22), dtype=np.float32) + names = tuple(f"f{i}" for i in range(22)) + payload = fit_cp_reference( + np.random.default_rng(0).standard_normal((40, 22)), + target_name="er", + dimension=CP_REFERENCE_DIMENSION, + feature_names=names, + cp_identity={}, + datasets=[{"n_cells": 40}], + ) + write_cp_reference(payload, tmp_path / "ref.json") + ref = load_cp_reference(tmp_path / "ref.json", target_name="er", dimension=CP_REFERENCE_DIMENSION) with pytest.raises(StaleCacheError, match="CP feature dimension mismatch"): - pipeline._cp_dropzero_zscore(pred, gt) + ref.transform(np.ones((4, 58), dtype=np.float32)) def test_aggregate_fov_result_extends_backbone_lists(): diff --git a/applications/dynacell/tests/test_evaluation_timepoint_guard.py b/applications/dynacell/tests/test_evaluation_timepoint_guard.py index 0564a79da..78eef0536 100644 --- a/applications/dynacell/tests/test_evaluation_timepoint_guard.py +++ b/applications/dynacell/tests/test_evaluation_timepoint_guard.py @@ -17,7 +17,7 @@ from dynacell.evaluation.provenance import write_metrics_provenance -from ._eval_fixtures import build_eval_config, live_pipeline_module +from ._eval_fixtures import build_eval_config, live_pipeline_module, make_cp_reference D, H, W = 3, 8, 8 @@ -100,6 +100,7 @@ def _feature_cache_config(tmp_path: Path, **flags: bool): config.compute_feature_metrics = True for name, value in flags.items(): config.feature_metrics[name] = value + make_cp_reference(config, tmp_path / "cp_reference.json") return config @@ -108,7 +109,9 @@ def _write_final_caches(save_dir: Path, feature_row: dict) -> None: np.save(save_dir / "pixel_metrics.npy", [{"SI_PSNR": 1.0, "SI_SSIM": 1.0, "SI_NRMSE": 1.0}]) np.save(save_dir / "mask_metrics.npy", [{"metric": "mask"}]) np.save(save_dir / "feature_metrics.npy", [feature_row]) - write_metrics_provenance(save_dir) + # The stamp must carry the hash of the reference _feature_cache_config points at. + reference_sha256 = make_cp_reference(_feature_cache_config(save_dir), save_dir / "cp_reference.json") + write_metrics_provenance(save_dir, cp_reference_sha256=reference_sha256) def _dataset_row(prefixes: tuple[str, ...], families: tuple[str, ...]) -> dict: @@ -150,3 +153,55 @@ def test_real_full_benchmark_columns_stay_valid(tmp_path: Path) -> None: pipeline = live_pipeline_module() _write_final_caches(tmp_path, dict.fromkeys(header, 0.0)) assert pipeline._final_metrics_cache_valid(_feature_cache_config(tmp_path)) + + +def test_cache_scored_in_another_cp_reference_is_invalid(tmp_path: Path) -> None: + """Rebuilding the CP reference invalidates every final-metrics cache stamped with the old one.""" + pipeline = live_pipeline_module() + _write_final_caches(tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES)) + config = _feature_cache_config(tmp_path) + assert pipeline._final_metrics_cache_valid(config) + + make_cp_reference(config, tmp_path / "cp_reference.json", seed=1) # same path, new content + assert not pipeline._final_metrics_cache_valid(config) + + +def test_missing_cp_reference_fails_before_any_work(tmp_path: Path, monkeypatch) -> None: + """A feature-metrics eval with no CP reference raises, naming the build command, before loading models.""" + pipeline = live_pipeline_module() + config = _feature_cache_config(tmp_path) + config.feature_metrics.cp.reference_path = str(tmp_path / "absent.json") + + def _no_models(*args, **kwargs): + raise AssertionError("models loaded before the CP reference was checked") + + monkeypatch.setattr(pipeline, "load_eval_models", _no_models) + with pytest.raises(FileNotFoundError, match="build_cp_reference.py --target er"): + pipeline.evaluate_predictions(config) + + +def test_cp_reference_of_another_recipe_is_refused(tmp_path: Path, monkeypatch) -> None: + """A reference fit on GLCM-off caches cannot score a GLCM-on eval.""" + pipeline = live_pipeline_module() + # Import after the fresh pipeline import, which re-creates dynacell.evaluation.* modules. + from dynacell.evaluation.cache import StaleCacheError + + config = _feature_cache_config(tmp_path) # reference built for this (GLCM-off) recipe + config.feature_metrics.cp.glcm = {"enabled": True, "levels": 32, "distances": [1]} + monkeypatch.setattr(pipeline, "load_eval_models", lambda *a, **k: pytest.fail("models loaded")) + with pytest.raises(StaleCacheError, match="different CP recipe"): + pipeline.evaluate_predictions(config) + + +def test_save_metrics_stamps_the_cp_reference_hash(tmp_path: Path) -> None: + """``save_metrics`` records the hash of the reference the CP metrics were scored in.""" + import json + + from dynacell.evaluation.provenance import PROVENANCE_FILENAME + + pipeline = live_pipeline_module() + config = _feature_cache_config(tmp_path) + reference_sha256 = make_cp_reference(config, tmp_path / "cp_reference.json") + pipeline.save_metrics(config, pixel_metrics=[{"FOV": "A/1/0", "Timepoint": 0, "PCC": 0.5}]) + stamp = json.loads((tmp_path / PROVENANCE_FILENAME).read_text()) + assert stamp["cp_reference_sha256"] == reference_sha256 diff --git a/applications/dynacell/tests/test_pipeline_cache.py b/applications/dynacell/tests/test_pipeline_cache.py index 7ee12bb67..084c6a473 100644 --- a/applications/dynacell/tests/test_pipeline_cache.py +++ b/applications/dynacell/tests/test_pipeline_cache.py @@ -2128,7 +2128,7 @@ def test_final_metrics_cache_gate_requires_ap_columns(tmp_path: Path) -> None: save_dir = tmp_path / "out" save_dir.mkdir() # Reuse also requires the numeric-provenance stamp save_metrics writes. - write_metrics_provenance(save_dir) + write_metrics_provenance(save_dir, cp_reference_sha256=None) np.save(save_dir / "pixel_metrics.npy", np.array([dict(_DUAL_SCALING_PIXEL_ROW)], dtype=object)) cfg = _make_config( **{ @@ -2167,7 +2167,7 @@ def test_final_metrics_cache_gate_requires_both_pixel_scalings(tmp_path: Path) - save_dir = tmp_path / "out" save_dir.mkdir() - write_metrics_provenance(save_dir) + write_metrics_provenance(save_dir, cp_reference_sha256=None) cfg = _make_config( **{ "compute_instance_ap": False, From ed475732dc780a6553ea1a243aebe9f31bf0a4dc Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 17:45:03 -0700 Subject: [PATCH 03/41] feat(dynacell/tools): build per-target CP references from GT caches build_cp_reference.py fits a target's CP reference on iPSC + A549 mock/denv/zikv GT cells. HEK and the movie sets are excluded. Each dataset ref is resolved as an eval resolves it (apply_dataset_ref on eval.yaml). The GT cache is opened via init_cache_context with require_complete_cache=true, so a cache of another store or CP recipe is refused. CP arrays are read read-only; any missing (position, timepoint) raises, and features are never computed. Non-finite cells are dropped with drop_paired_nonfinite_rows. The JSON records per- dataset cache paths, cell counts and a sha256 of the stacked GT matrix. Real er/3d build (to scratch): 7162 cells from 4 caches, 12/22 features kept, ~15 s wall. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../dynacell/tools/build_cp_reference.py | 266 ++++++++++++++++++ .../dynacell/tools/build_cp_reference_test.py | 142 ++++++++++ 2 files changed, 408 insertions(+) create mode 100644 applications/dynacell/tools/build_cp_reference.py create mode 100644 applications/dynacell/tools/build_cp_reference_test.py diff --git a/applications/dynacell/tools/build_cp_reference.py b/applications/dynacell/tools/build_cp_reference.py new file mode 100644 index 000000000..d3ab6e632 --- /dev/null +++ b/applications/dynacell/tools/build_cp_reference.py @@ -0,0 +1,266 @@ +r"""Build a target's CP (GLCM+) reference from existing GT CP feature caches. + +The reference fixes the CP feature space every eval of a target is scored in: a +feature keep-mask selected on GT cells only, plus one shared per-feature mean/std +over the same pooled GT (see :mod:`dynacell.evaluation.cp_reference`). It is fit +on the target's benchmark test sets -- iPSC + A549 mock/denv/zikv; HEK and the +movie sets are excluded. + +This tool only READS. Each dataset ref is resolved exactly as an eval resolves it +(``apply_dataset_ref`` on the packaged ``eval.yaml`` defaults), the GT cache is +opened through the pipeline's own ``init_cache_context`` with +``require_complete_cache=true`` (so a cache of another dataset or CP recipe is +refused), and every ``(position, timepoint)`` of the GT store must already be in +the CP cache. A miss raises; features are never computed. Non-finite cells are +dropped the way the pipeline drops them (``drop_paired_nonfinite_rows``). + +Run:: + + uv run --no-sync python applications/dynacell/tools/build_cp_reference.py \ + --target er --dry-run + uv run --no-sync python applications/dynacell/tools/build_cp_reference.py \ + --target er # writes the registry path under DATA_ROOT + uv run --no-sync python applications/dynacell/tools/build_cp_reference.py \ + --target er --out /tmp/er__3d.json +""" + +from __future__ import annotations + +import argparse +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +from iohub.ngff import open_ome_zarr +from omegaconf import DictConfig, OmegaConf + +import dynacell.evaluation +from dynacell.evaluation._ref_hook import apply_dataset_ref +from dynacell.evaluation.cache import StaleCacheError, open_features_group, read_features_from_group +from dynacell.evaluation.cp_reference import ( + CP_REFERENCE_DIMENSION, + fit_cp_reference, + write_cp_reference, +) +from dynacell.evaluation.metrics import CP_FEATURE_VERSION, active_cp_feature_names, drop_paired_nonfinite_rows +from dynacell.evaluation.paths import cp_reference_path +from dynacell.evaluation.pipeline_cache import cp_recipe_identity, init_cache_context + +#: The fit set per eval ``target_name``: iPSC + A549 mock/denv/zikv, as +#: ``(dataset, manifest target)`` refs -- the same refs the grouped leaves carry in +#: ``benchmark.dataset_ref``. HEK and the movie sets are deliberately absent. +DEFAULT_DATASETS: dict[str, tuple[tuple[str, str], ...]] = { + "nucleus": ( + ("aics-hipsc", "nucleus"), + ("a549-mantis-h2b-mock", "h2b"), + ("a549-mantis-h2b-denv", "h2b"), + ("a549-mantis-h2b-zikv", "h2b"), + ), + "membrane": ( + ("aics-hipsc", "membrane"), + ("a549-mantis-caax-mock", "caax"), + ("a549-mantis-caax-denv", "caax"), + ("a549-mantis-caax-zikv", "caax"), + ), + "er": ( + ("aics-hipsc", "sec61b"), + ("a549-mantis-sec61b-mock", "sec61b"), + ("a549-mantis-sec61b-denv", "sec61b"), + ("a549-mantis-sec61b-zikv", "sec61b"), + ), + "mitochondria": ( + ("aics-hipsc", "tomm20"), + ("a549-mantis-tomm20-mock", "tomm20"), + ("a549-mantis-tomm20-denv", "tomm20"), + ("a549-mantis-tomm20-zikv", "tomm20"), + ), +} + +_EVAL_YAML = Path(dynacell.evaluation.__file__).parent / "_configs" / "eval.yaml" + + +@dataclass +class DatasetCells: + """Finite GT CP cells read from one dataset's cache, plus what was read.""" + + record: dict[str, Any] + cells: np.ndarray + + +def eval_config_for(target_name: str, dataset: str, manifest_target: str) -> DictConfig: + """Return the packaged eval defaults resolved for one dataset ref, cache reads mandatory.""" + config = OmegaConf.load(_EVAL_YAML) + config.target_name = target_name + config.benchmark = {"dataset_ref": {"dataset": dataset, "target": manifest_target}} + config.io.require_complete_cache = True + apply_dataset_ref(config) + return config + + +def cp_identity_for(config: DictConfig) -> tuple[dict[str, Any], tuple[str, ...]]: + """Return the CP recipe identity and column names an eval of ``config`` would use.""" + norm = OmegaConf.to_container(config.feature_metrics.cp.norm, resolve=True) + glcm = OmegaConf.to_container(config.feature_metrics.cp.glcm, resolve=True) + return ( + cp_recipe_identity(CP_FEATURE_VERSION, norm, glcm), + tuple(active_cp_feature_names(bool(glcm["enabled"]))), + ) + + +def read_dataset_cells(config: DictConfig, n_features: int) -> DatasetCells: + """Read every GT CP cell of one dataset from its cache, read-only. + + Parameters + ---------- + config : DictConfig + Output of :func:`eval_config_for`. + n_features : int + Expected CP column count. + + Returns + ------- + DatasetCells + Finite cells stacked in position/timepoint order, and the provenance record. + + Raises + ------ + StaleCacheError + If the cache belongs to another store or CP recipe, a ``(position, + timepoint)`` of the GT store is missing from it, or an entry has the wrong + column count. + ValueError + If a GT position is not a 3-D volume. + """ + ctx = init_cache_context(config, side="gt") + blocks: list[np.ndarray] = [] + n_positions = n_timepoints = n_dropped = 0 + with ( + open_ome_zarr(Path(config.io.gt_path), mode="r") as plate, + open_features_group(ctx.paths, "cp", mode="r") as group, + ): + if group is None: + raise StaleCacheError(f"no CP feature cache at {ctx.paths.cp_features()}") + for pos_name, pos in plate.positions(): + t_count, _, z_depth = pos.data.shape[:3] + if z_depth < 2: + raise ValueError(f"{config.io.gt_path}/{pos_name} has Z={z_depth}; the CP reference is 3-D") + n_positions += 1 + for t in range(t_count): + feats = read_features_from_group(group, pos_name, t) + if feats is None: + raise StaleCacheError(f"CP cache miss at {pos_name}/t{t} in {ctx.paths.cp_features()}") + n_timepoints += 1 + if feats.shape[0] == 0: + continue + if feats.shape[1] != n_features: + raise StaleCacheError(f"{pos_name}/t{t} has {feats.shape[1]} CP columns, expected {n_features}") + finite, _ = drop_paired_nonfinite_rows(feats, feats) + n_dropped += feats.shape[0] - finite.shape[0] + blocks.append(np.asarray(finite, dtype=np.float64)) + cells = np.concatenate(blocks, axis=0) if blocks else np.empty((0, n_features)) + record = { + "dataset": config.benchmark.dataset_ref.dataset, + "target": config.benchmark.dataset_ref.target, + "gt_path": str(config.io.gt_path), + "gt_cache_dir": str(ctx.paths.root), + "cp_cache_path": str(ctx.paths.cp_features()), + "n_positions": n_positions, + "n_timepoints": n_timepoints, + "n_cells": int(cells.shape[0]), + "n_cells_dropped_nonfinite": int(n_dropped), + } + return DatasetCells(record=record, cells=cells) + + +def build( + target_name: str, datasets: list[tuple[str, str]], dimension: str +) -> tuple[list[DatasetCells], dict[str, Any], tuple[str, ...]]: + """Read every fit dataset and check they share one CP recipe. + + Returns + ------- + tuple + ``(per-dataset cells, cp_identity, feature_names)``. + + Raises + ------ + ValueError + If ``dimension`` is not :data:`CP_REFERENCE_DIMENSION`, or the datasets + disagree on the CP recipe. + """ + if dimension != CP_REFERENCE_DIMENSION: + raise ValueError(f"only the {CP_REFERENCE_DIMENSION!r} CP reference exists; got {dimension!r}") + identity: dict[str, Any] | None = None + names: tuple[str, ...] = () + read: list[DatasetCells] = [] + for dataset, manifest_target in datasets: + config = eval_config_for(target_name, dataset, manifest_target) + ds_identity, ds_names = cp_identity_for(config) + if identity is None: + identity, names = ds_identity, ds_names + elif (ds_identity, ds_names) != (identity, names): + raise ValueError(f"{dataset} resolves a different CP recipe: {ds_identity} vs {identity}") + read.append(read_dataset_cells(config, len(names))) + if identity is None: + raise ValueError("no datasets given") + return read, identity, names + + +def _parse_dataset(token: str) -> tuple[str, str]: + """Parse a ``dataset:target`` CLI token.""" + dataset, sep, target = token.partition(":") + if not sep or not dataset or not target: + raise argparse.ArgumentTypeError(f"expected DATASET:TARGET, got {token!r}") + return dataset, target + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point.""" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--target", required=True, choices=sorted(DEFAULT_DATASETS)) + parser.add_argument("--dimension", default=CP_REFERENCE_DIMENSION, choices=[CP_REFERENCE_DIMENSION]) + parser.add_argument( + "--datasets", + nargs="+", + type=_parse_dataset, + default=None, + metavar="DATASET:TARGET", + help="fit datasets as manifest refs (default: the target's iPSC + A549 mock/denv/zikv)", + ) + parser.add_argument("--out", type=Path, default=None, help="output JSON (default: the registry path)") + parser.add_argument("--dry-run", action="store_true", help="read and count cells; write nothing") + args = parser.parse_args(argv) + + datasets = args.datasets if args.datasets is not None else list(DEFAULT_DATASETS[args.target]) + out = args.out if args.out is not None else cp_reference_path(args.target, args.dimension) + start = time.perf_counter() + read, identity, names = build(args.target, datasets, args.dimension) + for ds in read: + r = ds.record + print( + f"{r['dataset']}:{r['target']} {r['n_positions']} positions, {r['n_timepoints']} timepoints, " + f"{r['n_cells']} cells ({r['n_cells_dropped_nonfinite']} non-finite dropped)\n {r['cp_cache_path']}" + ) + print(f"total {sum(ds.record['n_cells'] for ds in read)} cells, {len(names)} CP features, identity {identity}") + if args.dry_run: + print(f"dry run: would write {out} ({time.perf_counter() - start:.1f} s)") + return + payload = fit_cp_reference( + np.concatenate([ds.cells for ds in read], axis=0), + target_name=args.target, + dimension=args.dimension, + feature_names=names, + cp_identity=identity, + datasets=[ds.record for ds in read], + ) + write_cp_reference(payload, out) + print( + f"kept {len(payload['kept_feature_names'])}/{len(names)}: {payload['kept_feature_names']}\n" + f"wrote {out} sha256={payload['sha256']} ({time.perf_counter() - start:.1f} s)" + ) + + +if __name__ == "__main__": + main() diff --git a/applications/dynacell/tools/build_cp_reference_test.py b/applications/dynacell/tools/build_cp_reference_test.py new file mode 100644 index 000000000..26b4c0eb9 --- /dev/null +++ b/applications/dynacell/tools/build_cp_reference_test.py @@ -0,0 +1,142 @@ +"""Integration tests for ``build_cp_reference``. + +Two synthetic datasets are registered as real manifests (``DYNACELL_MANIFEST_ROOTS``), +each with a real HCS GT store and a real GT CP feature cache. The tool resolves them +through ``apply_dataset_ref`` + ``init_cache_context`` exactly as an eval would. + +Run:: + + uv run --no-sync pytest applications/dynacell/tools/build_cp_reference_test.py -q +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest +import yaml +from build_cp_reference import main +from iohub.ngff import open_ome_zarr + +from dynacell.evaluation.cache import ( + StaleCacheError, + cache_paths, + open_features_group, + save_manifest, + write_features_to_group, +) +from dynacell.evaluation.cp_reference import CP_REFERENCE_DIMENSION, load_cp_reference +from dynacell.evaluation.metrics import active_cp_feature_names + +_N_FEATURES = len(active_cp_feature_names(True)) # eval.yaml ships GLCM on +_POSITIONS = ("A/1/0", "A/1/1") +_T = 2 + + +def _dataset(root: Path, name: str, seed: int, *, z: int = 3, skip: tuple[str, int] | None = None) -> None: + """Register one dataset: manifest, GT store, and a GT CP cache holding every (pos, t).""" + store = root / "stores" / f"{name}.zarr" + cache_dir = root / "caches" / name + with open_ome_zarr(store, mode="w", layout="hcs", channel_names=["Structure"], version="0.5") as plate: + for pos_name in _POSITIONS: + row, col, fov = pos_name.split("/") + plate.create_position(row, col, fov).create_image("0", np.zeros((_T, 1, z, 4, 4), dtype=np.float32)) + rng = np.random.default_rng(seed) + with open_features_group(cache_paths(cache_dir), "cp", mode="a") as group: + for pos_name in _POSITIONS: + for t in range(_T): + if (pos_name, t) == skip: + continue + feats = rng.standard_normal((5, _N_FEATURES)) + seed + if (pos_name, t) == ("A/1/0", 0): + feats[0, 3] = np.nan # the pipeline drops non-finite cells + write_features_to_group(group, pos_name, t, feats) + manifest = { + "name": name, + "version": "1", + "description": "synthetic", + "cell_type": "HeLa", + "imaging_modality": "confocal", + "spacing": {"z": 0.3, "y": 0.1, "x": 0.1}, + "channels": {"source": "Phase3D"}, + "targets": { + "sec61b": { + "gene": "SEC61B", + "organelle": "er", + "display_name": "ER", + "target_channel": "Structure", + "stores": {"train": str(store), "test": str(store), "gt_cache_dir": str(cache_dir)}, + "splits": "splits/none.yaml", + } + }, + } + (root / "manifests" / name).mkdir(parents=True) + (root / "manifests" / name / "manifest.yaml").write_text(yaml.dump(manifest)) + + +@pytest.fixture +def registry(tmp_path: Path, monkeypatch) -> Path: + """Two registered synthetic datasets; returns the tmp root.""" + monkeypatch.setenv("DYNACELL_MANIFEST_ROOTS", str(tmp_path / "manifests")) + return tmp_path + + +_DATASETS = ["--datasets", "ds-a:sec61b", "ds-b:sec61b"] + + +def test_build_writes_a_loadable_reference(registry: Path) -> None: + """A real build records per-dataset cell counts and loads back as a verified reference.""" + _dataset(registry, "ds-a", seed=0) + _dataset(registry, "ds-b", seed=1) + out = registry / "er__3d.json" + + main(["--target", "er", *_DATASETS, "--out", str(out)]) + + payload = json.loads(out.read_text()) + fit = {d["dataset"]: d for d in payload["fit"]["datasets"]} + assert fit["ds-a"]["n_cells"] == 2 * _T * 5 - 1 # one non-finite cell dropped + assert fit["ds-a"]["n_cells_dropped_nonfinite"] == 1 + assert fit["ds-b"]["n_cells"] == 2 * _T * 5 - 1 + assert payload["fit"]["n_cells"] == 38 + assert fit["ds-a"]["cp_cache_path"].endswith("caches/ds-a/features/cp.zarr") + assert payload["cp_identity"]["cp_glcm_enabled"] is True + ref = load_cp_reference(out, target_name="er", dimension=CP_REFERENCE_DIMENSION) + assert ref.keep_mask.shape == (_N_FEATURES,) + + +def test_dry_run_writes_nothing(registry: Path, capsys) -> None: + """``--dry-run`` reads and counts every cell but writes no file.""" + _dataset(registry, "ds-a", seed=0) + _dataset(registry, "ds-b", seed=1) + out = registry / "er__3d.json" + main(["--target", "er", *_DATASETS, "--out", str(out), "--dry-run"]) + assert not out.exists() + assert "total 38 cells" in capsys.readouterr().out + + +def test_missing_timepoint_raises(registry: Path) -> None: + """A GT (position, timepoint) absent from the CP cache fails instead of being skipped.""" + _dataset(registry, "ds-a", seed=0) + _dataset(registry, "ds-b", seed=1, skip=("A/1/1", 1)) + with pytest.raises(StaleCacheError, match="CP cache miss at A/1/1/t1"): + main(["--target", "er", *_DATASETS, "--out", str(registry / "x.json")]) + + +def test_cache_of_another_recipe_is_refused(registry: Path) -> None: + """A GT cache stamped with another CP recipe is refused by the pipeline's own identity check.""" + _dataset(registry, "ds-a", seed=0) + save_manifest( + cache_paths(registry / "caches" / "ds-a"), + {"artifacts": {"cp_features": {"path": "features/cp.zarr", "cp_feature_version": "v1_old"}}}, + ) + with pytest.raises(StaleCacheError, match="cp_feature_version"): + main(["--target", "er", "--datasets", "ds-a:sec61b", "--out", str(registry / "x.json")]) + + +def test_2d_store_is_refused(registry: Path) -> None: + """A single-plane GT store cannot feed the 3-D reference.""" + _dataset(registry, "ds-a", seed=0, z=1) + with pytest.raises(ValueError, match="the CP reference is 3-D"): + main(["--target", "er", "--datasets", "ds-a:sec61b", "--out", str(registry / "x.json")]) From e9eed9403b70571c4d4386fe0f3f9310ad7e0114 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 18:29:53 -0700 Subject: [PATCH 04/41] fix(dynacell/eval): keep feature-less caches reusable without a CP hash metrics_provenance_matches required a cp_reference_sha256 key in every stamp, so a compute_feature_metrics=false dir -- which scores no CP value -- was invalidated just for predating the CP reference. The key is now required only when the caller passes a hash, i.e. when the config computes feature metrics. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../dynacell/src/dynacell/evaluation/provenance.py | 14 +++++++++----- .../src/dynacell/evaluation/provenance_test.py | 12 ++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/provenance.py b/applications/dynacell/src/dynacell/evaluation/provenance.py index d0a35a367..4cd03c539 100644 --- a/applications/dynacell/src/dynacell/evaluation/provenance.py +++ b/applications/dynacell/src/dynacell/evaluation/provenance.py @@ -118,15 +118,17 @@ def metrics_provenance_matches(save_dir: Path, *, cp_reference_sha256: str | Non save_dir : pathlib.Path Directory holding the metric CSV/NPY files. cp_reference_sha256 : str or None - Content hash of the CP reference the current run would score in - (``None`` when it computes no feature metrics). A sidecar written before - the CP reference existed carries no such key and never matches. + Content hash of the CP reference the current run would score in, or + ``None`` when it computes no feature metrics -- then no CP value is + reused and the recorded hash (or its absence) is irrelevant. When given, + a sidecar written before the CP reference existed carries no such key + and never matches. Returns ------- bool - True when the recorded ``cubic`` version equals the installed one and - the recorded CP reference hash equals ``cp_reference_sha256``. + True when the recorded ``cubic`` version equals the installed one and, + if ``cp_reference_sha256`` is given, the recorded hash equals it. """ path = save_dir / PROVENANCE_FILENAME if not path.is_file(): @@ -135,4 +137,6 @@ def metrics_provenance_matches(save_dir: Path, *, cp_reference_sha256: str | Non recorded = payload.get("versions", {}).get("cubic") if recorded != version("cubic"): return False + if cp_reference_sha256 is None: + return True return "cp_reference_sha256" in payload and payload["cp_reference_sha256"] == cp_reference_sha256 diff --git a/applications/dynacell/src/dynacell/evaluation/provenance_test.py b/applications/dynacell/src/dynacell/evaluation/provenance_test.py index 246fc06c4..229837832 100644 --- a/applications/dynacell/src/dynacell/evaluation/provenance_test.py +++ b/applications/dynacell/src/dynacell/evaluation/provenance_test.py @@ -52,6 +52,14 @@ def test_feature_less_stamp_matches_only_a_feature_less_run(tmp_path): assert not metrics_provenance_matches(tmp_path, cp_reference_sha256="abc") +def test_feature_less_run_ignores_the_cp_key(tmp_path): + """compute_feature_metrics=false reuses no CP value, so a missing or foreign hash is irrelevant.""" + (tmp_path / PROVENANCE_FILENAME).write_text(json.dumps({"versions": {"cubic": version("cubic")}})) + assert metrics_provenance_matches(tmp_path, cp_reference_sha256=None) + write_metrics_provenance(tmp_path, cp_reference_sha256="abc") + assert metrics_provenance_matches(tmp_path, cp_reference_sha256=None) + + def test_other_cp_reference_is_not_a_match(tmp_path): """CP values scored in another reference are not reusable after a rebuild.""" write_metrics_provenance(tmp_path, cp_reference_sha256="old") @@ -59,9 +67,9 @@ def test_other_cp_reference_is_not_a_match(tmp_path): def test_stamp_predating_cp_reference_is_not_a_match(tmp_path): - """A sidecar written before the CP reference existed carries no hash and never matches.""" + """A sidecar written before the CP reference existed carries no hash; a CP-scoring run refuses it.""" (tmp_path / PROVENANCE_FILENAME).write_text(json.dumps({"versions": {"cubic": version("cubic")}})) - assert not metrics_provenance_matches(tmp_path, cp_reference_sha256=None) + assert not metrics_provenance_matches(tmp_path, cp_reference_sha256="abc") def test_missing_sidecar_is_not_a_match(tmp_path): From 0940b5eaa2195389c58d9050f50918088a025177 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 18:30:06 -0700 Subject: [PATCH 05/41] fix(dynacell/eval): scale CP per test set and pin each scaler to its GT cells The pooled GT scaler squeezed within-dataset spread: between-dataset offset dominates the pooled variance (eta^2 up to 0.91, measured by the reviewer), leaving 0.02-0.13 sd on kept features. The reference (schema 2) keeps one GT-only mask pooled over iPSC + A549 mock/denv/zikv. It now holds one GT mean/std per test set, keyed by benchmark.dataset_ref.dataset, applied to pred and GT alike. - The builder fits a scaler for every dataset a grouped leaf of the target evaluates, HEK included; HEK stays out of the mask fit. Lite datasets reuse the parent named by their split's selection_criteria.lite_of and are never refit. - A dataset std below 1e-3 x the pooled GT std is floored and the floored features are recorded; a zero pooled std is still refused. - Each scaler records its positions, n_cells, GT-matrix sha256 and GT cache cp_features.built_at. An eval refuses a GT re-cache up front, a partial walk (limit_positions / exclude_fov_names) on a non-lite set, and a GT CP cell count that differs from the fit. - The caller loads the bound space once and threads it to evaluate_predictions and save_metrics, so the stamp is the reference the run scored with, never a re-read. - Atomic write (tmp + os.replace). A different existing reference is replaced only with --force; an identical hash is a no-op. - One cp_space(config) derives identity and names for eval and builder, reading the config keys directly. The dimension parameter is gone; the registry path is DATA_ROOT/cp_reference/.json. - Per-row CP goes through _cp_row_features. Tests assert the staged and per-row arrays equal the dataset-scaler transform (a per-side mutation fails them), and cover the _worker_run_fov handoff. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/README.md | 2 +- .../dynacell/evaluation/_configs/eval.yaml | 8 +- .../src/dynacell/evaluation/cp_reference.py | 519 ++++++++++++++---- .../dynacell/evaluation/cp_reference_test.py | 322 ++++++++--- .../evaluation/instance_metrics_test.py | 2 +- .../dynacell/src/dynacell/evaluation/paths.py | 25 +- .../src/dynacell/evaluation/pipeline.py | 176 +++--- applications/dynacell/tests/_eval_fixtures.py | 61 +- .../dynacell/tests/test_evaluate_compose.py | 5 +- .../test_evaluation_pipeline_parallel.py | 86 +-- .../tests/test_evaluation_timepoint_guard.py | 184 +++++-- .../dynacell/tools/build_cp_reference.py | 327 +++++------ .../dynacell/tools/build_cp_reference_test.py | 200 +++++-- 13 files changed, 1328 insertions(+), 589 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/README.md b/applications/dynacell/src/dynacell/evaluation/README.md index d0c966f10..4b3e2359d 100644 --- a/applications/dynacell/src/dynacell/evaluation/README.md +++ b/applications/dynacell/src/dynacell/evaluation/README.md @@ -24,7 +24,7 @@ End-to-end evaluation pipeline for virtual staining predictions against fluoresc - `io.gt_path` — fluorescence ground truth (channel: `io.gt_channel_name`) - `io.cell_segmentation_path` — *optional* precomputed cell segmentation HCS OME-Zarr. Required when `compute_feature_metrics=true` or when building CP/DINOv3/DynaCLR/CELL-DINO cache entries. Position layout must match GT/pred 1:1. - `io.gt_cache_dir`, `io.pred_cache_dir` — *optional* artifact cache directories; must be distinct. See [Caches](#caches). -- `feature_metrics.cp.reference_path` — the target's CP (GLCM+) reference, required when `compute_feature_metrics=true`. `null` (default) resolves to `DATA_ROOT/cp_reference/__3d.json` for full and lite evals alike. The reference holds a feature mask chosen on GT cells only, pooled over iPSC + A549 mock/denv/zikv, and one shared GT mean/std. Every eval applies both to pred AND GT, so all models are scored in one CP space. Build it from existing GT caches with `tools/build_cp_reference.py --target `. Its hash is stamped in `metrics_provenance.json`, and a rebuilt reference invalidates the final-metrics cache. +- `feature_metrics.cp.reference_path` — the target's CP (GLCM+) reference, required when `compute_feature_metrics=true`. `null` (default) resolves to `DATA_ROOT/cp_reference/.json` for full and lite evals alike. The reference holds one feature mask chosen on GT cells only, pooled over iPSC + A549 mock/denv/zikv, and one GT mean/std per test set (keyed by `benchmark.dataset_ref.dataset`; HEK gets its own, lite reuses its parent's). Every eval applies the mask and its dataset's scaler to pred AND GT. The eval refuses a partial position walk, a GT cell count or GT-cache `built_at` that differs from the fit. Build it from existing GT caches with `tools/build_cp_reference.py --target `. Its hash is stamped in `metrics_provenance.json`, and a rebuilt reference invalidates the final-metrics cache. ## Quick start diff --git a/applications/dynacell/src/dynacell/evaluation/_configs/eval.yaml b/applications/dynacell/src/dynacell/evaluation/_configs/eval.yaml index d327084ff..ba9d4cc7c 100644 --- a/applications/dynacell/src/dynacell/evaluation/_configs/eval.yaml +++ b/applications/dynacell/src/dynacell/evaluation/_configs/eval.yaml @@ -141,9 +141,11 @@ feature_metrics: enabled: true levels: 32 distances: [1] - # Per-target CP reference: GT-only feature mask + one shared mean/std, applied to - # BOTH pred and GT so every model is scored in the same CP space. null -> registry - # default DATA_ROOT/cp_reference/__3d.json (lite evals use it too). + # Per-target CP reference: one GT-only feature mask (pooled over iPSC + A549 + # mock/denv/zikv) + one GT mean/std per test set (lite -> its parent's), applied to + # BOTH pred and GT so every model is scored in the same CP space. Keyed by + # benchmark.dataset_ref.dataset. null -> registry default + # DATA_ROOT/cp_reference/.json (lite evals use it too). # Build with applications/dynacell/tools/build_cp_reference.py --target . reference_path: null # In-focus slab for the 2D deep-feature crops + per-cell similarity. When diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index d101bc3f0..2088ed0d6 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -1,4 +1,4 @@ -"""Per-target CP (GLCM+) reference: one shared feature space for every eval of a target. +"""Per-target CP (GLCM+) reference: one feature mask, one GT scaler per test set. The CP track used to be scored in a model-dependent space. Each eval dir pooled its own GT and prediction cells to choose a feature subset, then z-scored the @@ -7,29 +7,45 @@ between prediction and GT, which is the signature of over-smoothing, so that error could not register in CP KID/FID/cosine at all. -A CP reference fixes both. It is fit once per target on **GT cells only**, -pooled across the target's benchmark test sets (iPSC + A549 mock/denv/zikv; -HEK and the movie sets are excluded from the fit). It holds: - -* a feature keep-mask chosen by :func:`~dynacell.evaluation.feature_select.select_gt_features` - with the pipeline's criteria constants, and -* one per-feature mean/std over the kept features of that same pooled GT. - -Every eval of the target applies the same mask and the same scaler to BOTH -prediction and GT, at dataset level and per row. The artifact is JSON and is -built by ``applications/dynacell/tools/build_cp_reference.py`` from existing GT -CP caches. - -CP regionprops are always computed on the full GT/prediction volume. The -2D-track evals score 3D prediction stores, and the deep-feature focus slab never -touches CP. So every current eval runs in the ``3d`` CP space, and -:data:`CP_REFERENCE_DIMENSION` is the only dimension the registry holds. +A CP reference fixes both. Per target it holds: + +* **One feature keep-mask**, chosen by + :func:`~dynacell.evaluation.feature_select.select_gt_features` on **GT cells + only**, pooled over the target's :data:`MASK_FIT_DATASETS` (iPSC + A549 + mock/denv/zikv). HEK and every lite dataset stay out of the mask fit. +* **One scaler per test set** (keyed by ``benchmark.dataset_ref.dataset``): the + mean/std of that test set's GT cells on the kept features. The scaler is + applied identically to prediction and GT. It is per test set, not pooled, + because the between-dataset offset dominates the pooled variance (eta^2 up + to 0.91). A pooled scaler would squeeze the within-dataset spread to 0.02-0.13 + sd on the kept features. +* **Lite datasets reuse their parent's scaler** (the parent is read from the lite + split's ``selection_criteria.lite_of``); no scaler is ever fit on lite cells. + +Each scaler is fit on exactly the GT cells an eval of that dataset scores: every +position and timepoint of the GT store, read from the GT CP cache, with the +non-finite rows dropped. The reference records each dataset's position list, cell +count, GT-matrix hash and GT-cache ``cp_features.built_at``, and an eval refuses +to score if its GT side differs (:meth:`DatasetCPSpace.check_gt_cache`, +:meth:`DatasetCPSpace.check_positions`, :meth:`DatasetCPSpace.check_n_cells`). + +A per-dataset std can collapse on a feature the pooled mask keeps (a feature +that is near-constant within one test set). Such a std is floored at +:data:`STD_FLOOR_FRACTION` of that feature's pooled GT std, and the floored +features are recorded per dataset. + +CP regionprops are always computed on the full GT/prediction volume, so every +eval, the 2D-track ones included, scores in this one 3-D space. + +The artifact is JSON and is built by ``applications/dynacell/tools/build_cp_reference.py`` +from existing GT CP caches. """ from __future__ import annotations import hashlib import json +import os from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path @@ -38,20 +54,52 @@ import numpy as np from omegaconf import DictConfig, OmegaConf -from dynacell.evaluation.cache import StaleCacheError +from dynacell.evaluation.cache import StaleCacheError, cache_paths, load_manifest from dynacell.evaluation.feature_select import ( DEFAULT_CORR_THRESHOLD, DEFAULT_FREQ_CUT, DEFAULT_UNIQUE_CUT, select_gt_features, ) +from dynacell.evaluation.metrics import CP_FEATURE_VERSION, active_cp_feature_names from dynacell.evaluation.paths import cp_reference_path +from dynacell.evaluation.pipeline_cache import cp_recipe_identity #: Schema of the reference JSON. Bump on any change to its keys or their meaning. -CP_REFERENCE_SCHEMA = 1 - -#: The CP feature space every current eval runs in (CP regionprops are 3D). -CP_REFERENCE_DIMENSION = "3d" +CP_REFERENCE_SCHEMA = 2 + +#: A per-dataset std below this fraction of the feature's pooled GT std is floored to it. +STD_FLOOR_FRACTION = 1e-3 + +#: The mask-fit set per eval ``target_name``, as ``(dataset, manifest target)`` refs -- +#: the refs the grouped leaves carry in ``benchmark.dataset_ref``. Its keys are the +#: targets that have a CP reference. HEK and the lite datasets are deliberately absent. +MASK_FIT_DATASETS: dict[str, tuple[tuple[str, str], ...]] = { + "nucleus": ( + ("aics-hipsc", "nucleus"), + ("a549-mantis-h2b-mock", "h2b"), + ("a549-mantis-h2b-denv", "h2b"), + ("a549-mantis-h2b-zikv", "h2b"), + ), + "membrane": ( + ("aics-hipsc", "membrane"), + ("a549-mantis-caax-mock", "caax"), + ("a549-mantis-caax-denv", "caax"), + ("a549-mantis-caax-zikv", "caax"), + ), + "er": ( + ("aics-hipsc", "sec61b"), + ("a549-mantis-sec61b-mock", "sec61b"), + ("a549-mantis-sec61b-denv", "sec61b"), + ("a549-mantis-sec61b-zikv", "sec61b"), + ), + "mitochondria": ( + ("aics-hipsc", "tomm20"), + ("a549-mantis-tomm20-mock", "tomm20"), + ("a549-mantis-tomm20-denv", "tomm20"), + ("a549-mantis-tomm20-zikv", "tomm20"), + ), +} #: Keys excluded from the content hash: the hash itself, and the build timestamp, #: so rebuilding from identical GT caches yields an identical hash and does not @@ -59,12 +107,32 @@ _UNHASHED_KEYS = frozenset({"sha256", "created_at"}) -def _build_command(target_name: str, dimension: str) -> str: +def _build_command(target_name: str) -> str: """Return the command that builds the registry reference for one target.""" - return ( - "uv run --no-sync python applications/dynacell/tools/build_cp_reference.py " - f"--target {target_name} --dimension {dimension}" - ) + return f"uv run --no-sync python applications/dynacell/tools/build_cp_reference.py --target {target_name}" + + +def cp_space(config: DictConfig) -> tuple[dict[str, Any], tuple[str, ...]]: + """Return the CP recipe identity and column names an eval of ``config`` produces. + + The one derivation shared by the eval (to check a reference) and the builder + (to stamp one). Reads ``feature_metrics.cp.norm`` / ``feature_metrics.cp.glcm`` + directly, so a config missing them raises instead of defaulting. + + Parameters + ---------- + config : DictConfig + Eval config. + + Returns + ------- + tuple + ``(cp_identity, feature_names)``. + """ + cp = config.feature_metrics.cp + norm = OmegaConf.to_container(cp.norm, resolve=True) + glcm = OmegaConf.to_container(cp.glcm, resolve=True) + return cp_recipe_identity(CP_FEATURE_VERSION, norm, glcm), tuple(active_cp_feature_names(bool(cp.glcm.enabled))) def payload_sha256(payload: dict[str, Any]) -> str: @@ -87,8 +155,8 @@ def payload_sha256(payload: dict[str, Any]) -> str: def gt_matrix_sha256(gt: np.ndarray) -> str: """Return a sha256 over a stacked GT cell matrix (shape + float64 C-order bytes). - Recorded in the reference so a later GT re-cache that changes any fitted - value, or the cell count, is detectable against the artifact. + Recorded per dataset so a later GT re-cache that changes any fitted value, or + the cell count, is detectable against the artifact. """ arr = np.ascontiguousarray(gt, dtype=np.float64) digest = hashlib.sha256(f"{arr.shape[0]}x{arr.shape[1]}:".encode()) @@ -96,33 +164,53 @@ def gt_matrix_sha256(gt: np.ndarray) -> str: return digest.hexdigest() +@dataclass +class DatasetFit: + """One test set's GT cells and the record of where they came from. + + Parameters + ---------- + dataset : str + ``benchmark.dataset_ref.dataset`` of the test set. + cells : np.ndarray + ``(n_cells, n_features)`` finite raw GT CP cells, every position and timepoint. + record : dict + JSON-serializable provenance: at least ``positions`` (sorted names), + ``gt_cache_dir`` and ``cp_cache_built_at``. + in_mask_fit : bool + Whether these cells enter the pooled mask fit. + """ + + dataset: str + cells: np.ndarray + record: dict[str, Any] + in_mask_fit: bool + + def fit_cp_reference( - gt: np.ndarray, + fits: list[DatasetFit], *, target_name: str, - dimension: str, feature_names: tuple[str, ...], cp_identity: dict[str, Any], - datasets: list[dict[str, Any]], + lite: dict[str, dict[str, Any]], ) -> dict[str, Any]: - """Fit a CP reference on pooled GT cells and return its JSON payload. + """Fit a CP reference and return its JSON payload. Parameters ---------- - gt : np.ndarray - Shape ``(n_cells, n_features)``: every finite GT cell of every fit - dataset, stacked in ``datasets`` order. + fits : list of DatasetFit + Every non-lite test set that needs a scaler. Those with ``in_mask_fit`` + are pooled for the feature mask and the floor's pooled std. target_name : str Eval ``target_name`` (``nucleus``/``membrane``/``er``/``mitochondria``). - dimension : str - CP feature-space dimension, :data:`CP_REFERENCE_DIMENSION`. feature_names : tuple of str - Column names of ``gt``, from ``active_cp_feature_names``. + Column names of every ``cells`` matrix. cp_identity : dict - Dataset-independent CP recipe identity - (:func:`~dynacell.evaluation.pipeline_cache.cp_recipe_identity`). - datasets : list of dict - Per-dataset fit inputs (dataset ref, GT cache paths read, ``n_cells``). + Dataset-independent CP recipe identity (from :func:`cp_space`). + lite : dict + ``{lite dataset: {"parent": ..., "gt_cache_dir": ..., "cp_cache_built_at": ...}}``. + Each parent must be one of ``fits``. Returns ------- @@ -132,83 +220,149 @@ def fit_cp_reference( Raises ------ ValueError - If ``gt`` does not match ``feature_names``, holds a non-finite value, or - selection keeps a zero-variance feature. + If a matrix does not match ``feature_names`` or holds a non-finite value, + a dataset has no cells, a dataset appears twice, a lite parent has no + scaler, no dataset is in the mask fit, or the pooled fit keeps a + zero-variance feature. """ - if gt.ndim != 2 or gt.shape[1] != len(feature_names): - raise ValueError(f"GT matrix shape {gt.shape} does not match {len(feature_names)} CP feature names") - if not np.isfinite(gt).all(): - raise ValueError("GT matrix holds non-finite values; drop them before fitting") - n_cells = sum(int(d["n_cells"]) for d in datasets) - if n_cells != gt.shape[0]: - raise ValueError(f"datasets report {n_cells} cells but the GT matrix has {gt.shape[0]} rows") - - gt = np.asarray(gt, dtype=np.float64) + names = [f.dataset for f in fits] + if len(set(names)) != len(names): + raise ValueError(f"duplicate datasets: {names}") + for fit in fits: + if fit.cells.ndim != 2 or fit.cells.shape[1] != len(feature_names): + raise ValueError(f"{fit.dataset}: GT matrix shape {fit.cells.shape} does not match {len(feature_names)}") + if fit.cells.shape[0] == 0: + raise ValueError(f"{fit.dataset}: no GT cells") + if not np.isfinite(fit.cells).all(): + raise ValueError(f"{fit.dataset}: GT matrix holds non-finite values; drop them before fitting") + for lite_name, entry in lite.items(): + if entry["parent"] not in names: + raise ValueError(f"lite dataset {lite_name} reuses {entry['parent']}, which has no scaler") + + pooled = np.concatenate([np.asarray(f.cells, dtype=np.float64) for f in fits if f.in_mask_fit], axis=0) + if pooled.shape[0] == 0: + raise ValueError("no dataset is in the mask fit") keep_mask = select_gt_features( - gt, freq_cut=DEFAULT_FREQ_CUT, unique_cut=DEFAULT_UNIQUE_CUT, corr_threshold=DEFAULT_CORR_THRESHOLD + pooled, freq_cut=DEFAULT_FREQ_CUT, unique_cut=DEFAULT_UNIQUE_CUT, corr_threshold=DEFAULT_CORR_THRESHOLD ) - kept = gt[:, keep_mask] - mean = kept.mean(axis=0) - std = kept.std(axis=0) - if not (std > 0).all(): - raise ValueError(f"selection kept zero-variance features: {np.array(feature_names)[keep_mask][std == 0]}") + kept_names = [n for n, k in zip(feature_names, keep_mask, strict=True) if k] + pooled_std = pooled[:, keep_mask].std(axis=0) + if not (pooled_std > 0).all(): + raise ValueError(f"pooled fit kept zero-variance features: {np.array(kept_names)[pooled_std == 0]}") + floor = STD_FLOOR_FRACTION * pooled_std + + scalers: dict[str, dict[str, Any]] = {} + for fit in fits: + kept = np.asarray(fit.cells, dtype=np.float64)[:, keep_mask] + std = kept.std(axis=0) + floored = std < floor + scalers[fit.dataset] = { + **fit.record, + "in_mask_fit": fit.in_mask_fit, + "mean": [float(v) for v in kept.mean(axis=0)], + "std": [float(v) for v in np.where(floored, floor, std)], + "floored_features": [n for n, f in zip(kept_names, floored, strict=True) if f], + "n_cells": int(kept.shape[0]), + "gt_matrix_sha256": gt_matrix_sha256(fit.cells), + } payload: dict[str, Any] = { "schema": CP_REFERENCE_SCHEMA, "target_name": target_name, - "dimension": dimension, "feature_names": list(feature_names), "keep_mask": [bool(b) for b in keep_mask], - "kept_feature_names": [n for n, k in zip(feature_names, keep_mask, strict=True) if k], - "mean": [float(v) for v in mean], - "std": [float(v) for v in std], + "kept_feature_names": kept_names, "criteria": { "freq_cut": DEFAULT_FREQ_CUT, "unique_cut": DEFAULT_UNIQUE_CUT, "corr_threshold": DEFAULT_CORR_THRESHOLD, - "fit_on": "gt_only", + "mask_fit_on": "gt_only_pooled", + "scaler_fit_on": "gt_only_per_dataset", + "std_floor_fraction_of_pooled": STD_FLOOR_FRACTION, }, "cp_identity": cp_identity, - "fit": { - "datasets": datasets, - "n_cells": int(gt.shape[0]), - "gt_matrix_sha256": gt_matrix_sha256(gt), + "mask_fit": { + "datasets": [f.dataset for f in fits if f.in_mask_fit], + "n_cells": int(pooled.shape[0]), + "gt_matrix_sha256": gt_matrix_sha256(pooled), + "pooled_std": [float(v) for v in pooled_std], }, + "scalers": scalers, + "lite": lite, "created_at": datetime.now(UTC).isoformat(), } payload["sha256"] = payload_sha256(payload) return payload -def write_cp_reference(payload: dict[str, Any], path: Path) -> None: - """Write a reference payload to ``path`` (parents created).""" +def write_cp_reference(payload: dict[str, Any], path: Path, *, force: bool = False) -> bool: + """Write a reference payload atomically (tmp file in the same dir, then ``os.replace``). + + Parameters + ---------- + payload : dict + Output of :func:`fit_cp_reference`. + path : pathlib.Path + Destination; parents are created. + force : bool, optional + Overwrite an existing reference whose content hash differs. + + Returns + ------- + bool + ``True`` if the file was written, ``False`` if an identical reference was + already there (no-op). + + Raises + ------ + FileExistsError + If ``path`` holds a reference with a different hash and ``force`` is off: + replacing it changes CP values and invalidates every eval stamped with it. + """ + if path.exists(): + existing = json.loads(path.read_text())["sha256"] + if existing == payload["sha256"]: + return False + if not force: + raise FileExistsError( + f"{path} holds a different CP reference (sha256 {existing[:12]} vs new {payload['sha256'][:12]}); " + "replacing it invalidates every eval scored with it. Pass --force to replace it." + ) path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=1) + "\n") + tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") + tmp.write_text(json.dumps(payload, indent=1) + "\n") + os.replace(tmp, path) + return True @dataclass(frozen=True) -class CPReference: - """A loaded, hash-verified CP reference. +class DatasetCPSpace: + """The CP space one eval scores in: the reference mask plus one dataset's scaler. Picklable (plain arrays + scalars) so it can be shipped to spawn workers. """ - path: str - sha256: str + reference_path: str + reference_sha256: str target_name: str - dimension: str + dataset: str + scaler_dataset: str + is_lite: bool feature_names: tuple[str, ...] keep_mask: np.ndarray mean: np.ndarray std: np.ndarray - cp_identity: dict[str, Any] + fit_positions: frozenset[str] + fit_n_cells: int + gt_cache_dir: str | None + cp_cache_built_at: str | None def _check_width(self, x: np.ndarray) -> None: """Raise when a CP matrix does not have the reference's column count.""" if x.ndim != 2 or x.shape[1] != len(self.feature_names): raise StaleCacheError( f"CP feature dimension mismatch: matrix has shape {x.shape}, the CP reference " - f"{self.path} has {len(self.feature_names)} features. A CP feature cache was built " + f"{self.reference_path} has {len(self.feature_names)} features. A CP feature cache was built " "with a different recipe (e.g. a GLCM toggle or a CP_FEATURE_VERSION change without " "a cache rebuild). Rebuild with force_recompute.gt_cp=true and/or pred_cp=true " "(or force_recompute.all=true)." @@ -220,11 +374,144 @@ def select(self, x: np.ndarray) -> np.ndarray: return x[:, self.keep_mask] def transform(self, x: np.ndarray) -> np.ndarray: - """Return the kept columns of a raw CP matrix, standardized by the shared GT scaler.""" + """Return the kept columns of a raw CP matrix, standardized by this dataset's GT scaler.""" return (self.select(x) - self.mean) / self.std + def check_gt_cache(self, gt_cache_dir: str | None) -> None: + """Refuse an eval whose GT CP cache was re-cached after the reference was fit. + + Compares the eval's GT cache manifest ``cp_features.built_at`` with the one + recorded at build time. A lite dataset is checked against its own cache + only when the build recorded one (its scaler is the parent's, so a lite + cache is otherwise not what the scaler describes); when nothing is + recorded the check is skipped. + + Parameters + ---------- + gt_cache_dir : str or None + The eval's ``io.gt_cache_dir``. + + Raises + ------ + ValueError + If the eval reads a different GT cache dir than the recorded one. + StaleCacheError + If the recorded and current ``built_at`` differ. + """ + if self.cp_cache_built_at is None: + return + if gt_cache_dir is None or Path(gt_cache_dir) != Path(self.gt_cache_dir): + raise ValueError( + f"{self.dataset}: the CP reference was fit on GT cache {self.gt_cache_dir}, " + f"this eval reads {gt_cache_dir}" + ) + entry = load_manifest(cache_paths(gt_cache_dir))["artifacts"].get("cp_features") + built_at = entry["built_at"] if entry is not None else None + if built_at != self.cp_cache_built_at: + raise StaleCacheError( + f"{self.dataset}: GT CP cache {gt_cache_dir} was built at {built_at}, but the CP reference " + f"{self.reference_path} was fit on the cache built at {self.cp_cache_built_at}. " + f"Rebuild the reference:\n {_build_command(self.target_name)}" + ) + + def check_positions(self, positions: list[str]) -> None: + """Refuse an eval whose GT positions are not the ones the scaler was fit on. + + Parameters + ---------- + positions : list of str + GT position names the eval scores (after any exclusion/limit). + + Raises + ------ + ValueError + If the positions are not a subset of the fit's, or, for a non-lite + dataset, not exactly the fit's (e.g. ``limit_positions`` or + ``io.exclude_fov_names``: a partial walk's CP values are not the + cells the scaler describes). + """ + extra = set(positions) - self.fit_positions + if extra: + raise ValueError( + f"{self.dataset}: GT positions {sorted(extra)[:5]} are not in the CP reference fit of " + f"{self.scaler_dataset} ({self.reference_path})" + ) + if not self.is_lite and set(positions) != self.fit_positions: + missing = sorted(self.fit_positions - set(positions)) + raise ValueError( + f"{self.dataset}: CP metrics need every GT position the scaler was fit on; this eval " + f"skips {len(missing)} (e.g. {missing[:5]}). limit_positions / io.exclude_fov_names " + "cannot be combined with compute_feature_metrics=true." + ) + + def check_n_cells(self, n_gt_cells: int) -> None: + """Refuse a dataset-level CP stage whose GT cell count differs from the fit's. + + Lite datasets are exempt: they score a subset of the parent's cells, and + :meth:`CPReference.for_dataset` already required the parent to be recorded. + + Raises + ------ + ValueError + If ``n_gt_cells`` (finite GT CP rows scored) differs from the recorded count. + """ + if not self.is_lite and n_gt_cells != self.fit_n_cells: + raise ValueError( + f"{self.dataset}: this eval scores {n_gt_cells} finite GT CP cells, the CP reference scaler was " + f"fit on {self.fit_n_cells}. The GT CP cache changed since the reference was built." + ) + + +@dataclass(frozen=True) +class CPReference: + """A loaded, hash-verified CP reference.""" + + path: str + sha256: str + target_name: str + feature_names: tuple[str, ...] + keep_mask: np.ndarray + cp_identity: dict[str, Any] + scalers: dict[str, dict[str, Any]] + lite: dict[str, dict[str, Any]] + + def for_dataset(self, dataset: str) -> DatasetCPSpace: + """Bind the reference to one eval dataset (a lite dataset gets its parent's scaler). + + Raises + ------ + KeyError + If the reference has neither a scaler nor a lite entry for ``dataset``. + """ + is_lite = dataset in self.lite + if not is_lite and dataset not in self.scalers: + raise KeyError( + f"CP reference {self.path} has no scaler for dataset {dataset!r} " + f"(scalers: {sorted(self.scalers)}, lite: {sorted(self.lite)}); rebuild it:\n" + f" {_build_command(self.target_name)}" + ) + scaler_dataset = self.lite[dataset]["parent"] if is_lite else dataset + scaler = self.scalers[scaler_dataset] + own = self.lite[dataset] if is_lite else scaler + return DatasetCPSpace( + reference_path=self.path, + reference_sha256=self.sha256, + target_name=self.target_name, + dataset=dataset, + scaler_dataset=scaler_dataset, + is_lite=is_lite, + feature_names=self.feature_names, + keep_mask=self.keep_mask, + mean=np.asarray(scaler["mean"], dtype=np.float64), + std=np.asarray(scaler["std"], dtype=np.float64), + fit_positions=frozenset(scaler["positions"]), + fit_n_cells=int(scaler["n_cells"]), + gt_cache_dir=own["gt_cache_dir"], + cp_cache_built_at=own["cp_cache_built_at"], + ) + -def load_cp_reference(path: Path, *, target_name: str, dimension: str) -> CPReference: +def load_cp_reference(path: Path, *, target_name: str) -> CPReference: """Load and hash-verify a CP reference. Parameters @@ -233,8 +520,6 @@ def load_cp_reference(path: Path, *, target_name: str, dimension: str) -> CPRefe Reference JSON. target_name : str The eval's ``target_name``; the reference must have been fit for it. - dimension : str - The eval's CP feature-space dimension. Returns ------- @@ -247,12 +532,12 @@ def load_cp_reference(path: Path, *, target_name: str, dimension: str) -> CPRefe If ``path`` does not exist; the message names the build command. ValueError If the stored hash disagrees with the content, or the reference was fit - for another target, dimension, or schema. + for another target or schema. """ if not path.is_file(): raise FileNotFoundError( - f"CP reference {path} not found. CP (GLCM+) feature metrics are scored in a " - f"per-target GT feature space; build it with:\n {_build_command(target_name, dimension)}\n" + f"CP reference {path} not found. CP (GLCM+) feature metrics are scored in a per-target " + f"GT feature space; build it with:\n {_build_command(target_name)}\n" "(or point feature_metrics.cp.reference_path at an existing reference)." ) payload = json.loads(path.read_text()) @@ -260,54 +545,51 @@ def load_cp_reference(path: Path, *, target_name: str, dimension: str) -> CPRefe raise ValueError(f"CP reference {path} content does not match its recorded sha256 (edited by hand?)") if payload["schema"] != CP_REFERENCE_SCHEMA: raise ValueError(f"CP reference {path} has schema {payload['schema']}, expected {CP_REFERENCE_SCHEMA}") - if payload["target_name"] != target_name or payload["dimension"] != dimension: - raise ValueError( - f"CP reference {path} was fit for ({payload['target_name']!r}, {payload['dimension']!r}), " - f"but this eval is ({target_name!r}, {dimension!r})" - ) + if payload["target_name"] != target_name: + raise ValueError(f"CP reference {path} was fit for {payload['target_name']!r}, this eval is {target_name!r}") return CPReference( path=str(path), sha256=payload["sha256"], target_name=payload["target_name"], - dimension=payload["dimension"], feature_names=tuple(payload["feature_names"]), keep_mask=np.asarray(payload["keep_mask"], dtype=bool), - mean=np.asarray(payload["mean"], dtype=np.float64), - std=np.asarray(payload["std"], dtype=np.float64), cp_identity=payload["cp_identity"], + scalers=payload["scalers"], + lite=payload["lite"], ) def resolve_cp_reference_path(config: DictConfig) -> Path: """Return ``feature_metrics.cp.reference_path``, or the registry default when null. - The registry default lives under ``DATA_ROOT`` for full and lite evals alike, - so a lite eval scores in exactly the full benchmark's feature space. + The registry default lives under ``DATA_ROOT`` for full and lite evals alike. + + Raises + ------ + ValueError + If the default is requested for a target with no CP reference. """ - override = OmegaConf.select(config, "feature_metrics.cp.reference_path", default=None) + override = config.feature_metrics.cp.reference_path if override is not None: return Path(override) - return cp_reference_path(config.target_name, CP_REFERENCE_DIMENSION) + if config.target_name not in MASK_FIT_DATASETS: + raise ValueError(f"no CP reference for target {config.target_name!r}; expected {sorted(MASK_FIT_DATASETS)}") + return cp_reference_path(config.target_name) -def load_checked_cp_reference( - config: DictConfig, *, cp_identity: dict[str, Any], feature_names: tuple[str, ...] -) -> CPReference: - """Load the eval's CP reference and refuse one built for a different CP recipe. +def eval_cp_space(config: DictConfig) -> DatasetCPSpace: + """Load the eval's CP reference, refuse a foreign recipe, and bind it to the eval's dataset. Parameters ---------- config : DictConfig - Eval config (``target_name``, ``feature_metrics.cp.reference_path``). - cp_identity : dict - The eval's own CP recipe identity. - feature_names : tuple of str - The eval's CP column names. + Eval config (``target_name``, ``feature_metrics.cp.*``, + ``benchmark.dataset_ref.dataset``). Returns ------- - CPReference - The verified reference. + DatasetCPSpace + The verified reference bound to the eval's dataset. Raises ------ @@ -315,15 +597,16 @@ def load_checked_cp_reference( If the reference does not exist. StaleCacheError If the reference's CP recipe identity or feature names differ from the eval's. + KeyError + If the reference has no scaler for the eval's dataset. """ - ref = load_cp_reference( - resolve_cp_reference_path(config), target_name=config.target_name, dimension=CP_REFERENCE_DIMENSION - ) - if ref.cp_identity != cp_identity or ref.feature_names != feature_names: + identity, names = cp_space(config) + ref = load_cp_reference(resolve_cp_reference_path(config), target_name=config.target_name) + if ref.cp_identity != identity or ref.feature_names != names: raise StaleCacheError( f"CP reference {ref.path} was built for a different CP recipe.\n" - f" reference identity: {ref.cp_identity}\n eval identity: {cp_identity}\n" - f" reference features: {len(ref.feature_names)}; eval features: {len(feature_names)}\n" - f"Rebuild it from caches of the current recipe:\n {_build_command(ref.target_name, ref.dimension)}" + f" reference identity: {ref.cp_identity}\n eval identity: {identity}\n" + f" reference features: {len(ref.feature_names)}; eval features: {len(names)}\n" + f"Rebuild it from caches of the current recipe:\n {_build_command(ref.target_name)}" ) - return ref + return ref.for_dataset(config.benchmark.dataset_ref.dataset) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py index 254635aca..2244f68a1 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -9,8 +9,11 @@ import pytest from omegaconf import OmegaConf +from dynacell.evaluation import cp_reference +from dynacell.evaluation.cache import StaleCacheError, cache_paths, save_manifest from dynacell.evaluation.cp_reference import ( - CP_REFERENCE_DIMENSION, + STD_FLOOR_FRACTION, + DatasetFit, fit_cp_reference, load_cp_reference, payload_sha256, @@ -18,44 +21,58 @@ write_cp_reference, ) from dynacell.evaluation.feature_metrics import compute_feature_similarity_pairwise -from dynacell.evaluation.feature_select import select_features +from dynacell.evaluation.feature_select import select_gt_features from dynacell.evaluation.paths import DATA_ROOT, LITE_DATA_ROOT -from dynacell.evaluation.pipeline import _BackboneLists, _extend_backbone, _stage_cp_dataset_inputs +from dynacell.evaluation.pipeline import ( + _BackboneLists, + _cp_row_features, + _extend_backbone, + _stage_cp_dataset_inputs, +) _N_FEATURES = 8 _NAMES = tuple(f"f{i}" for i in range(_N_FEATURES)) +_POSITIONS = ["A/1/0", "A/1/1"] -def _gt(n: int = 300, seed: int = 0) -> np.ndarray: - """Synthetic GT CP matrix: 6 independent columns, one near-duplicate, one constant.""" +def _gt(n: int = 300, seed: int = 0, offset: float = 10.0, scale: float = 1.0) -> np.ndarray: + """Synthetic GT CP matrix: 6 independent columns, one near-duplicate of f0, one constant.""" rng = np.random.default_rng(seed) - x = rng.standard_normal((n, _N_FEATURES)) * np.arange(1, _N_FEATURES + 1) + 10.0 + x = (rng.standard_normal((n, _N_FEATURES)) * np.arange(1, _N_FEATURES + 1)) * scale + offset x[:, 6] = x[:, 0] * 2.0 + 1e-6 * rng.standard_normal(n) # |r| ~ 1 with f0 x[:, 7] = 3.0 # constant return x -def _reference(tmp_path: Path, gt: np.ndarray): - """Fit, write and load a reference on ``gt``.""" - payload = fit_cp_reference( - gt, - target_name="er", - dimension=CP_REFERENCE_DIMENSION, - feature_names=_NAMES, - cp_identity={"cp_feature_version": "test"}, - datasets=[{"dataset": "synthetic", "n_cells": gt.shape[0]}], +def _fit(dataset: str, cells: np.ndarray, *, in_mask_fit: bool = True, built_at: str | None = None) -> DatasetFit: + return DatasetFit( + dataset=dataset, + cells=cells, + record={"positions": _POSITIONS, "gt_cache_dir": "/c/" + dataset, "cp_cache_built_at": built_at}, + in_mask_fit=in_mask_fit, ) - path = tmp_path / "er__3d.json" - write_cp_reference(payload, path) - return load_cp_reference(path, target_name="er", dimension=CP_REFERENCE_DIMENSION), payload -def _old_per_side_zscore(pred: np.ndarray, target: np.ndarray) -> tuple[np.ndarray, np.ndarray]: - """The retired transform: each side standardized by its own mean/std.""" - return ( - (pred - pred.mean(axis=0)) / (pred.std(axis=0) + 1e-8), - (target - target.mean(axis=0)) / (target.std(axis=0) + 1e-8), +def _reference(tmp_path: Path, fits: list[DatasetFit], lite: dict | None = None): + """Fit, write and load a reference; returns ``(CPReference, payload)``.""" + payload = fit_cp_reference( + fits, target_name="er", feature_names=_NAMES, cp_identity={"cp_feature_version": "test"}, lite=lite or {} + ) + path = tmp_path / "er.json" + write_cp_reference(payload, path, force=True) + return load_cp_reference(path, target_name="er"), payload + + +@pytest.fixture +def two_sets(tmp_path: Path): + """A reference over two test sets whose GT differ by a large offset and scale, plus HEK and one lite.""" + a, b, hek = _gt(seed=0), _gt(seed=1, offset=40.0, scale=3.0), _gt(seed=2, offset=-5.0) + ref, payload = _reference( + tmp_path, + [_fit("set-a", a), _fit("set-b", b), _fit("hek", hek, in_mask_fit=False)], + lite={"set-a-lite": {"parent": "set-a", "gt_cache_dir": "/lite", "cp_cache_built_at": None}}, ) + return ref, payload, {"set-a": a, "set-b": b, "hek": hek} def _lists(pred: np.ndarray, gt: np.ndarray) -> _BackboneLists: @@ -67,63 +84,222 @@ def _lists(pred: np.ndarray, gt: np.ndarray) -> _BackboneLists: return bb -def test_fit_drops_constant_and_correlated_and_records_provenance(tmp_path: Path) -> None: - gt = _gt() - ref, payload = _reference(tmp_path, gt) - assert not ref.keep_mask[7] - assert ref.keep_mask[0] ^ ref.keep_mask[6] - np.testing.assert_allclose(ref.mean, gt[:, ref.keep_mask].mean(axis=0)) - np.testing.assert_allclose(ref.std, gt[:, ref.keep_mask].std(axis=0)) - assert payload["fit"]["n_cells"] == gt.shape[0] - assert payload["criteria"]["fit_on"] == "gt_only" - assert len(payload["fit"]["gt_matrix_sha256"]) == 64 - # Transformed GT is exactly standardized by the shared scaler. - z = ref.transform(gt) - np.testing.assert_allclose(z.mean(axis=0), 0.0, atol=1e-10) - np.testing.assert_allclose(z.std(axis=0), 1.0, atol=1e-10) +def test_mask_is_pooled_over_the_mask_fit_sets_only(two_sets) -> None: + """The mask is the GT-only selection over the mask-fit sets; HEK does not enter it.""" + ref, payload, cells = two_sets + expected = select_gt_features(np.vstack([cells["set-a"], cells["set-b"]])) + np.testing.assert_array_equal(ref.keep_mask, expected) + assert not ref.keep_mask[7] # constant in every set + # HEK entering the fit would change the mask: its offset reshapes the pooled correlations. + with_hek = select_gt_features(np.vstack([cells["set-a"], cells["set-b"], cells["hek"]])) + assert not np.array_equal(with_hek, expected) + assert payload["mask_fit"]["datasets"] == ["set-a", "set-b"] + assert payload["mask_fit"]["n_cells"] == 600 + assert payload["scalers"]["hek"]["in_mask_fit"] is False + + +def test_each_set_gets_its_own_gt_scaler(two_sets) -> None: + """A set's scaler standardizes exactly that set's GT; a lite set reuses its parent's.""" + ref, payload, cells = two_sets + for name in ("set-a", "set-b", "hek"): + space = ref.for_dataset(name) + z = space.transform(cells[name]) + np.testing.assert_allclose(z.mean(axis=0), 0.0, atol=1e-9) + np.testing.assert_allclose(z.std(axis=0), 1.0, atol=1e-9) + assert payload["scalers"][name]["n_cells"] == 300 + # set-b's GT in set-a's space is far from standardized: the scalers really differ. + assert np.abs(ref.for_dataset("set-a").transform(cells["set-b"]).mean(axis=0)).max() > 5 + lite = ref.for_dataset("set-a-lite") + assert lite.is_lite and lite.scaler_dataset == "set-a" + np.testing.assert_array_equal(lite.mean, ref.for_dataset("set-a").mean) + with pytest.raises(KeyError, match="no scaler for dataset 'unknown'"): + ref.for_dataset("unknown") + + +def test_std_floor_fires_on_a_feature_constant_within_one_set(tmp_path: Path) -> None: + """A pooled-kept feature that is constant within one set gets floored std, and is recorded.""" + a, b = _gt(seed=0), _gt(seed=1) + # Varies in the pool (via set-a) but is ~constant within set-b. Distinct values, so the + # pooled variance filter keeps it; its set-b std (~1e-9) is far below the floor. + b[:, 2] = 5.0 + 1e-9 * np.random.default_rng(3).standard_normal(b.shape[0]) + ref, payload = _reference(tmp_path, [_fit("set-a", a), _fit("set-b", b)]) + kept = payload["kept_feature_names"] + assert "f2" in kept + assert payload["scalers"]["set-b"]["floored_features"] == ["f2"] + assert payload["scalers"]["set-a"]["floored_features"] == [] + j = kept.index("f2") + pooled_std = np.vstack([a, b])[:, ref.keep_mask].std(axis=0)[j] + assert payload["scalers"]["set-b"]["std"][j] == pytest.approx(STD_FLOOR_FRACTION * pooled_std) + assert np.isfinite(ref.for_dataset("set-b").transform(b)).all() + + +def test_pooled_zero_variance_is_refused(monkeypatch) -> None: + """A kept feature with zero POOLED variance is refused: the floor would have nothing to scale by. + + The variance filter normally drops such a column first, so the selector is + replaced by keep-everything to reach the guard. + """ + # Patch the module object this file imported: other tests re-create dynacell.evaluation.* + # in sys.modules, so a dotted-path patch could land on a different module. + monkeypatch.setattr(cp_reference, "select_gt_features", lambda gt, **kw: np.ones(gt.shape[1], dtype=bool)) + with pytest.raises(ValueError, match="zero-variance"): + fit_cp_reference([_fit("set-a", _gt())], target_name="er", feature_names=_NAMES, cp_identity={}, lite={}) + + +def test_empty_dataset_is_refused() -> None: + with pytest.raises(ValueError, match="no GT cells"): + fit_cp_reference( + [_fit("set-a", np.empty((0, _N_FEATURES)))], target_name="er", feature_names=_NAMES, cp_identity={}, lite={} + ) def test_hash_ignores_build_time_and_detects_edits(tmp_path: Path) -> None: """Identical inputs hash identically; a hand edit fails the load.""" - (tmp_path / "rebuild").mkdir() - _, first = _reference(tmp_path, _gt()) - _, second = _reference(tmp_path / "rebuild", _gt()) - assert first["sha256"] == second["sha256"] == payload_sha256({**first, "created_at": "another time"}) - - path = tmp_path / "er__3d.json" + _, first = _reference(tmp_path, [_fit("set-a", _gt())]) + assert first["sha256"] == payload_sha256({**first, "created_at": "another time"}) + path = tmp_path / "er.json" edited = json.loads(path.read_text()) - edited["std"][0] *= 2.0 + edited["scalers"]["set-a"]["std"][0] *= 2.0 path.write_text(json.dumps(edited)) with pytest.raises(ValueError, match="does not match its recorded sha256"): - load_cp_reference(path, target_name="er", dimension=CP_REFERENCE_DIMENSION) + load_cp_reference(path, target_name="er") + + +def test_write_is_atomic_and_refuses_a_different_reference(tmp_path: Path) -> None: + """Identical hash is a no-op; a different one needs force; no tmp file is left behind.""" + first = fit_cp_reference([_fit("set-a", _gt())], target_name="er", feature_names=_NAMES, cp_identity={}, lite={}) + other = fit_cp_reference( + [_fit("set-a", _gt(seed=5))], target_name="er", feature_names=_NAMES, cp_identity={}, lite={} + ) + path = tmp_path / "er.json" + assert write_cp_reference(first, path) is True + assert write_cp_reference({**first, "created_at": "later"}, path) is False + with pytest.raises(FileExistsError, match="--force"): + write_cp_reference(other, path) + assert json.loads(path.read_text())["sha256"] == first["sha256"] + assert write_cp_reference(other, path, force=True) is True + assert json.loads(path.read_text())["sha256"] == other["sha256"] + assert sorted(p.name for p in tmp_path.iterdir()) == ["er.json"] def test_load_refuses_missing_and_other_target(tmp_path: Path) -> None: - with pytest.raises(FileNotFoundError, match="build_cp_reference.py --target er --dimension 3d"): - load_cp_reference(tmp_path / "absent.json", target_name="er", dimension=CP_REFERENCE_DIMENSION) - _reference(tmp_path, _gt()) + with pytest.raises(FileNotFoundError, match="build_cp_reference.py --target er"): + load_cp_reference(tmp_path / "absent.json", target_name="er") + _reference(tmp_path, [_fit("set-a", _gt())]) with pytest.raises(ValueError, match="was fit for"): - load_cp_reference(tmp_path / "er__3d.json", target_name="nucleus", dimension=CP_REFERENCE_DIMENSION) + load_cp_reference(tmp_path / "er.json", target_name="nucleus") def test_registry_path_is_shared_by_lite_and_full() -> None: """``reference_path: null`` resolves under DATA_ROOT, never LITE_DATA_ROOT.""" config = OmegaConf.create({"target_name": "mitochondria", "feature_metrics": {"cp": {"reference_path": None}}}) path = resolve_cp_reference_path(config) - assert path == DATA_ROOT / "cp_reference" / "mitochondria__3d.json" + assert path == DATA_ROOT / "cp_reference" / "mitochondria.json" assert LITE_DATA_ROOT not in path.parents config.feature_metrics.cp.reference_path = "/x/ref.json" assert resolve_cp_reference_path(config) == Path("/x/ref.json") + config.target_name = "golgi" + config.feature_metrics.cp.reference_path = None + with pytest.raises(ValueError, match="no CP reference for target"): + resolve_cp_reference_path(config) + + +def test_positions_must_match_the_fit(two_sets) -> None: + """Non-lite: exactly the fit's positions. Lite: any subset of its parent's.""" + ref, _, _ = two_sets + ref.for_dataset("set-a").check_positions(_POSITIONS) + with pytest.raises(ValueError, match="cannot be combined"): + ref.for_dataset("set-a").check_positions(_POSITIONS[:1]) + with pytest.raises(ValueError, match="not in the CP reference fit"): + ref.for_dataset("set-a").check_positions([*_POSITIONS, "B/1/0"]) + ref.for_dataset("set-a-lite").check_positions(_POSITIONS[:1]) + with pytest.raises(ValueError, match="not in the CP reference fit"): + ref.for_dataset("set-a-lite").check_positions(["B/1/0"]) + + +def test_gt_recache_is_detected(tmp_path: Path) -> None: + """A GT cache whose cp_features.built_at moved after the build is refused.""" + cache_dir = tmp_path / "cache" + save_manifest(cache_paths(cache_dir), {"artifacts": {"cp_features": {"built_at": "t0"}}}) + fit = DatasetFit( + dataset="set-a", + cells=_gt(), + record={"positions": _POSITIONS, "gt_cache_dir": str(cache_dir), "cp_cache_built_at": "t0"}, + in_mask_fit=True, + ) + ref, _ = _reference(tmp_path, [fit]) + space = ref.for_dataset("set-a") + space.check_gt_cache(str(cache_dir)) + with pytest.raises(ValueError, match="this eval reads"): + space.check_gt_cache(str(tmp_path / "elsewhere")) + save_manifest(cache_paths(cache_dir), {"artifacts": {"cp_features": {"built_at": "t1"}}}) + with pytest.raises(StaleCacheError, match="was built at t1"): + space.check_gt_cache(str(cache_dir)) + +def test_lite_without_a_recorded_cache_skips_the_recache_check(two_sets) -> None: + ref, _, _ = two_sets + ref.for_dataset("set-a-lite").check_gt_cache(None) -def test_two_models_share_one_mask(tmp_path: Path) -> None: + +def test_staged_dataset_arrays_are_the_dataset_scaler_transform(two_sets, tmp_path: Path) -> None: + """Dataset-level CP metric arrays are exactly ``transform`` with the eval dataset's scaler. + + Catches a per-side z-score (or any other scaler) sneaking back in: set-b's + scaler is far from set-a's, and the prediction is off-scale on purpose. + """ + ref, _, cells = two_sets + gt = cells["set-b"] + pred = 0.5 * gt + 7.0 + space = ref.for_dataset("set-b") + staged = _stage_cp_dataset_inputs(_lists(pred, gt), space, gt.shape[0], tmp_path) + np.testing.assert_array_equal(staged[1], space.transform(pred)) + np.testing.assert_array_equal(staged[2], space.transform(gt)) + np.testing.assert_array_equal(staged[3], pred[:, ref.keep_mask]) # probe: masked, unscaled + np.testing.assert_array_equal(staged[4], gt[:, ref.keep_mask]) + assert not np.allclose(staged[1], ref.for_dataset("set-a").transform(pred)) + per_side = (pred[:, ref.keep_mask] - pred[:, ref.keep_mask].mean(0)) / pred[:, ref.keep_mask].std(0) + assert not np.allclose(staged[1], per_side) + sidecar = json.loads((tmp_path / "cp_selected_feature_mask.json").read_text()) + assert (sidecar["dataset"], sidecar["scaler_dataset"], sidecar["reference_sha256"]) == ( + "set-b", + "set-b", + ref.sha256, + ) + + +def test_staged_cell_count_must_match_the_fit(two_sets, tmp_path: Path) -> None: + """A non-lite eval scoring a different number of GT cells than the fit is refused; lite is exempt.""" + ref, _, cells = two_sets + gt = cells["set-a"] + with pytest.raises(ValueError, match="scores 299 finite GT CP cells"): + _stage_cp_dataset_inputs(_lists(gt, gt), ref.for_dataset("set-a"), 299, tmp_path) + _stage_cp_dataset_inputs(_lists(gt[:40], gt[:40]), ref.for_dataset("set-a-lite"), 40, tmp_path) + + +def test_row_features_use_the_dataset_scaler(two_sets) -> None: + """Per-row CP inputs are the same dataset-scaler transform as the dataset-level stage.""" + ref, _, cells = two_sets + gt = cells["set-b"][:20] + pred = gt * 0.5 + space = ref.for_dataset("set-b") + p, g = _cp_row_features(pred, gt, space) + np.testing.assert_array_equal(p, space.transform(pred)) + np.testing.assert_array_equal(g, space.transform(gt)) + empty = np.empty((0, _N_FEATURES)) + assert _cp_row_features(empty, empty, space)[0] is empty + with pytest.raises(StaleCacheError, match="CP feature dimension mismatch"): + _cp_row_features(np.ones((4, 58)), np.ones((4, 58)), space) + + +def test_two_models_share_one_mask(two_sets, tmp_path: Path) -> None: """Two different predictions against one GT are scored on the identical feature subset. - The retired pooled GT+pred selection gives these two models different masks, - which is the model-dependence the reference removes. + The retired selection, which pooled GT with each model's prediction, gives these + two models different masks. """ - gt = _gt() - ref, _ = _reference(tmp_path, gt) + ref, _, cells = two_sets + gt = cells["set-a"] rng = np.random.default_rng(1) pred_a = gt + 0.1 * rng.standard_normal(gt.shape) pred_b = gt.copy() @@ -134,40 +310,36 @@ def test_two_models_share_one_mask(tmp_path: Path) -> None: for i, pred in enumerate((pred_a, pred_b)): save_dir = tmp_path / f"model_{i}" save_dir.mkdir() - staged = _stage_cp_dataset_inputs(_lists(pred, gt), ref, save_dir) - sidecar = json.loads((save_dir / "cp_selected_feature_mask.json").read_text()) - assert sidecar["reference_sha256"] == ref.sha256 - assert staged[1].shape[1] == staged[2].shape[1] == int(ref.keep_mask.sum()) - # Probe inputs are masked but unscaled. - np.testing.assert_array_equal(staged[4], gt[:, ref.keep_mask]) - masks.append(sidecar["keep_mask"]) + _stage_cp_dataset_inputs(_lists(pred, gt), ref.for_dataset("set-a"), gt.shape[0], save_dir) + masks.append(json.loads((save_dir / "cp_selected_feature_mask.json").read_text())["keep_mask"]) assert masks[0] == masks[1] == [bool(b) for b in ref.keep_mask] - - old_a = select_features(gt, pred_a)[2] - old_b = select_features(gt, pred_b)[2] + old_a = select_gt_features(np.vstack([gt, pred_a])) + old_b = select_gt_features(np.vstack([gt, pred_b])) assert not np.array_equal(old_a, old_b) -def test_over_smoothing_registers_in_the_shared_space(tmp_path: Path) -> None: +def test_over_smoothing_registers_in_the_shared_space(two_sets) -> None: """``pred = 0.5 * GT + c`` (compressed range + offset) now scores clearly off GT. The retired per-side z-score maps any per-feature affine distortion of GT back - onto GT exactly, so it reported a perfect match (cosine 1, KID ~0). The shared - GT scaler keeps the distortion, so the same prediction is now penalized. + onto GT exactly, so it reported a perfect match. The dataset's GT scaler keeps + the distortion, so the same prediction is now penalized. """ - gt = _gt(n=400) - ref, _ = _reference(tmp_path, gt) + ref, _, cells = two_sets + gt = cells["set-a"] pred = 0.5 * gt + 7.0 + space = ref.for_dataset("set-a") - old_p, old_t = _old_per_side_zscore(ref.select(pred), ref.select(gt)) + sel_p, sel_t = space.select(pred), space.select(gt) + old_p = (sel_p - sel_p.mean(0)) / (sel_p.std(0) + 1e-8) + old_t = (sel_t - sel_t.mean(0)) / (sel_t.std(0) + 1e-8) old = compute_feature_similarity_pairwise(old_p, old_t, "CP", compute_fid=False) - new = compute_feature_similarity_pairwise(ref.transform(pred), ref.transform(gt), "CP", compute_fid=False) - + new = compute_feature_similarity_pairwise(space.transform(pred), space.transform(gt), "CP", compute_fid=False) # KID of GT against itself: the estimator's floor for "indistinguishable" (not exactly 0). - floor = compute_feature_similarity_pairwise(ref.transform(gt), ref.transform(gt), "CP", compute_fid=False) + floor = compute_feature_similarity_pairwise(space.transform(gt), space.transform(gt), "CP", compute_fid=False) assert old["CP_Median_Cosine_Similarity"] == pytest.approx(1.0, abs=1e-6) assert old["CP_KID"] == pytest.approx(floor["CP_KID"], abs=1e-4) assert new["CP_Median_Cosine_Similarity"] < 0.9 assert new["CP_KID"] > 1.0 - assert new["CP_KID"] > 100 * abs(floor["CP_KID"]) + assert new["CP_KID"] > 20 * abs(floor["CP_KID"]) diff --git a/applications/dynacell/src/dynacell/evaluation/instance_metrics_test.py b/applications/dynacell/src/dynacell/evaluation/instance_metrics_test.py index 75948ea72..11e28f843 100644 --- a/applications/dynacell/src/dynacell/evaluation/instance_metrics_test.py +++ b/applications/dynacell/src/dynacell/evaluation/instance_metrics_test.py @@ -148,7 +148,7 @@ def test_instance_ap_is_written_to_mask_metrics_csv(tmp_path): }, } ) - save_metrics(config, mask_metrics=[{"FOV": "0/0/0", "Timepoint": 0, **row}]) + save_metrics(config, mask_metrics=[{"FOV": "0/0/0", "Timepoint": 0, **row}], cp_reference_sha256=None) assert not list(tmp_path.glob("instance_ap*.csv")) written = tmp_path / "mask_metrics.csv" diff --git a/applications/dynacell/src/dynacell/evaluation/paths.py b/applications/dynacell/src/dynacell/evaluation/paths.py index 8f02509e4..5ba798fa2 100644 --- a/applications/dynacell/src/dynacell/evaluation/paths.py +++ b/applications/dynacell/src/dynacell/evaluation/paths.py @@ -960,25 +960,21 @@ def pred_cache_dir( return Path(data_root) / test_set / subroot / organelle / model / train_set / leaf -#: Eval ``target_name`` values that have a CP reference (one per benchmark target). -_CP_REFERENCE_TARGETS: frozenset[str] = frozenset({"nucleus", "membrane", "er", "mitochondria"}) - - -def cp_reference_path(target_name: str, dimension: str, data_root: str | Path = DATA_ROOT) -> Path: +def cp_reference_path(target_name: str, data_root: str | Path = DATA_ROOT) -> Path: """Return the registry path of a target's CP (GLCM+) reference. - ``DATA_ROOT/cp_reference/__.json``. Keyed by the eval + ``DATA_ROOT/cp_reference/.json``. Keyed by the eval ``target_name`` (``mitochondria``, not the grammar's ``mito``), since that is what an eval config carries. Lite evals use this same ``DATA_ROOT`` path, never - ``LITE_DATA_ROOT``: the reference is fit on the full test sets so lite and full - score in one feature space. See :mod:`dynacell.evaluation.cp_reference`. + ``LITE_DATA_ROOT``: one reference holds the mask and every test set's scaler, + and a lite dataset reuses its parent's scaler. See + :mod:`dynacell.evaluation.cp_reference`, which also owns the set of targets + that have a reference. Parameters ---------- target_name : str Eval target (``nucleus``/``membrane``/``er``/``mitochondria``). - dimension : str - CP feature-space dimension (``3d``). data_root : str or pathlib.Path, optional Registry root. Defaults to :data:`DATA_ROOT`. @@ -986,15 +982,8 @@ def cp_reference_path(target_name: str, dimension: str, data_root: str | Path = ------- pathlib.Path The reference JSON path. - - Raises - ------ - ValueError - If ``target_name`` has no CP reference. """ - if target_name not in _CP_REFERENCE_TARGETS: - raise ValueError(f"no CP reference for target {target_name!r}; expected one of {sorted(_CP_REFERENCE_TARGETS)}") - return Path(data_root) / "cp_reference" / f"{target_name}__{dimension}.json" + return Path(data_root) / "cp_reference" / f"{target_name}.json" def metrics_repo_dir( diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline.py b/applications/dynacell/src/dynacell/evaluation/pipeline.py index 29c86711c..7b682d651 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline.py @@ -18,7 +18,7 @@ from dynacell.evaluation._ref_hook import apply_dataset_ref from dynacell.evaluation.cache import FeatureKind -from dynacell.evaluation.cp_reference import CPReference, load_checked_cp_reference +from dynacell.evaluation.cp_reference import DatasetCPSpace, eval_cp_space from dynacell.evaluation.cross_condition_probe import run_for_group as _cross_condition_run_for_group from dynacell.evaluation.feature_metrics import ( compute_feature_similarity, @@ -26,8 +26,6 @@ ) from dynacell.evaluation.linear_probe import indistinguishability, paired_auroc from dynacell.evaluation.metrics import ( - CP_FEATURE_VERSION, - active_cp_feature_names, ascupy, build_crops, compute_pixel_metrics, @@ -41,7 +39,6 @@ ) from dynacell.evaluation.model_loader import EvalModels, init_cache_contexts, load_eval_models from dynacell.evaluation.pipeline_cache import ( - cp_recipe_identity, cpdino_infer_kwargs, flush_manifest, fov_cp_features, @@ -99,23 +96,15 @@ def _build_focus_slabs_map(config: DictConfig, gt_positions) -> dict[str, list[s } -def _eval_cp_reference(config: DictConfig) -> CPReference: - """Load the CP reference this eval scores in, refusing one of a different CP recipe. +def _cp_row_features(pred_cp: np.ndarray, gt_cp: np.ndarray, cp_space: DatasetCPSpace) -> tuple[np.ndarray, np.ndarray]: + """Per-(FOV, timepoint) CP inputs: the reference mask + this dataset's GT scaler on both sides. - The recipe identity and column names come from the same ``feature_metrics.cp.*`` - config and :data:`CP_FEATURE_VERSION` that key the CP feature caches, so a - reference built from caches of another recipe cannot be applied to this run's - features. See :mod:`dynacell.evaluation.cp_reference`. + The same transform as the dataset-level stage, so a per-row CP value is + comparable across models and FOVs. Empty inputs (no cells) pass through. """ - norm_node = OmegaConf.select(config, "feature_metrics.cp.norm", default=None) - glcm_node = OmegaConf.select(config, "feature_metrics.cp.glcm", default=None) - norm = OmegaConf.to_container(norm_node, resolve=True) if norm_node is not None else {} - glcm = OmegaConf.to_container(glcm_node, resolve=True) if glcm_node is not None else {} - return load_checked_cp_reference( - config, - cp_identity=cp_recipe_identity(CP_FEATURE_VERSION, norm, glcm), - feature_names=tuple(active_cp_feature_names(bool(glcm.get("enabled", False)))), - ) + if not (pred_cp.size and gt_cp.size): + return pred_cp, gt_cp + return cp_space.transform(pred_cp), cp_space.transform(gt_cp) def _real_vs_pred_probe( @@ -338,37 +327,53 @@ def _extend_backbone( def _stage_cp_dataset_inputs( - cp: _BackboneLists, cp_reference: CPReference, save_dir: Path + cp: _BackboneLists, cp_space: DatasetCPSpace, n_gt_cells: int, save_dir: Path ) -> tuple[str, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: """Stage the dataset-level CP inputs in the reference feature space. - KID/FID/cosine get the reference-masked features standardized by the shared GT - scaler (identical for pred and GT); the linear probe gets the reference-masked - raw features, since it fits its own per-fold scaler. Writes - ``cp_selected_feature_mask.json`` recording which reference was applied. + KID/FID/cosine get the reference-masked features standardized by this + dataset's GT scaler (identical for pred and GT); the linear probe gets the + reference-masked raw features, since it fits its own per-fold scaler. Writes + ``cp_selected_feature_mask.json`` recording which reference and scaler were + applied. + + Parameters + ---------- + cp : _BackboneLists + Aggregated CP rows of the run. + cp_space : DatasetCPSpace + Reference bound to the eval's dataset. + n_gt_cells : int + Finite GT CP rows the run scored (before the pairwise pred/GT drop); + must equal the scaler's fit count for a non-lite dataset. + save_dir : pathlib.Path + Eval dir receiving the sidecar. Returns ------- tuple ``("CP", pred_metric, target_metric, pred_probe, target_probe, pred_fovs, target_fovs)``. """ + cp_space.check_n_cells(n_gt_cells) pred_cp_raw = np.concatenate(cp.pred_feats, axis=0) target_cp_raw = np.concatenate(cp.gt_feats, axis=0) mask_payload = { - "reference_path": cp_reference.path, - "reference_sha256": cp_reference.sha256, - "feature_names": list(cp_reference.feature_names), - "keep_mask": [bool(b) for b in cp_reference.keep_mask], - "n_kept": int(cp_reference.keep_mask.sum()), - "n_total": int(cp_reference.keep_mask.size), + "reference_path": cp_space.reference_path, + "reference_sha256": cp_space.reference_sha256, + "dataset": cp_space.dataset, + "scaler_dataset": cp_space.scaler_dataset, + "feature_names": list(cp_space.feature_names), + "keep_mask": [bool(b) for b in cp_space.keep_mask], + "n_kept": int(cp_space.keep_mask.sum()), + "n_total": int(cp_space.keep_mask.size), } (save_dir / "cp_selected_feature_mask.json").write_text(json.dumps(mask_payload, indent=2)) return ( "CP", - cp_reference.transform(pred_cp_raw), - cp_reference.transform(target_cp_raw), - cp_reference.select(pred_cp_raw), - cp_reference.select(target_cp_raw), + cp_space.transform(pred_cp_raw), + cp_space.transform(target_cp_raw), + cp_space.select(pred_cp_raw), + cp_space.select(target_cp_raw), np.concatenate(cp.pred_fovs, axis=0), np.concatenate(cp.gt_fovs, axis=0), ) @@ -399,6 +404,7 @@ class FovResult: dynaclr: _BackboneLists = field(default_factory=_BackboneLists) celldino: _BackboneLists = field(default_factory=_BackboneLists) morphem: _BackboneLists = field(default_factory=_BackboneLists) + n_gt_cp_finite: int = 0 # finite GT CP rows, before the pairwise pred/GT non-finite drop timings: list[tuple[str, int | None, str, float]] = field(default_factory=list) @@ -581,14 +587,15 @@ def _process_one_fov( celldino_feature_extractor, morphem_feature_extractor, microssim_sim, - cp_reference: CPReference | None, + cp_space: DatasetCPSpace | None, predict_cached=None, target_cached=None, ) -> FovResult: """Compute everything one FOV contributes to the eval and return a FovResult. - ``cp_reference`` is the target's CP reference (``None`` only when - ``compute_feature_metrics=false``); per-timepoint CP metrics are scored in it. + ``cp_space`` is the target's CP reference bound to the eval's dataset (``None`` + only when ``compute_feature_metrics=false``); per-timepoint CP metrics are + scored in it. No side effects on shared parent state (no segmentation_results plate writes, no manifest flush). The parent aggregator handles those — see @@ -852,6 +859,7 @@ def _nuclei_seeds(segment_fn, **infer) -> np.ndarray | None: ) microssim_data: list[dict] = [] + n_gt_cp_finite = 0 fov_pixel_metrics: list[dict] = [] fov_mask_metrics: list[dict] = [] fov_feature_metrics: list[dict] = [] @@ -942,13 +950,10 @@ def _nuclei_seeds(segment_fn, **infer) -> np.ndarray | None: pred_dynaclr = pred_per_t["dynaclr"][t] pred_celldino = pred_per_t["celldino"][t] if pred_per_t["celldino"] is not None else None pred_morphem = pred_per_t["morphem"][t] if pred_per_t["morphem"] is not None else None + if gt_cp_per_t[t].size: + n_gt_cp_finite += int(np.isfinite(gt_cp_per_t[t]).all(axis=1).sum()) pred_cp, gt_cp_t = drop_paired_nonfinite_rows(pred_cp, gt_cp_per_t[t]) - # Same reference mask + shared GT scaler as the dataset-level block, so a - # per-row CP value is comparable across models and FOVs. - if pred_cp.size and gt_cp_t.size: - pred_cp_z, gt_cp_z = cp_reference.transform(pred_cp), cp_reference.transform(gt_cp_t) - else: - pred_cp_z, gt_cp_z = pred_cp, gt_cp_t + pred_cp_z, gt_cp_z = _cp_row_features(pred_cp, gt_cp_t, cp_space) # Prefixes come from _COLUMN_PREFIX, not literals: these per-timepoint # columns must match the dataset-level ones the parent derives from the # same map, or the two halves of a _* family drift apart. @@ -1017,6 +1022,7 @@ def _nuclei_seeds(segment_fn, **infer) -> np.ndarray | None: dynaclr=dynaclr, celldino=celldino, morphem=morphem, + n_gt_cp_finite=n_gt_cp_finite, timings=get_timings()[timings_start:], ) @@ -1197,7 +1203,7 @@ def _worker_run_fov( pos_name: str, cuda_empty_every_n: int, microssim_sim, - cp_reference: CPReference | None, + cp_space: DatasetCPSpace | None, ) -> FovResult: """Worker entry point: process one FOV by name and return FovResult. @@ -1209,7 +1215,7 @@ def _worker_run_fov( ``microssim_sim`` is the leaf-level fitted MicroMS3IM (or ``None`` when ``compute_microssim=false``); shipped per submission rather than via worker state because the parent fits it after the position list is - finalized and before any worker pool spawns. ``cp_reference`` is shipped the + finalized and before any worker pool spawns. ``cp_space`` is shipped the same way so every worker scores CP in the parent's verified reference. """ _worker_setup(config) @@ -1253,7 +1259,7 @@ def _worker_run_fov( state["celldino"], state["morphem"], microssim_sim, - cp_reference, + cp_space, ) # Worker-side manifest flush so interrupted runs preserve progress even @@ -1265,7 +1271,9 @@ def _worker_run_fov( return result -def evaluate_predictions(config: DictConfig, *, models: EvalModels | None = None): +def evaluate_predictions( + config: DictConfig, *, models: EvalModels | None = None, cp_space: DatasetCPSpace | None = None +): """Evaluate predictions on all test images. Parameters @@ -1280,6 +1288,10 @@ def evaluate_predictions(config: DictConfig, *, models: EvalModels | None = None preserves the historical single-condition behavior. Note: under ``runtime.executor=process``, workers still load their own model copies; this kwarg saves only the parent-side load. + cp_space : DatasetCPSpace | None, optional + The CP reference bound to this eval's dataset (:func:`eval_cp_space`). + Required exactly when ``compute_feature_metrics=true``; the caller loads it + so the same verified reference is stamped by :func:`save_metrics`. """ # Phase 1 runtime resolution: lock in executor + thread caps before any # heavy work. fov_workers may be provisional when "auto"; re-resolved in @@ -1294,9 +1306,15 @@ def evaluate_predictions(config: DictConfig, *, models: EvalModels | None = None OmegaConf.resolve(config) reset_timings() _validate_instance_ap_config(config) - # Before any model load or cache write: a missing or foreign-recipe CP reference - # must fail the run up front, not after the per-FOV loop. - cp_reference = _eval_cp_reference(config) if config.compute_feature_metrics else None + if config.compute_feature_metrics != (cp_space is not None): + raise ValueError( + f"cp_space must be given exactly when compute_feature_metrics=true " + f"(compute_feature_metrics={config.compute_feature_metrics}, cp_space={cp_space is not None})" + ) + # Before any model load or cache write: a GT CP cache re-cached after the reference + # was fit must fail the run up front, not after the per-FOV loop. + if cp_space is not None: + cp_space.check_gt_cache(OmegaConf.select(config, "io.gt_cache_dir", default=None)) use_gpu = bool(getattr(config, "use_gpu", True)) @@ -1335,6 +1353,7 @@ def evaluate_predictions(config: DictConfig, *, models: EvalModels | None = None seg_path = Path(io_config.cell_segmentation_path) if io_config.cell_segmentation_path is not None else None parent_lists: dict[str, _BackboneLists] = {name: _BackboneLists() for name in _BACKBONE_KEYS} + n_gt_cp_finite = [0] # summed across FOVs by _aggregate; checked against the CP scaler's fit # Deep-feature extractors by backbone key. Bound at function scope because the # precompute gate and the dataset-metrics block both derive from it and sit in @@ -1495,6 +1514,10 @@ def _keep(name: str) -> bool: t_counts["nuclei"] = nuclei_by_name[pos_name].data.shape[0] if len(set(t_counts.values())) > 1: raise ValueError(f"Timepoint count mismatch at position {pos_name!r}: {t_counts}") + # The CP scaler describes a fixed set of GT cells: refuse a position set it was not fit on + # before any per-FOV work (limit_positions / exclude_fov_names on a non-lite dataset). + if cp_space is not None: + cp_space.check_positions([name for name, _ in gt_positions]) # Leaf-level MicroMS3IM calibration: fit α once on a random # subsample of (FOV, t) volumes and reuse the fitted sim for @@ -1602,6 +1625,7 @@ def _keep(name: str) -> bool: extend_worker_timings = runtime.executor == "process" def _aggregate(result: FovResult) -> None: + n_gt_cp_finite[0] += result.n_gt_cp_finite _aggregate_fov_result( result, segmentation_results, @@ -1643,7 +1667,7 @@ def _aggregate(result: FovResult) -> None: celldino_feature_extractor, morphem_feature_extractor, microssim_sim, - cp_reference, + cp_space, predict_cached=cached_pair[0], target_cached=cached_pair[1], ) @@ -1682,7 +1706,7 @@ def _aggregate(result: FovResult) -> None: pos_name, runtime.cuda_empty_cache_every_n_timepoints, microssim_sim, - cp_reference, + cp_space, ): pos_name for pos_name in pos_names_in_order } @@ -1726,7 +1750,9 @@ def _aggregate(result: FovResult) -> None: prefix_inputs: list[tuple[str, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]] = [] if parent_lists["cp"].pred_feats: - prefix_inputs.append(_stage_cp_dataset_inputs(parent_lists["cp"], cp_reference, save_dir)) + prefix_inputs.append( + _stage_cp_dataset_inputs(parent_lists["cp"], cp_space, n_gt_cp_finite[0], save_dir) + ) for key in deep_kinds: # cp is handled above (reference space) bb = parent_lists[key] @@ -1799,8 +1825,26 @@ def _compute_one(args): return all_pixel_metrics, all_mask_metrics, all_feature_metrics -def save_metrics(config: DictConfig, pixel_metrics=None, mask_metrics=None, feature_metrics=None): - """Save metrics to files.""" +def save_metrics( + config: DictConfig, + pixel_metrics=None, + mask_metrics=None, + feature_metrics=None, + *, + cp_reference_sha256: str | None, +): + """Save metrics to files. + + ``cp_reference_sha256`` is the hash of the CP reference the run SCORED with + (``DatasetCPSpace.reference_sha256``; ``None`` without feature metrics). It is + passed in rather than re-read here, so a reference rebuilt mid-run cannot be + stamped on values it did not produce. + """ + if config.compute_feature_metrics != (cp_reference_sha256 is not None): + raise ValueError( + "cp_reference_sha256 must be given exactly when compute_feature_metrics=true " + f"(compute_feature_metrics={config.compute_feature_metrics})" + ) save_dir = Path(config.save.save_dir) save_dir.mkdir(parents=True, exist_ok=True) @@ -1823,12 +1867,7 @@ def save_metrics(config: DictConfig, pixel_metrics=None, mask_metrics=None, feat # whether the cache is comparable instead of assuming it is. Stamping first meant a # crash partway through the loop above left a fresh stamp certifying a previous # run's rows -- the exact misattribution the sidecar exists to make detectable. - write_metrics_provenance(save_dir, cp_reference_sha256=_current_cp_reference_sha256(config)) - - -def _current_cp_reference_sha256(config: DictConfig) -> str | None: - """Content hash of the CP reference this config scores in; ``None`` without feature metrics.""" - return _eval_cp_reference(config).sha256 if config.compute_feature_metrics else None + write_metrics_provenance(save_dir, cp_reference_sha256=cp_reference_sha256) #: Pixel columns that only a dual-scaling run writes. Their absence marks a pixel @@ -1859,7 +1898,8 @@ def _final_metrics_cache_valid(config: DictConfig) -> bool: # An unstamped dir has nothing to reuse, so it needs no reference to be rejected. if not (save_dir / PROVENANCE_FILENAME).is_file(): return False - if not metrics_provenance_matches(save_dir, cp_reference_sha256=_current_cp_reference_sha256(config)): + current_sha256 = eval_cp_space(config).reference_sha256 if config.compute_feature_metrics else None + if not metrics_provenance_matches(save_dir, cp_reference_sha256=current_sha256): return False pixel_ok = (save_dir / config.save.pixel_metrics_filename).exists() mask_path = save_dir / config.save.mask_metrics_filename @@ -2170,12 +2210,16 @@ def get_models() -> EvalModels: print(f"[grouped] ({idx + 1}/{len(conditions)}) {name!r}: reusing cached final metrics") pixel_metrics, mask_metrics, feature_metrics = _load_cached_final_metrics(merged) else: - pixel_metrics, mask_metrics, feature_metrics = evaluate_predictions(merged, models=get_models()) + cp_space = eval_cp_space(merged) if merged.compute_feature_metrics else None + pixel_metrics, mask_metrics, feature_metrics = evaluate_predictions( + merged, models=get_models(), cp_space=cp_space + ) save_metrics( merged, pixel_metrics=pixel_metrics, mask_metrics=mask_metrics, feature_metrics=feature_metrics, + cp_reference_sha256=cp_space.reference_sha256 if cp_space is not None else None, ) results.append((name, (pixel_metrics, mask_metrics, feature_metrics))) condition_save_dirs.append(Path(merged.save.save_dir)) @@ -2215,13 +2259,17 @@ def evaluate_model(config: DictConfig): print("Found existing metrics.") pixel_metrics, mask_metrics, feature_metrics = _load_cached_final_metrics(config) else: - pixel_metrics, mask_metrics, feature_metrics = evaluate_predictions(config) + # Loaded here, before any model load, and threaded to both calls so the hash + # stamped by save_metrics is the reference the metrics were scored with. + cp_space = eval_cp_space(config) if config.compute_feature_metrics else None + pixel_metrics, mask_metrics, feature_metrics = evaluate_predictions(config, cp_space=cp_space) with region_timer("save_metrics_csvs", ""): save_metrics( config, pixel_metrics=pixel_metrics, mask_metrics=mask_metrics, feature_metrics=feature_metrics, + cp_reference_sha256=cp_space.reference_sha256 if cp_space is not None else None, ) # Re-dump so save_metrics_csvs lands in eval_timing.csv. evaluate_predictions # dumps once before save_metrics runs; this second dump overwrites with the diff --git a/applications/dynacell/tests/_eval_fixtures.py b/applications/dynacell/tests/_eval_fixtures.py index 3a190d4ad..25a0e0f11 100644 --- a/applications/dynacell/tests/_eval_fixtures.py +++ b/applications/dynacell/tests/_eval_fixtures.py @@ -207,33 +207,50 @@ def build_fixture(root: Path): return pred_path, gt_path, gt_cache_dir, pred_cache_dir -def make_cp_reference(config, path: Path, seed: int = 0) -> str: +#: Dataset name the synthetic CP reference fits a scaler for (``benchmark.dataset_ref.dataset``). +CP_TEST_DATASET = "synthetic-set" + + +def make_cp_reference(config, path: Path, seed: int = 0, n_cells: int = 64) -> str: """Fit a real CP reference matching ``config``'s CP recipe and point ``config`` at it. - The GT matrix is synthetic but the reference is built by the production - :func:`~dynacell.evaluation.cp_reference.fit_cp_reference`, with the identity and - column names ``evaluate_predictions`` derives from the same config. Sets - ``config.feature_metrics.cp.reference_path`` and returns the reference's sha256. + The GT matrix is synthetic, but the reference is built by the production + :func:`~dynacell.evaluation.cp_reference.fit_cp_reference` with the identity and + column names :func:`~dynacell.evaluation.cp_reference.cp_space` derives from the + same config. It fits one scaler, for :data:`CP_TEST_DATASET` over the plate's + ``A/1/{i}`` positions, and records no GT cache stamp (so the re-cache check is + skipped). Sets ``feature_metrics.cp`` (GLCM off unless already configured), + ``feature_metrics.cp.reference_path`` and ``benchmark.dataset_ref.dataset``, and + returns the reference's sha256. """ - from dynacell.evaluation.cp_reference import CP_REFERENCE_DIMENSION, fit_cp_reference, write_cp_reference - from dynacell.evaluation.metrics import CP_FEATURE_VERSION, active_cp_feature_names - from dynacell.evaluation.pipeline_cache import cp_recipe_identity - - cp = OmegaConf.select(config, "feature_metrics.cp", default=None) - norm = OmegaConf.to_container(cp.norm, resolve=True) if cp is not None and "norm" in cp else {} - glcm = OmegaConf.to_container(cp.glcm, resolve=True) if cp is not None and "glcm" in cp else {} - names = tuple(active_cp_feature_names(bool(glcm.get("enabled", False)))) - gt = np.random.default_rng(seed).standard_normal((64, len(names))) + # Imported here like the other helpers in this file: live_pipeline_module() drops + # every dynacell.evaluation.* module, so a top-level binding would go stale. + from dynacell.evaluation.cp_reference import DatasetFit, cp_space, fit_cp_reference, write_cp_reference + + if OmegaConf.select(config, "feature_metrics.cp", default=None) is None: + OmegaConf.update( + config, + "feature_metrics.cp", + {"norm": {"p_lo": 1.0, "p_hi": 99.0}, "glcm": {"enabled": False}, "reference_path": None}, + merge=True, + ) + OmegaConf.update(config, "benchmark", {"dataset_ref": {"dataset": CP_TEST_DATASET, "target": "sec61b"}}, merge=True) + identity, names = cp_space(config) + fit = DatasetFit( + dataset=CP_TEST_DATASET, + cells=np.random.default_rng(seed).standard_normal((n_cells, len(names))), + record={ + "positions": [f"A/1/{i}" for i in range(N_POSITIONS)], + "gt_cache_dir": None, + "cp_cache_built_at": None, + }, + in_mask_fit=True, + ) payload = fit_cp_reference( - gt, - target_name=config.target_name, - dimension=CP_REFERENCE_DIMENSION, - feature_names=names, - cp_identity=cp_recipe_identity(CP_FEATURE_VERSION, norm, glcm), - datasets=[{"dataset": "synthetic", "n_cells": gt.shape[0]}], + [fit], target_name=config.target_name, feature_names=names, cp_identity=identity, lite={} ) - write_cp_reference(payload, path) - OmegaConf.update(config, "feature_metrics.cp.reference_path", str(path), merge=True) + write_cp_reference(payload, path, force=True) + config.feature_metrics.cp.reference_path = str(path) return payload["sha256"] diff --git a/applications/dynacell/tests/test_evaluate_compose.py b/applications/dynacell/tests/test_evaluate_compose.py index 7a6ab733b..1c18e6c3e 100644 --- a/applications/dynacell/tests/test_evaluate_compose.py +++ b/applications/dynacell/tests/test_evaluate_compose.py @@ -178,7 +178,7 @@ def test_evaluate_model_wires_hook(monkeypatch, tmp_path) -> None: """``evaluate_model`` runs ``apply_dataset_ref`` before ``evaluate_predictions``.""" captured: list[DictConfig] = [] - def _fake_evaluate_predictions(cfg: DictConfig): + def _fake_evaluate_predictions(cfg: DictConfig, *, cp_space): captured.append(cfg) return ([], [], []) @@ -188,12 +188,15 @@ def _fake_save_metrics(*_args, **_kwargs) -> None: monkeypatch.setattr("dynacell.evaluation.pipeline.evaluate_predictions", _fake_evaluate_predictions) monkeypatch.setattr("dynacell.evaluation.pipeline.save_metrics", _fake_save_metrics) + # Feature metrics off: evaluate_model would otherwise load the CP reference before + # evaluate_predictions, and this test is about the dataset_ref splice only. cfg = _compose_eval_cfg( [ "target=er_sec61b", "predict_set=ipsc_confocal", "io.pred_path=/tmp/fake", f"save.save_dir={tmp_path}", + "compute_feature_metrics=false", ] ) diff --git a/applications/dynacell/tests/test_evaluation_pipeline_parallel.py b/applications/dynacell/tests/test_evaluation_pipeline_parallel.py index c0b8e526d..f414a8656 100644 --- a/applications/dynacell/tests/test_evaluation_pipeline_parallel.py +++ b/applications/dynacell/tests/test_evaluation_pipeline_parallel.py @@ -18,10 +18,10 @@ from __future__ import annotations import importlib +import inspect import pickle import numpy as np -import pytest def _live_pipeline_module(): @@ -135,36 +135,6 @@ def test_fov_result_pickle_handles_empty_backbones(): assert restored.celldino.gt_feats == [] -def test_cp_reference_raises_on_dim_mismatch(tmp_path): - """A CP matrix of another width raises an actionable StaleCacheError, not IndexError. - - A stale CP cache built with a different recipe (e.g. a GLCM toggle or a - CP_FEATURE_VERSION change without a rebuild) would otherwise crash the - boolean-mask indexing with a cryptic IndexError; it must instead name the remedy. - """ - from dynacell.evaluation.cache import StaleCacheError - from dynacell.evaluation.cp_reference import ( - CP_REFERENCE_DIMENSION, - fit_cp_reference, - load_cp_reference, - write_cp_reference, - ) - - names = tuple(f"f{i}" for i in range(22)) - payload = fit_cp_reference( - np.random.default_rng(0).standard_normal((40, 22)), - target_name="er", - dimension=CP_REFERENCE_DIMENSION, - feature_names=names, - cp_identity={}, - datasets=[{"n_cells": 40}], - ) - write_cp_reference(payload, tmp_path / "ref.json") - ref = load_cp_reference(tmp_path / "ref.json", target_name="er", dimension=CP_REFERENCE_DIMENSION) - with pytest.raises(StaleCacheError, match="CP feature dimension mismatch"): - ref.transform(np.ones((4, 58), dtype=np.float32)) - - def test_aggregate_fov_result_extends_backbone_lists(): """Aggregator must extend each backbone's six lists with worker contributions.""" pipeline = _live_pipeline_module() @@ -218,3 +188,57 @@ def create_position(self, row, col, fov): assert len(bb.gt_fovs) == 2 assert len(bb.pred_ts) == 2 assert len(bb.gt_ts) == 2 + + +def test_worker_run_fov_hands_the_cp_space_to_process_one_fov(tmp_path, monkeypatch): + """``_worker_run_fov`` forwards the parent's CP space, and the space survives the pickle hop. + + Under ``executor=process`` the parent ships the verified CP space with every + submission; the worker must score with that object, not reload or drop it. + """ + from dynacell.evaluation.cp_reference import DatasetFit, fit_cp_reference, load_cp_reference, write_cp_reference + + from ._eval_fixtures import build_eval_config, make_hcs_plate + + pipeline = _live_pipeline_module() + names = tuple(f"f{i}" for i in range(5)) + fit = DatasetFit( + dataset="ds", + cells=np.random.default_rng(0).standard_normal((40, 5)), + record={"positions": ["A/1/0"], "gt_cache_dir": None, "cp_cache_built_at": None}, + in_mask_fit=True, + ) + write_cp_reference( + fit_cp_reference([fit], target_name="er", feature_names=names, cp_identity={}, lite={}), tmp_path / "er.json" + ) + cp_space = pickle.loads(pickle.dumps(load_cp_reference(tmp_path / "er.json", target_name="er").for_dataset("ds"))) + + make_hcs_plate(tmp_path / "pred.zarr", "prediction", seed=0, n_positions=1) + make_hcs_plate(tmp_path / "gt.zarr", "target", seed=1, n_positions=1) + config = build_eval_config( + tmp_path / "pred.zarr", + tmp_path / "gt.zarr", + tmp_path / "g", + tmp_path / "p", + tmp_path, + executor="process", + fov_workers=1, + ) + seen = {} + signature = inspect.signature(pipeline._process_one_fov) + + def _capture(*args, **kwargs): + seen["cp_space"] = signature.bind(*args, **kwargs).arguments["cp_space"] + return "result" + + monkeypatch.setattr(pipeline, "_worker_setup", lambda cfg: None) + monkeypatch.setattr(pipeline, "_process_one_fov", _capture) + monkeypatch.setattr(pipeline, "flush_manifest", lambda ctx: None) + monkeypatch.setitem(pipeline._WORKER_STATE, "cache_ctx", None) + monkeypatch.setitem(pipeline._WORKER_STATE, "pred_cache_ctx", None) + for key in ("seg_model", "dinov3", "dynaclr", "celldino", "morphem"): + monkeypatch.setitem(pipeline._WORKER_STATE, key, None) + + assert pipeline._worker_run_fov(config, "A/1/0", 0, None, cp_space) == "result" + assert seen["cp_space"] is cp_space + np.testing.assert_array_equal(seen["cp_space"].mean, cp_space.mean) diff --git a/applications/dynacell/tests/test_evaluation_timepoint_guard.py b/applications/dynacell/tests/test_evaluation_timepoint_guard.py index 78eef0536..5cccbd4fc 100644 --- a/applications/dynacell/tests/test_evaluation_timepoint_guard.py +++ b/applications/dynacell/tests/test_evaluation_timepoint_guard.py @@ -9,15 +9,20 @@ from __future__ import annotations +import json +from importlib.metadata import version from pathlib import Path import numpy as np import pytest from iohub.ngff import open_ome_zarr -from dynacell.evaluation.provenance import write_metrics_provenance +from dynacell.evaluation.cache import cache_paths, save_manifest +from dynacell.evaluation.cp_reference import payload_sha256 +from dynacell.evaluation.model_loader import EvalModels +from dynacell.evaluation.provenance import PROVENANCE_FILENAME, write_metrics_provenance -from ._eval_fixtures import build_eval_config, live_pipeline_module, make_cp_reference +from ._eval_fixtures import CP_TEST_DATASET, build_eval_config, live_pipeline_module, make_cp_reference D, H, W = 3, 8, 8 @@ -87,7 +92,11 @@ def test_matched_per_position_t_passes(tmp_path: Path, monkeypatch) -> None: def _feature_cache_config(tmp_path: Path, **flags: bool): - """Config with feature metrics on and the given ``feature_metrics.compute_*`` flags.""" + """Config with feature metrics on, the given ``feature_metrics.compute_*`` flags, and a CP reference. + + Returns ``(config, reference_sha256)``; the reference is built once, at + ``tmp_path / "cp_reference.json"``. + """ config = build_eval_config( tmp_path / "pred.zarr", tmp_path / "gt.zarr", @@ -100,17 +109,14 @@ def _feature_cache_config(tmp_path: Path, **flags: bool): config.compute_feature_metrics = True for name, value in flags.items(): config.feature_metrics[name] = value - make_cp_reference(config, tmp_path / "cp_reference.json") - return config + return config, make_cp_reference(config, tmp_path / "cp_reference.json") -def _write_final_caches(save_dir: Path, feature_row: dict) -> None: - """Write a stamped pixel/mask/feature NPY cache set holding one feature row.""" +def _write_final_caches(save_dir: Path, feature_row: dict, reference_sha256: str) -> None: + """Write a pixel/mask/feature NPY cache set holding one feature row, stamped with ``reference_sha256``.""" np.save(save_dir / "pixel_metrics.npy", [{"SI_PSNR": 1.0, "SI_SSIM": 1.0, "SI_NRMSE": 1.0}]) np.save(save_dir / "mask_metrics.npy", [{"metric": "mask"}]) np.save(save_dir / "feature_metrics.npy", [feature_row]) - # The stamp must carry the hash of the reference _feature_cache_config points at. - reference_sha256 = make_cp_reference(_feature_cache_config(save_dir), save_dir / "cp_reference.json") write_metrics_provenance(save_dir, cp_reference_sha256=reference_sha256) @@ -129,18 +135,20 @@ def _dataset_row(prefixes: tuple[str, ...], families: tuple[str, ...]) -> dict: def test_cache_without_fid_is_invalid_once_fid_is_enabled(tmp_path: Path) -> None: """A dir written with FID off is recomputed when FID is on, reused while it stays off.""" pipeline = live_pipeline_module() - no_fid = tuple(f for f in _ALL_FAMILIES if f != "FID") - _write_final_caches(tmp_path, _dataset_row(("CP", "DINOv3"), no_fid)) + config, sha256 = _feature_cache_config(tmp_path) + _write_final_caches(tmp_path, _dataset_row(("CP", "DINOv3"), tuple(f for f in _ALL_FAMILIES if f != "FID")), sha256) - assert not pipeline._final_metrics_cache_valid(_feature_cache_config(tmp_path)) - assert pipeline._final_metrics_cache_valid(_feature_cache_config(tmp_path, compute_fid=False)) + assert not pipeline._final_metrics_cache_valid(config) + config.feature_metrics.compute_fid = False + assert pipeline._final_metrics_cache_valid(config) def test_full_cache_stays_valid(tmp_path: Path) -> None: """A dir carrying every family for every prefix it scored remains reusable.""" pipeline = live_pipeline_module() - _write_final_caches(tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES)) - assert pipeline._final_metrics_cache_valid(_feature_cache_config(tmp_path)) + config, sha256 = _feature_cache_config(tmp_path) + _write_final_caches(tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES), sha256) + assert pipeline._final_metrics_cache_valid(config) def test_real_full_benchmark_columns_stay_valid(tmp_path: Path) -> None: @@ -151,57 +159,147 @@ def test_real_full_benchmark_columns_stay_valid(tmp_path: Path) -> None: with real.open() as f: header = f.readline().rstrip("\n").split(",") pipeline = live_pipeline_module() - _write_final_caches(tmp_path, dict.fromkeys(header, 0.0)) - assert pipeline._final_metrics_cache_valid(_feature_cache_config(tmp_path)) + config, sha256 = _feature_cache_config(tmp_path) + _write_final_caches(tmp_path, dict.fromkeys(header, 0.0), sha256) + assert pipeline._final_metrics_cache_valid(config) def test_cache_scored_in_another_cp_reference_is_invalid(tmp_path: Path) -> None: """Rebuilding the CP reference invalidates every final-metrics cache stamped with the old one.""" pipeline = live_pipeline_module() - _write_final_caches(tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES)) - config = _feature_cache_config(tmp_path) + config, sha256 = _feature_cache_config(tmp_path) + _write_final_caches(tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES), sha256) assert pipeline._final_metrics_cache_valid(config) make_cp_reference(config, tmp_path / "cp_reference.json", seed=1) # same path, new content assert not pipeline._final_metrics_cache_valid(config) -def test_missing_cp_reference_fails_before_any_work(tmp_path: Path, monkeypatch) -> None: - """A feature-metrics eval with no CP reference raises, naming the build command, before loading models.""" +def test_feature_less_cache_ignores_the_cp_reference(tmp_path: Path) -> None: + """A compute_feature_metrics=false dir with no CP hash in its stamp stays reusable, with no reference.""" pipeline = live_pipeline_module() - config = _feature_cache_config(tmp_path) - config.feature_metrics.cp.reference_path = str(tmp_path / "absent.json") + config = build_eval_config( + tmp_path / "pred.zarr", + tmp_path / "gt.zarr", + tmp_path / "g", + tmp_path / "p", + tmp_path, + executor="serial", + fov_workers=1, + ) + np.save(tmp_path / "pixel_metrics.npy", [{"SI_PSNR": 1.0, "SI_SSIM": 1.0, "SI_NRMSE": 1.0}]) + np.save(tmp_path / "mask_metrics.npy", [{"metric": "mask"}]) + (tmp_path / PROVENANCE_FILENAME).write_text(json.dumps({"versions": {"cubic": version("cubic")}})) + assert pipeline._final_metrics_cache_valid(config) - def _no_models(*args, **kwargs): - raise AssertionError("models loaded before the CP reference was checked") - monkeypatch.setattr(pipeline, "load_eval_models", _no_models) +def test_missing_cp_reference_names_the_build_command(tmp_path: Path) -> None: + """Binding an eval with no CP reference raises FileNotFoundError with the build command.""" + pipeline = live_pipeline_module() + config, _ = _feature_cache_config(tmp_path) + config.feature_metrics.cp.reference_path = str(tmp_path / "absent.json") with pytest.raises(FileNotFoundError, match="build_cp_reference.py --target er"): - pipeline.evaluate_predictions(config) + pipeline.eval_cp_space(config) -def test_cp_reference_of_another_recipe_is_refused(tmp_path: Path, monkeypatch) -> None: +def test_cp_reference_of_another_recipe_is_refused(tmp_path: Path) -> None: """A reference fit on GLCM-off caches cannot score a GLCM-on eval.""" pipeline = live_pipeline_module() - # Import after the fresh pipeline import, which re-creates dynacell.evaluation.* modules. - from dynacell.evaluation.cache import StaleCacheError - - config = _feature_cache_config(tmp_path) # reference built for this (GLCM-off) recipe + config, _ = _feature_cache_config(tmp_path) # reference built for this (GLCM-off) recipe config.feature_metrics.cp.glcm = {"enabled": True, "levels": 32, "distances": [1]} - monkeypatch.setattr(pipeline, "load_eval_models", lambda *a, **k: pytest.fail("models loaded")) - with pytest.raises(StaleCacheError, match="different CP recipe"): - pipeline.evaluate_predictions(config) + with pytest.raises(Exception, match="different CP recipe") as err: + pipeline.eval_cp_space(config) + # Checked by name: live_pipeline_module() re-creates dynacell.evaluation.cache, so the + # class imported at the top of this file is not the one the fresh module raises. + assert type(err.value).__name__ == "StaleCacheError" -def test_save_metrics_stamps_the_cp_reference_hash(tmp_path: Path) -> None: - """``save_metrics`` records the hash of the reference the CP metrics were scored in.""" - import json +def test_feature_metrics_need_a_cp_space(tmp_path: Path) -> None: + """``evaluate_predictions`` refuses feature metrics without the caller's CP space, before any model load.""" + pipeline = live_pipeline_module() + config, _ = _feature_cache_config(tmp_path) + with pytest.raises(ValueError, match="cp_space must be given exactly when"): + pipeline.evaluate_predictions(config) - from dynacell.evaluation.provenance import PROVENANCE_FILENAME +def test_evaluate_model_stamps_the_reference_it_scored_with(tmp_path: Path, monkeypatch) -> None: + """The hash stamped by save_metrics is the reference loaded before scoring, never a re-read. + + The reference file is rebuilt while "scoring" runs; the stamp must still carry the + hash of the space evaluate_predictions received. + """ pipeline = live_pipeline_module() - config = _feature_cache_config(tmp_path) - reference_sha256 = make_cp_reference(config, tmp_path / "cp_reference.json") - pipeline.save_metrics(config, pixel_metrics=[{"FOV": "A/1/0", "Timepoint": 0, "PCC": 0.5}]) + config, sha256 = _feature_cache_config(tmp_path) + config.force_recompute.final_metrics = True + seen = {} + + def _fake_evaluate_predictions(cfg, *, cp_space): + seen["cp_space"] = cp_space + make_cp_reference(cfg, tmp_path / "cp_reference.json", seed=7) # rebuilt mid-run + return [{"FOV": "A/1/0", "Timepoint": 0, "PCC": 0.5}], [], [] + + monkeypatch.setattr(pipeline, "check_cubic_pin", lambda: None) + monkeypatch.setattr(pipeline, "apply_dataset_ref", lambda cfg: None) + monkeypatch.setattr(pipeline, "evaluate_predictions", _fake_evaluate_predictions) + getattr(pipeline.evaluate_model, "__wrapped__", pipeline.evaluate_model)(config) + + assert seen["cp_space"].reference_sha256 == sha256 stamp = json.loads((tmp_path / PROVENANCE_FILENAME).read_text()) - assert stamp["cp_reference_sha256"] == reference_sha256 + assert stamp["cp_reference_sha256"] == sha256 + + +def _feature_plates(tmp_path: Path) -> None: + """Pred/GT/segmentation plates with three positions, as the feature path needs.""" + _make_plate(tmp_path / "pred.zarr", "prediction", [2, 2, 2]) + _make_plate(tmp_path / "gt.zarr", "target", [2, 2, 2]) + _make_plate(tmp_path / "seg.zarr", "cell_segmentation", [2, 2, 2]) + + +def _stub_models() -> EvalModels: + """A model bundle whose extractors are never called: the run must stop before the FOV loop.""" + return EvalModels( + seg_model=None, + dinov3=object(), + dynaclr=object(), + celldino=None, + morphem=None, + dinov3_model_name=None, + dynaclr_ckpt_path=None, + dynaclr_encoder_cfg=None, + celldino_weights_path=None, + morphem_model_name=None, + ) + + +def test_partial_position_walk_is_refused_for_cp(tmp_path: Path, monkeypatch) -> None: + """``limit_positions`` on a non-lite dataset fails before any per-FOV work: its CP cells are not the fit's.""" + pipeline = live_pipeline_module() + _feature_plates(tmp_path) + config, _ = _feature_cache_config(tmp_path) + config.io.cell_segmentation_path = str(tmp_path / "seg.zarr") + config.limit_positions = 2 + monkeypatch.setattr(pipeline, "_calibrate_microssim", lambda *a, **k: pytest.fail("reached per-FOV work")) + cp_space = pipeline.eval_cp_space(config) + with pytest.raises(ValueError, match="cannot be combined with compute_feature_metrics"): + pipeline.evaluate_predictions(config, models=_stub_models(), cp_space=cp_space) + + +def test_gt_recache_after_the_reference_is_refused_up_front(tmp_path: Path, monkeypatch) -> None: + """A GT CP cache whose ``built_at`` moved since the build fails before any model load.""" + pipeline = live_pipeline_module() + config, _ = _feature_cache_config(tmp_path) + save_manifest( + cache_paths(tmp_path / "gt_cache"), + {"artifacts": {"cp_features": {"path": "features/cp.zarr", "built_at": "2026-09-26T00:00:00+00:00"}}}, + ) + reference = json.loads((tmp_path / "cp_reference.json").read_text()) + reference["scalers"][CP_TEST_DATASET].update( + gt_cache_dir=str(tmp_path / "gt_cache"), cp_cache_built_at="2026-09-01T00:00:00+00:00" + ) + reference["sha256"] = payload_sha256(reference) + (tmp_path / "cp_reference.json").write_text(json.dumps(reference)) + monkeypatch.setattr(pipeline, "load_eval_models", lambda *a, **k: pytest.fail("models loaded")) + cp_space = pipeline.eval_cp_space(config) + with pytest.raises(Exception, match="was built at 2026-09-26") as err: + pipeline.evaluate_predictions(config, cp_space=cp_space) + assert type(err.value).__name__ == "StaleCacheError" diff --git a/applications/dynacell/tools/build_cp_reference.py b/applications/dynacell/tools/build_cp_reference.py index d3ab6e632..d29afaa9c 100644 --- a/applications/dynacell/tools/build_cp_reference.py +++ b/applications/dynacell/tools/build_cp_reference.py @@ -1,152 +1,151 @@ r"""Build a target's CP (GLCM+) reference from existing GT CP feature caches. -The reference fixes the CP feature space every eval of a target is scored in: a -feature keep-mask selected on GT cells only, plus one shared per-feature mean/std -over the same pooled GT (see :mod:`dynacell.evaluation.cp_reference`). It is fit -on the target's benchmark test sets -- iPSC + A549 mock/denv/zikv; HEK and the -movie sets are excluded. +The reference fixes the CP feature space every eval of a target is scored in (see +:mod:`dynacell.evaluation.cp_reference`): + +* one feature keep-mask selected on GT cells only, pooled over the target's + ``MASK_FIT_DATASETS`` (iPSC + A549 mock/denv/zikv); +* one GT scaler per test set, for EVERY dataset a grouped leaf of the target + evaluates on (``benchmark.dataset_ref.dataset`` under ``--leaves-root``), + HEK included (its own scaler, outside the mask fit); +* lite datasets mapped to their parent's scaler, the parent read from the lite + split's ``selection_criteria.lite_of``. No scaler is fit on lite cells. This tool only READS. Each dataset ref is resolved exactly as an eval resolves it (``apply_dataset_ref`` on the packaged ``eval.yaml`` defaults), the GT cache is opened through the pipeline's own ``init_cache_context`` with -``require_complete_cache=true`` (so a cache of another dataset or CP recipe is +``require_complete_cache=true`` (so a cache of another store or CP recipe is refused), and every ``(position, timepoint)`` of the GT store must already be in the CP cache. A miss raises; features are never computed. Non-finite cells are dropped the way the pipeline drops them (``drop_paired_nonfinite_rows``). +The write is atomic, and an existing reference with a different content hash is +only replaced under ``--force`` (an identical one is left untouched). + Run:: - uv run --no-sync python applications/dynacell/tools/build_cp_reference.py \ - --target er --dry-run - uv run --no-sync python applications/dynacell/tools/build_cp_reference.py \ - --target er # writes the registry path under DATA_ROOT - uv run --no-sync python applications/dynacell/tools/build_cp_reference.py \ - --target er --out /tmp/er__3d.json + uv run --no-sync python applications/dynacell/tools/build_cp_reference.py --target er --dry-run + uv run --no-sync python applications/dynacell/tools/build_cp_reference.py --target er # registry path + uv run --no-sync python applications/dynacell/tools/build_cp_reference.py --target er --out /tmp/er.json """ from __future__ import annotations import argparse import time -from dataclasses import dataclass from pathlib import Path from typing import Any import numpy as np +import yaml from iohub.ngff import open_ome_zarr from omegaconf import DictConfig, OmegaConf import dynacell.evaluation +from dynacell.data.manifests import DatasetRef, load_splits +from dynacell.data.manifests import load_manifest as load_dataset_manifest +from dynacell.data.resolver import resolve_dataset_ref from dynacell.evaluation._ref_hook import apply_dataset_ref -from dynacell.evaluation.cache import StaleCacheError, open_features_group, read_features_from_group +from dynacell.evaluation.cache import ( + StaleCacheError, + cache_paths, + load_manifest, + open_features_group, + read_features_from_group, +) from dynacell.evaluation.cp_reference import ( - CP_REFERENCE_DIMENSION, + MASK_FIT_DATASETS, + DatasetFit, + cp_space, fit_cp_reference, write_cp_reference, ) -from dynacell.evaluation.metrics import CP_FEATURE_VERSION, active_cp_feature_names, drop_paired_nonfinite_rows +from dynacell.evaluation.metrics import drop_paired_nonfinite_rows from dynacell.evaluation.paths import cp_reference_path -from dynacell.evaluation.pipeline_cache import cp_recipe_identity, init_cache_context - -#: The fit set per eval ``target_name``: iPSC + A549 mock/denv/zikv, as -#: ``(dataset, manifest target)`` refs -- the same refs the grouped leaves carry in -#: ``benchmark.dataset_ref``. HEK and the movie sets are deliberately absent. -DEFAULT_DATASETS: dict[str, tuple[tuple[str, str], ...]] = { - "nucleus": ( - ("aics-hipsc", "nucleus"), - ("a549-mantis-h2b-mock", "h2b"), - ("a549-mantis-h2b-denv", "h2b"), - ("a549-mantis-h2b-zikv", "h2b"), - ), - "membrane": ( - ("aics-hipsc", "membrane"), - ("a549-mantis-caax-mock", "caax"), - ("a549-mantis-caax-denv", "caax"), - ("a549-mantis-caax-zikv", "caax"), - ), - "er": ( - ("aics-hipsc", "sec61b"), - ("a549-mantis-sec61b-mock", "sec61b"), - ("a549-mantis-sec61b-denv", "sec61b"), - ("a549-mantis-sec61b-zikv", "sec61b"), - ), - "mitochondria": ( - ("aics-hipsc", "tomm20"), - ("a549-mantis-tomm20-mock", "tomm20"), - ("a549-mantis-tomm20-denv", "tomm20"), - ("a549-mantis-tomm20-zikv", "tomm20"), - ), -} +from dynacell.evaluation.pipeline_cache import init_cache_context _EVAL_YAML = Path(dynacell.evaluation.__file__).parent / "_configs" / "eval.yaml" +_LEAVES_ROOT = Path(__file__).resolve().parents[1] / "configs/benchmarks/virtual_staining/_internal/leaf/grouped" - -@dataclass -class DatasetCells: - """Finite GT CP cells read from one dataset's cache, plus what was read.""" - - record: dict[str, Any] - cells: np.ndarray +Ref = tuple[str, str] -def eval_config_for(target_name: str, dataset: str, manifest_target: str) -> DictConfig: +def eval_config_for(target_name: str, ref: Ref) -> DictConfig: """Return the packaged eval defaults resolved for one dataset ref, cache reads mandatory.""" config = OmegaConf.load(_EVAL_YAML) config.target_name = target_name - config.benchmark = {"dataset_ref": {"dataset": dataset, "target": manifest_target}} + config.benchmark = {"dataset_ref": {"dataset": ref[0], "target": ref[1]}} config.io.require_complete_cache = True apply_dataset_ref(config) return config -def cp_identity_for(config: DictConfig) -> tuple[dict[str, Any], tuple[str, ...]]: - """Return the CP recipe identity and column names an eval of ``config`` would use.""" - norm = OmegaConf.to_container(config.feature_metrics.cp.norm, resolve=True) - glcm = OmegaConf.to_container(config.feature_metrics.cp.glcm, resolve=True) - return ( - cp_recipe_identity(CP_FEATURE_VERSION, norm, glcm), - tuple(active_cp_feature_names(bool(glcm["enabled"]))), - ) +def leaf_dataset_refs(target_name: str, leaves_root: Path) -> set[Ref]: + """Return every ``(dataset, manifest target)`` a grouped leaf of ``target_name`` evaluates on. + + Raises + ------ + ValueError + If no grouped leaf of the target is found. + """ + refs: set[Ref] = set() + for leaf in sorted(leaves_root.glob("*/eval_grouped.yaml")): + doc = yaml.safe_load(leaf.read_text()) + if doc.get("target_name") != target_name: + continue + for cond in doc["conditions"]: + ref = cond["benchmark"]["dataset_ref"] + refs.add((ref["dataset"], ref["target"])) + if not refs: + raise ValueError(f"no grouped leaf under {leaves_root} evaluates target {target_name!r}") + return refs + + +def lite_parent(ref: Ref) -> str | None: + """Return the parent dataset a lite dataset was subset from, or ``None`` for a non-lite one. + + Read from the recorded provenance -- the dataset split's + ``selection_criteria.lite_of`` written by ``generate_lite_benchmark_configs.py`` + -- never inferred from the name. Only lite splits carry that key. + """ + resolved = resolve_dataset_ref(DatasetRef(dataset=ref[0], target=ref[1])) + manifest = load_dataset_manifest(resolved.manifest_path) + splits = load_splits(resolved.manifest_path.parent / manifest.targets[ref[1]].splits) + return (splits.selection_criteria or {}).get("lite_of") -def read_dataset_cells(config: DictConfig, n_features: int) -> DatasetCells: - """Read every GT CP cell of one dataset from its cache, read-only. +def _gt_positions(config: DictConfig) -> dict[str, tuple[int, int]]: + """Return ``{position: (T, Z)}`` of the eval's GT store, metadata only.""" + with open_ome_zarr(Path(config.io.gt_path), mode="r") as plate: + return {name: (int(pos.data.shape[0]), int(pos.data.shape[2])) for name, pos in plate.positions()} - Parameters - ---------- - config : DictConfig - Output of :func:`eval_config_for`. - n_features : int - Expected CP column count. - Returns - ------- - DatasetCells - Finite cells stacked in position/timepoint order, and the provenance record. +def read_dataset_fit(target_name: str, ref: Ref, n_features: int, in_mask_fit: bool) -> DatasetFit: + """Read every finite GT CP cell of one dataset from its cache, read-only. Raises ------ StaleCacheError - If the cache belongs to another store or CP recipe, a ``(position, - timepoint)`` of the GT store is missing from it, or an entry has the wrong - column count. + If the cache belongs to another store or CP recipe, records no + ``cp_features`` entry, a ``(position, timepoint)`` of the GT store is + missing from it, or an entry has the wrong column count. ValueError If a GT position is not a 3-D volume. """ + config = eval_config_for(target_name, ref) ctx = init_cache_context(config, side="gt") + entry = ctx.manifest["artifacts"].get("cp_features") + if entry is None: + raise StaleCacheError(f"{ref[0]}: GT cache {ctx.paths.root} records no cp_features artifact") + positions = _gt_positions(config) blocks: list[np.ndarray] = [] - n_positions = n_timepoints = n_dropped = 0 - with ( - open_ome_zarr(Path(config.io.gt_path), mode="r") as plate, - open_features_group(ctx.paths, "cp", mode="r") as group, - ): + n_timepoints = n_dropped = 0 + with open_features_group(ctx.paths, "cp", mode="r") as group: if group is None: raise StaleCacheError(f"no CP feature cache at {ctx.paths.cp_features()}") - for pos_name, pos in plate.positions(): - t_count, _, z_depth = pos.data.shape[:3] + for pos_name, (t_count, z_depth) in positions.items(): if z_depth < 2: - raise ValueError(f"{config.io.gt_path}/{pos_name} has Z={z_depth}; the CP reference is 3-D") - n_positions += 1 + raise ValueError(f"{config.io.gt_path}/{pos_name} has Z={z_depth}; CP regionprops are 3-D") for t in range(t_count): feats = read_features_from_group(group, pos_name, t) if feats is None: @@ -161,105 +160,109 @@ def read_dataset_cells(config: DictConfig, n_features: int) -> DatasetCells: blocks.append(np.asarray(finite, dtype=np.float64)) cells = np.concatenate(blocks, axis=0) if blocks else np.empty((0, n_features)) record = { - "dataset": config.benchmark.dataset_ref.dataset, - "target": config.benchmark.dataset_ref.target, + "target": ref[1], "gt_path": str(config.io.gt_path), "gt_cache_dir": str(ctx.paths.root), "cp_cache_path": str(ctx.paths.cp_features()), - "n_positions": n_positions, + "cp_cache_built_at": entry["built_at"], + "positions": sorted(positions), "n_timepoints": n_timepoints, - "n_cells": int(cells.shape[0]), "n_cells_dropped_nonfinite": int(n_dropped), } - return DatasetCells(record=record, cells=cells) + return DatasetFit(dataset=ref[0], cells=cells, record=record, in_mask_fit=in_mask_fit) + + +def lite_entry(target_name: str, ref: Ref, parent: str, parent_positions: list[str]) -> dict[str, Any]: + """Return a lite dataset's reference entry: its parent, and its own GT cache stamp if one exists. + Raises + ------ + ValueError + If the lite GT store holds a position the parent's fit does not. + """ + config = eval_config_for(target_name, ref) + extra = set(_gt_positions(config)) - set(parent_positions) + if extra: + raise ValueError(f"lite {ref[0]} has positions outside its parent {parent}: {sorted(extra)[:5]}") + gt_cache_dir = str(config.io.gt_cache_dir) + entry = load_manifest(cache_paths(gt_cache_dir))["artifacts"].get("cp_features") + return { + "parent": parent, + "target": ref[1], + "gt_cache_dir": gt_cache_dir, + "cp_cache_built_at": entry["built_at"] if entry is not None else None, + } -def build( - target_name: str, datasets: list[tuple[str, str]], dimension: str -) -> tuple[list[DatasetCells], dict[str, Any], tuple[str, ...]]: - """Read every fit dataset and check they share one CP recipe. - Returns - ------- - tuple - ``(per-dataset cells, cp_identity, feature_names)``. +def build(target_name: str, leaves_root: Path) -> dict[str, Any]: + """Read every dataset of the target and return the fitted reference payload. Raises ------ ValueError - If ``dimension`` is not :data:`CP_REFERENCE_DIMENSION`, or the datasets - disagree on the CP recipe. + If the datasets resolve different CP recipes, or a lite parent has no scaler. """ - if dimension != CP_REFERENCE_DIMENSION: - raise ValueError(f"only the {CP_REFERENCE_DIMENSION!r} CP reference exists; got {dimension!r}") - identity: dict[str, Any] | None = None - names: tuple[str, ...] = () - read: list[DatasetCells] = [] - for dataset, manifest_target in datasets: - config = eval_config_for(target_name, dataset, manifest_target) - ds_identity, ds_names = cp_identity_for(config) - if identity is None: - identity, names = ds_identity, ds_names - elif (ds_identity, ds_names) != (identity, names): - raise ValueError(f"{dataset} resolves a different CP recipe: {ds_identity} vs {identity}") - read.append(read_dataset_cells(config, len(names))) - if identity is None: - raise ValueError("no datasets given") - return read, identity, names - - -def _parse_dataset(token: str) -> tuple[str, str]: - """Parse a ``dataset:target`` CLI token.""" - dataset, sep, target = token.partition(":") - if not sep or not dataset or not target: - raise argparse.ArgumentTypeError(f"expected DATASET:TARGET, got {token!r}") - return dataset, target + mask_refs = set(MASK_FIT_DATASETS[target_name]) + leaf_refs = leaf_dataset_refs(target_name, leaves_root) + parents = {ref: lite_parent(ref) for ref in sorted(leaf_refs | mask_refs)} + scaler_refs = sorted(ref for ref, parent in parents.items() if parent is None) + lite_refs = sorted(ref for ref, parent in parents.items() if parent is not None) + if mask_refs - set(scaler_refs): + raise ValueError(f"mask-fit datasets {sorted(mask_refs - set(scaler_refs))} are lite") + + identity, names = cp_space(eval_config_for(target_name, scaler_refs[0])) + for ref in [*scaler_refs[1:], *lite_refs]: + if cp_space(eval_config_for(target_name, ref)) != (identity, names): + raise ValueError(f"{ref[0]} resolves a different CP recipe than {scaler_refs[0][0]}") + + fits = [read_dataset_fit(target_name, ref, len(names), ref in mask_refs) for ref in scaler_refs] + positions = {fit.dataset: fit.record["positions"] for fit in fits} + lite: dict[str, dict[str, Any]] = {} + for ref in lite_refs: + parent = parents[ref] + if parent not in positions: + raise ValueError(f"lite {ref[0]} reuses {parent}, which no grouped leaf or mask fit evaluates") + lite[ref[0]] = lite_entry(target_name, ref, parent, positions[parent]) + return fit_cp_reference(fits, target_name=target_name, feature_names=names, cp_identity=identity, lite=lite) + + +def _summary(payload: dict[str, Any]) -> str: + """Human-readable per-dataset summary of a payload.""" + lines = [ + f"kept {len(payload['kept_feature_names'])}/{len(payload['feature_names'])}: {payload['kept_feature_names']}" + ] + lines.append(f"mask fit: {payload['mask_fit']['n_cells']} cells from {payload['mask_fit']['datasets']}") + for name, s in sorted(payload["scalers"].items()): + lines.append( + f" {name}{' [mask]' if s['in_mask_fit'] else ''}: {len(s['positions'])} positions, " + f"{s['n_timepoints']} timepoints, {s['n_cells']} cells ({s['n_cells_dropped_nonfinite']} non-finite " + f"dropped), floored {s['floored_features'] or 'none'}" + ) + for name, entry in sorted(payload["lite"].items()): + lines.append(f" {name} -> scaler of {entry['parent']} (own cache built_at {entry['cp_cache_built_at']})") + return "\n".join(lines) def main(argv: list[str] | None = None) -> None: """CLI entry point.""" parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--target", required=True, choices=sorted(DEFAULT_DATASETS)) - parser.add_argument("--dimension", default=CP_REFERENCE_DIMENSION, choices=[CP_REFERENCE_DIMENSION]) - parser.add_argument( - "--datasets", - nargs="+", - type=_parse_dataset, - default=None, - metavar="DATASET:TARGET", - help="fit datasets as manifest refs (default: the target's iPSC + A549 mock/denv/zikv)", - ) + parser.add_argument("--target", required=True, choices=sorted(MASK_FIT_DATASETS)) + parser.add_argument("--leaves-root", type=Path, default=_LEAVES_ROOT, help="grouped eval leaves to scan") parser.add_argument("--out", type=Path, default=None, help="output JSON (default: the registry path)") - parser.add_argument("--dry-run", action="store_true", help="read and count cells; write nothing") + parser.add_argument("--dry-run", action="store_true", help="read and fit; write nothing") + parser.add_argument("--force", action="store_true", help="replace an existing reference with a different hash") args = parser.parse_args(argv) - datasets = args.datasets if args.datasets is not None else list(DEFAULT_DATASETS[args.target]) - out = args.out if args.out is not None else cp_reference_path(args.target, args.dimension) + out = args.out if args.out is not None else cp_reference_path(args.target) start = time.perf_counter() - read, identity, names = build(args.target, datasets, args.dimension) - for ds in read: - r = ds.record - print( - f"{r['dataset']}:{r['target']} {r['n_positions']} positions, {r['n_timepoints']} timepoints, " - f"{r['n_cells']} cells ({r['n_cells_dropped_nonfinite']} non-finite dropped)\n {r['cp_cache_path']}" - ) - print(f"total {sum(ds.record['n_cells'] for ds in read)} cells, {len(names)} CP features, identity {identity}") + payload = build(args.target, args.leaves_root) + print(_summary(payload)) + elapsed = time.perf_counter() - start if args.dry_run: - print(f"dry run: would write {out} ({time.perf_counter() - start:.1f} s)") + print(f"dry run: would write {out} sha256={payload['sha256']} ({elapsed:.1f} s)") return - payload = fit_cp_reference( - np.concatenate([ds.cells for ds in read], axis=0), - target_name=args.target, - dimension=args.dimension, - feature_names=names, - cp_identity=identity, - datasets=[ds.record for ds in read], - ) - write_cp_reference(payload, out) - print( - f"kept {len(payload['kept_feature_names'])}/{len(names)}: {payload['kept_feature_names']}\n" - f"wrote {out} sha256={payload['sha256']} ({time.perf_counter() - start:.1f} s)" - ) + written = write_cp_reference(payload, out, force=args.force) + print(f"{'wrote' if written else 'unchanged (identical hash)'} {out} sha256={payload['sha256']} ({elapsed:.1f} s)") if __name__ == "__main__": diff --git a/applications/dynacell/tools/build_cp_reference_test.py b/applications/dynacell/tools/build_cp_reference_test.py index 26b4c0eb9..48517d34e 100644 --- a/applications/dynacell/tools/build_cp_reference_test.py +++ b/applications/dynacell/tools/build_cp_reference_test.py @@ -1,8 +1,11 @@ """Integration tests for ``build_cp_reference``. -Two synthetic datasets are registered as real manifests (``DYNACELL_MANIFEST_ROOTS``), -each with a real HCS GT store and a real GT CP feature cache. The tool resolves them -through ``apply_dataset_ref`` + ``init_cache_context`` exactly as an eval would. +Synthetic datasets are registered as real manifests (``DYNACELL_MANIFEST_ROOTS``), +each with a real HCS GT store, a real GT CP feature cache with its manifest, and +(for the lite one) a split recording ``selection_criteria.lite_of``. A synthetic +grouped-leaf tree names which datasets the target is evaluated on. The tool +resolves everything through ``apply_dataset_ref`` + ``init_cache_context`` exactly +as an eval would. Run:: @@ -17,7 +20,7 @@ import numpy as np import pytest import yaml -from build_cp_reference import main +from build_cp_reference import eval_config_for, main from iohub.ngff import open_ome_zarr from dynacell.evaluation.cache import ( @@ -27,32 +30,33 @@ save_manifest, write_features_to_group, ) -from dynacell.evaluation.cp_reference import CP_REFERENCE_DIMENSION, load_cp_reference +from dynacell.evaluation.cp_reference import MASK_FIT_DATASETS, cp_space, load_cp_reference from dynacell.evaluation.metrics import active_cp_feature_names _N_FEATURES = len(active_cp_feature_names(True)) # eval.yaml ships GLCM on _POSITIONS = ("A/1/0", "A/1/1") _T = 2 - - -def _dataset(root: Path, name: str, seed: int, *, z: int = 3, skip: tuple[str, int] | None = None) -> None: - """Register one dataset: manifest, GT store, and a GT CP cache holding every (pos, t).""" +_BUILT_AT = "2026-09-01T00:00:00+00:00" + + +def _dataset( + root: Path, + name: str, + seed: int, + *, + z: int = 3, + skip: tuple[str, int] | None = None, + lite_of: str | None = None, + positions: tuple[str, ...] = _POSITIONS, + cache: bool = True, +) -> None: + """Register one dataset: manifest, split, GT store, and a GT CP cache holding every (pos, t).""" store = root / "stores" / f"{name}.zarr" cache_dir = root / "caches" / name with open_ome_zarr(store, mode="w", layout="hcs", channel_names=["Structure"], version="0.5") as plate: - for pos_name in _POSITIONS: + for pos_name in positions: row, col, fov = pos_name.split("/") plate.create_position(row, col, fov).create_image("0", np.zeros((_T, 1, z, 4, 4), dtype=np.float32)) - rng = np.random.default_rng(seed) - with open_features_group(cache_paths(cache_dir), "cp", mode="a") as group: - for pos_name in _POSITIONS: - for t in range(_T): - if (pos_name, t) == skip: - continue - feats = rng.standard_normal((5, _N_FEATURES)) + seed - if (pos_name, t) == ("A/1/0", 0): - feats[0, 3] = np.nan # the pipeline drops non-finite cells - write_features_to_group(group, pos_name, t, feats) manifest = { "name": name, "version": "1", @@ -67,76 +71,172 @@ def _dataset(root: Path, name: str, seed: int, *, z: int = 3, skip: tuple[str, i "organelle": "er", "display_name": "ER", "target_channel": "Structure", - "stores": {"train": str(store), "test": str(store), "gt_cache_dir": str(cache_dir)}, - "splits": "splits/none.yaml", + "stores": {"test": str(store), "gt_cache_dir": str(cache_dir)}, + "splits": "splits/sec61b.yaml", } }, } - (root / "manifests" / name).mkdir(parents=True) + split = { + "split_version": "1", + "random_seed": 0, + "selection_criteria": {"lite_of": lite_of} if lite_of is not None else {"positions": "all"}, + "train": {"count": 0, "fovs": []}, + "test": {"count": len(positions), "fovs": list(positions)}, + } + (root / "manifests" / name / "splits").mkdir(parents=True) (root / "manifests" / name / "manifest.yaml").write_text(yaml.dump(manifest)) + (root / "manifests" / name / "splits" / "sec61b.yaml").write_text(yaml.dump(split)) + if not cache: + return + rng = np.random.default_rng(seed) + with open_features_group(cache_paths(cache_dir), "cp", mode="a") as group: + for pos_name in positions: + for t in range(_T): + if (pos_name, t) == skip: + continue + feats = rng.standard_normal((5, _N_FEATURES)) * (1 + seed) + 3 * seed + if (pos_name, t) == ("A/1/0", 0): + feats[0, 3] = np.nan # the pipeline drops non-finite cells + write_features_to_group(group, pos_name, t, feats) + identity, _ = cp_space(eval_config_for("er", (name, "sec61b"))) + save_manifest( + cache_paths(cache_dir), + { + "artifacts": { + "cp_features": { + "path": "features/cp.zarr", + "built_at": _BUILT_AT, + "spacing": [0.3, 0.1, 0.1], + **identity, + } + } + }, + ) + + +def _leaves(root: Path, datasets: list[str]) -> Path: + """Write one grouped leaf for target ``er`` evaluating ``datasets``; return the leaves root.""" + leaves = root / "leaves" + (leaves / "er_bucket").mkdir(parents=True) + doc = { + "target_name": "er", + "conditions": [{"name": d, "benchmark": {"dataset_ref": {"dataset": d, "target": "sec61b"}}} for d in datasets], + } + (leaves / "er_bucket" / "eval_grouped.yaml").write_text(yaml.dump(doc)) + (leaves / "other_bucket").mkdir() + (leaves / "other_bucket" / "eval_grouped.yaml").write_text(yaml.dump({"target_name": "nucleus", "conditions": []})) + return leaves @pytest.fixture def registry(tmp_path: Path, monkeypatch) -> Path: - """Two registered synthetic datasets; returns the tmp root.""" + """Mask-fit sets ds-a + ds-b for ``er``, a HEK-like ds-h, and a lite ds-a-lite; returns the tmp root.""" monkeypatch.setenv("DYNACELL_MANIFEST_ROOTS", str(tmp_path / "manifests")) + monkeypatch.setitem(MASK_FIT_DATASETS, "er", (("ds-a", "sec61b"), ("ds-b", "sec61b"))) return tmp_path -_DATASETS = ["--datasets", "ds-a:sec61b", "ds-b:sec61b"] +def _build_all(root: Path) -> Path: + """Register ds-a, ds-b, ds-h and ds-a-lite and a leaf evaluating all four; return the leaves root.""" + _dataset(root, "ds-a", seed=0) + _dataset(root, "ds-b", seed=1) + _dataset(root, "ds-h", seed=2) + _dataset(root, "ds-a-lite", seed=0, lite_of="ds-a", positions=_POSITIONS[:1], cache=False) + return _leaves(root, ["ds-a", "ds-b", "ds-h", "ds-a-lite"]) def test_build_writes_a_loadable_reference(registry: Path) -> None: - """A real build records per-dataset cell counts and loads back as a verified reference.""" - _dataset(registry, "ds-a", seed=0) - _dataset(registry, "ds-b", seed=1) - out = registry / "er__3d.json" + """A real build: one mask over ds-a + ds-b, a scaler per non-lite set, the lite set mapped to its parent.""" + leaves = _build_all(registry) + out = registry / "er.json" - main(["--target", "er", *_DATASETS, "--out", str(out)]) + main(["--target", "er", "--leaves-root", str(leaves), "--out", str(out)]) payload = json.loads(out.read_text()) - fit = {d["dataset"]: d for d in payload["fit"]["datasets"]} - assert fit["ds-a"]["n_cells"] == 2 * _T * 5 - 1 # one non-finite cell dropped - assert fit["ds-a"]["n_cells_dropped_nonfinite"] == 1 - assert fit["ds-b"]["n_cells"] == 2 * _T * 5 - 1 - assert payload["fit"]["n_cells"] == 38 - assert fit["ds-a"]["cp_cache_path"].endswith("caches/ds-a/features/cp.zarr") - assert payload["cp_identity"]["cp_glcm_enabled"] is True - ref = load_cp_reference(out, target_name="er", dimension=CP_REFERENCE_DIMENSION) - assert ref.keep_mask.shape == (_N_FEATURES,) + assert payload["mask_fit"]["datasets"] == ["ds-a", "ds-b"] + assert sorted(payload["scalers"]) == ["ds-a", "ds-b", "ds-h"] + assert payload["scalers"]["ds-h"]["in_mask_fit"] is False + assert payload["lite"] == { + "ds-a-lite": { + "parent": "ds-a", + "target": "sec61b", + "gt_cache_dir": str(registry / "caches" / "ds-a-lite"), + "cp_cache_built_at": None, + } + } + a = payload["scalers"]["ds-a"] + assert a["n_cells"] == 2 * _T * 5 - 1 # one non-finite cell dropped + assert a["n_cells_dropped_nonfinite"] == 1 + assert a["positions"] == list(_POSITIONS) + assert a["cp_cache_built_at"] == _BUILT_AT + assert payload["mask_fit"]["n_cells"] == 38 + ref = load_cp_reference(out, target_name="er") + assert ref.for_dataset("ds-a-lite").scaler_dataset == "ds-a" + # Scalers differ per set: ds-b was drawn with a different offset and scale. + assert np.abs(np.array(a["mean"]) - np.array(payload["scalers"]["ds-b"]["mean"])).max() > 1 + + +def test_rebuild_is_a_no_op_and_a_changed_reference_needs_force(registry: Path, capsys) -> None: + """An identical rebuild writes nothing; after a GT re-cache the new reference needs ``--force``.""" + leaves = _build_all(registry) + out = registry / "er.json" + args = ["--target", "er", "--leaves-root", str(leaves), "--out", str(out)] + main(args) + main(args) + assert "unchanged (identical hash)" in capsys.readouterr().out + with open_features_group(cache_paths(registry / "caches" / "ds-b"), "cp", mode="a") as group: + write_features_to_group(group, "A/1/0", 0, np.ones((5, _N_FEATURES)) * 9.0) + with pytest.raises(FileExistsError, match="--force"): + main(args) + main([*args, "--force"]) def test_dry_run_writes_nothing(registry: Path, capsys) -> None: - """``--dry-run`` reads and counts every cell but writes no file.""" - _dataset(registry, "ds-a", seed=0) - _dataset(registry, "ds-b", seed=1) - out = registry / "er__3d.json" - main(["--target", "er", *_DATASETS, "--out", str(out), "--dry-run"]) + """``--dry-run`` reads and fits everything but writes no file.""" + leaves = _build_all(registry) + out = registry / "er.json" + main(["--target", "er", "--leaves-root", str(leaves), "--out", str(out), "--dry-run"]) assert not out.exists() - assert "total 38 cells" in capsys.readouterr().out + printed = capsys.readouterr().out + assert "mask fit: 38 cells" in printed and "ds-a-lite -> scaler of ds-a" in printed def test_missing_timepoint_raises(registry: Path) -> None: """A GT (position, timepoint) absent from the CP cache fails instead of being skipped.""" _dataset(registry, "ds-a", seed=0) _dataset(registry, "ds-b", seed=1, skip=("A/1/1", 1)) + leaves = _leaves(registry, ["ds-a", "ds-b"]) with pytest.raises(StaleCacheError, match="CP cache miss at A/1/1/t1"): - main(["--target", "er", *_DATASETS, "--out", str(registry / "x.json")]) + main(["--target", "er", "--leaves-root", str(leaves), "--out", str(registry / "x.json")]) def test_cache_of_another_recipe_is_refused(registry: Path) -> None: """A GT cache stamped with another CP recipe is refused by the pipeline's own identity check.""" _dataset(registry, "ds-a", seed=0) + _dataset(registry, "ds-b", seed=1) save_manifest( cache_paths(registry / "caches" / "ds-a"), - {"artifacts": {"cp_features": {"path": "features/cp.zarr", "cp_feature_version": "v1_old"}}}, + {"artifacts": {"cp_features": {"path": "features/cp.zarr", "built_at": _BUILT_AT, "cp_feature_version": "v1"}}}, ) + leaves = _leaves(registry, ["ds-a", "ds-b"]) with pytest.raises(StaleCacheError, match="cp_feature_version"): - main(["--target", "er", "--datasets", "ds-a:sec61b", "--out", str(registry / "x.json")]) + main(["--target", "er", "--leaves-root", str(leaves), "--out", str(registry / "x.json")]) + + +def test_lite_outside_its_parent_is_refused(registry: Path) -> None: + """A lite store holding a position its parent's scaler was not fit on is refused at build time.""" + _dataset(registry, "ds-a", seed=0) + _dataset(registry, "ds-b", seed=1) + _dataset(registry, "ds-a-lite", seed=0, lite_of="ds-a", positions=("B/1/0",), cache=False) + leaves = _leaves(registry, ["ds-a", "ds-b", "ds-a-lite"]) + with pytest.raises(ValueError, match="outside its parent"): + main(["--target", "er", "--leaves-root", str(leaves), "--out", str(registry / "x.json")]) def test_2d_store_is_refused(registry: Path) -> None: """A single-plane GT store cannot feed the 3-D reference.""" _dataset(registry, "ds-a", seed=0, z=1) - with pytest.raises(ValueError, match="the CP reference is 3-D"): - main(["--target", "er", "--datasets", "ds-a:sec61b", "--out", str(registry / "x.json")]) + _dataset(registry, "ds-b", seed=1) + leaves = _leaves(registry, ["ds-a", "ds-b"]) + with pytest.raises(ValueError, match="CP regionprops are 3-D"): + main(["--target", "er", "--leaves-root", str(leaves), "--out", str(registry / "x.json")]) From 6d7f70d8a976ad2f39dbc97b650fa354c777701e Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 18:30:19 -0700 Subject: [PATCH 06/41] fix(dynacell/eval): mask the cross-condition CP probe with the target's reference The infection probe pruned CP columns with select_features on each (condition pair, source) cohort, so its feature subset moved with the model and the pair. It now applies the keep-mask of the CP reference named in the eval dirs' cp_selected_feature_mask.json, hash-checked. It raises when the two dirs were scored in different references. The per-plate MADScaler is unchanged; a comment now says it removes per- condition offsets by design. select_features has no caller left, so it and its tests are removed; the GT-only selector inlines the pruning helper. The determinism test now pins the exact expected mask. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../evaluation/cross_condition_probe.py | 52 ++++++++++-- .../src/dynacell/evaluation/feature_select.py | 82 +++---------------- .../evaluation/feature_select_test.py | 77 ++--------------- .../tests/test_cross_condition_probe.py | 71 +++++++++++++++- .../tests/test_evaluation_pipeline.py | 4 +- 5 files changed, 132 insertions(+), 154 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cross_condition_probe.py b/applications/dynacell/src/dynacell/evaluation/cross_condition_probe.py index 96c19ddfe..42c081881 100644 --- a/applications/dynacell/src/dynacell/evaluation/cross_condition_probe.py +++ b/applications/dynacell/src/dynacell/evaluation/cross_condition_probe.py @@ -26,6 +26,7 @@ import argparse import csv +import json import sys from pathlib import Path from typing import get_args @@ -33,7 +34,7 @@ import numpy as np from dynacell.evaluation.cache import FeatureKind -from dynacell.evaluation.feature_select import select_features +from dynacell.evaluation.cp_reference import load_cp_reference from dynacell.evaluation.linear_probe import MADScaler, paired_auroc _FEATURE_TYPES: tuple[str, ...] = get_args(FeatureKind) @@ -117,6 +118,38 @@ def _load_embeddings( return result +#: Sidecar the eval writes beside its CP embeddings, naming the CP reference it scored in. +_CP_SIDECAR = "cp_selected_feature_mask.json" + + +def _cp_reference_mask(eval_dirs: list[Path]) -> np.ndarray: + """Return the CP reference keep-mask shared by ``eval_dirs``. + + Read from each eval dir's ``cp_selected_feature_mask.json``, which names the + target's CP reference (path + hash). The mask is a property of the target, not + of the model or condition, so the probe compares conditions on the same feature + subset every eval scored. + + Raises + ------ + FileNotFoundError + If an eval dir has no sidecar. + ValueError + If the dirs name different references (ambiguous), or the reference on + disk no longer has the recorded hash. + """ + sidecars = [json.loads((d / _CP_SIDECAR).read_text()) for d in eval_dirs] + refs = {(s["reference_path"], s["reference_sha256"]) for s in sidecars} + if len(refs) != 1: + raise ValueError(f"eval dirs {[str(d) for d in eval_dirs]} were scored in different CP references: {refs}") + ((path, sha256),) = refs + ref_payload = json.loads(Path(path).read_text()) + ref = load_cp_reference(Path(path), target_name=ref_payload["target_name"]) + if ref.sha256 != sha256: + raise ValueError(f"CP reference {path} changed since these evals (sha256 {ref.sha256[:12]} vs {sha256[:12]})") + return ref.keep_mask + + def _probe_pair( eval_dirs_by_condition: dict[str, Path], pair: tuple[str, str], @@ -160,19 +193,22 @@ def _probe_pair( if x0.shape[1] != x1.shape[1]: raise ValueError(f"feature dim mismatch for {feature} {source}: {c0}={x0.shape[1]} vs {c1}={x1.shape[1]}") - # CP regionprops: variance + correlation prune on the pooled cohort - # to drop near-constant or redundant columns. Skipped for the deep - # embeddings, which are dense learned features. + # CP regionprops: the target's CP reference mask (GT-only, pooled over the + # benchmark test sets), so the columns are the ones every eval scored and do not + # move with the model or the condition pair. Skipped for the deep embeddings, + # which are dense learned features. if feature == "cp": - x0, x1, _ = select_features(x0, x1) - if x0.size == 0 or x1.size == 0: - row["skipped_reason"] = "all CP columns dropped by select_features" - return row + mask = _cp_reference_mask([eval_dirs_by_condition[c0], eval_dirs_by_condition[c1]]) + x0, x1 = x0[:, mask], x1[:, mask] # Per-plate MAD normalization: cancels per-plate intensity offsets # (illumination, exposure, gain) that would otherwise dominate the # classifier — especially on CP regionprops, where raw intensity # columns make different plates trivially separable (AUROC = 1.0). + # By design this also removes any per-condition feature OFFSET: the probe + # measures whether infection changes the within-plate cell distribution, not + # its location. This is deliberately NOT the CP reference's per-test-set + # scaler, which keeps pred-vs-GT offsets visible for the KID/FID metrics. x0_scaled = MADScaler().fit_transform(x0.astype(np.float64)) x1_scaled = MADScaler().fit_transform(x1.astype(np.float64)) # Tag FOV ids by condition so the two sides cannot collide. diff --git a/applications/dynacell/src/dynacell/evaluation/feature_select.py b/applications/dynacell/src/dynacell/evaluation/feature_select.py index a2b339a54..1004b508e 100644 --- a/applications/dynacell/src/dynacell/evaluation/feature_select.py +++ b/applications/dynacell/src/dynacell/evaluation/feature_select.py @@ -17,8 +17,8 @@ # Defaults shared with downstream consumers (e.g. the cp_selected_feature_mask # JSON sidecar emitted by the evaluation pipeline, and the CP reference). Keep -# `select_features`, `select_gt_features`, `variance_threshold`, and -# `correlation_threshold` keyword defaults aligned +# `select_gt_features`, `variance_threshold`, and `correlation_threshold` +# keyword defaults aligned # with these constants so the sidecar cannot drift from the actual call. DEFAULT_FREQ_CUT = 0.05 DEFAULT_UNIQUE_CUT = 0.01 @@ -161,20 +161,6 @@ def correlation_threshold( return keep -def _prune_mask( - X: np.ndarray, - freq_cut: float, - unique_cut: float, - corr_threshold: float, -) -> np.ndarray: - """Variance pruning then correlation pruning on the survivors; return the keep-mask.""" - mask_var = variance_threshold(X, freq_cut=freq_cut, unique_cut=unique_cut) - mask_corr_local = correlation_threshold(X[:, mask_var], threshold=corr_threshold, method="pearson") - keep_mask = np.zeros(X.shape[1], dtype=bool) - keep_mask[np.flatnonzero(mask_var)[mask_corr_local]] = True - return keep_mask - - def select_gt_features( gt: np.ndarray, freq_cut: float = DEFAULT_FREQ_CUT, @@ -183,10 +169,12 @@ def select_gt_features( ) -> np.ndarray: """Select features on ground-truth cells alone, never on predictions. - Same criteria as :func:`select_features`, but the fit set holds only GT - rows, so the kept feature subset is a property of the target and cannot - move with the model being evaluated. Used to build the per-target CP - reference (:mod:`dynacell.evaluation.cp_reference`). + Variance pruning first (cheap), then correlation pruning on the survivors. + The fit set holds only GT rows, so the kept feature subset is a property of + the target and cannot move with the model being evaluated (pooling a + prediction into the fit is what used to give every model its own mask). + Used to build the per-target CP reference + (:mod:`dynacell.evaluation.cp_reference`). Parameters ---------- @@ -205,52 +193,8 @@ def select_gt_features( np.ndarray Boolean keep-mask of shape ``(n_features,)``. """ - return _prune_mask(gt, freq_cut=freq_cut, unique_cut=unique_cut, corr_threshold=corr_threshold) - - -def select_features( - gt: np.ndarray, - pred: np.ndarray, - freq_cut: float = DEFAULT_FREQ_CUT, - unique_cut: float = DEFAULT_UNIQUE_CUT, - corr_threshold: float = DEFAULT_CORR_THRESHOLD, -) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - """Pool ``(gt, pred)``, apply both filters in sequence, return filtered pair + mask. - - Variance pruning happens first (cheap), correlation pruning second on the - survivors. - - Parameters - ---------- - gt : np.ndarray - Shape ``(n_gt, n_features)``, float64. - pred : np.ndarray - Shape ``(n_pred, n_features)``, float64. - freq_cut : float, optional - Forwarded to :func:`variance_threshold`. Defaults to ``0.05``. - unique_cut : float, optional - Forwarded to :func:`variance_threshold`. Defaults to ``0.01``. - corr_threshold : float, optional - Forwarded to :func:`correlation_threshold`. Defaults to ``0.9``. - - Returns - ------- - gt_filtered : np.ndarray - Shape ``(n_gt, n_kept)``. - pred_filtered : np.ndarray - Shape ``(n_pred, n_kept)``. - keep_mask : np.ndarray - Boolean mask of shape ``(n_features,)``. Composition of variance + - correlation pruning. - - Raises - ------ - ValueError - If ``gt`` and ``pred`` disagree on the feature dimension. - """ - if gt.shape[1] != pred.shape[1]: - raise ValueError(f"feature dim mismatch: gt.shape[1]={gt.shape[1]} vs pred.shape[1]={pred.shape[1]}.") - - pooled = np.vstack([gt, pred]) - keep_mask = _prune_mask(pooled, freq_cut=freq_cut, unique_cut=unique_cut, corr_threshold=corr_threshold) - return gt[:, keep_mask], pred[:, keep_mask], keep_mask + mask_var = variance_threshold(gt, freq_cut=freq_cut, unique_cut=unique_cut) + mask_corr_local = correlation_threshold(gt[:, mask_var], threshold=corr_threshold, method="pearson") + keep_mask = np.zeros(gt.shape[1], dtype=bool) + keep_mask[np.flatnonzero(mask_var)[mask_corr_local]] = True + return keep_mask diff --git a/applications/dynacell/src/dynacell/evaluation/feature_select_test.py b/applications/dynacell/src/dynacell/evaluation/feature_select_test.py index 873e9138b..5d10421b2 100644 --- a/applications/dynacell/src/dynacell/evaluation/feature_select_test.py +++ b/applications/dynacell/src/dynacell/evaluation/feature_select_test.py @@ -7,7 +7,6 @@ from dynacell.evaluation.feature_select import ( correlation_threshold, - select_features, select_gt_features, variance_threshold, ) @@ -101,66 +100,6 @@ def test_correlation_threshold_rejects_unsupported_method( correlation_threshold(X, threshold=0.9, method="spearman") -def test_select_features_applies_both_filters( - rng: np.random.Generator, -) -> None: - # features: [constant, A, A+ε, B] - n_gt = 60 - n_pred = 60 - n_total = n_gt + n_pred - - a = rng.standard_normal(n_total).astype(np.float64) - b = rng.standard_normal(n_total).astype(np.float64) - eps = 1e-4 * rng.standard_normal(n_total).astype(np.float64) - - pooled = np.column_stack( - [ - np.zeros(n_total), # feature 0: constant - a, # feature 1: A - a + eps, # feature 2: A + ε - b, # feature 3: independent - ] - ) - - gt = pooled[:n_gt] - pred = pooled[n_gt:] - - gt_f, pred_f, keep_mask = select_features(gt, pred, freq_cut=0.05, unique_cut=0.01, corr_threshold=0.9) - - assert keep_mask.dtype == bool - assert keep_mask.shape == (4,) - # Feature 0 dropped by variance pruning. - assert not keep_mask[0] - # Feature 3 always kept (independent). - assert keep_mask[3] - # Exactly one of {1, 2} survives the correlation pruning. - assert keep_mask[1] ^ keep_mask[2] - assert keep_mask.sum() == 2 - assert gt_f.shape == (n_gt, 2) - assert pred_f.shape == (n_pred, 2) - - -def test_select_features_shape_mismatch_raises( - rng: np.random.Generator, -) -> None: - gt = rng.standard_normal((10, 5)).astype(np.float64) - pred = rng.standard_normal((10, 6)).astype(np.float64) - with pytest.raises(ValueError, match="dim mismatch|shape"): - select_features(gt, pred) - - -def test_select_features_returns_aligned_filtered_arrays( - rng: np.random.Generator, -) -> None: - gt = rng.standard_normal((30, 8)).astype(np.float64) - pred = rng.standard_normal((25, 8)).astype(np.float64) - gt_f, pred_f, keep_mask = select_features(gt, pred) - - assert gt_f.shape[0] == 30 - assert pred_f.shape[0] == 25 - assert gt_f.shape[1] == pred_f.shape[1] == int(keep_mask.sum()) - - def test_select_gt_features_drops_low_variance_and_correlated( rng: np.random.Generator, ) -> None: @@ -188,14 +127,10 @@ def test_select_gt_features_drops_low_variance_and_correlated( assert keep[4] and keep[5] -def test_select_gt_features_is_deterministic_and_ignores_predictions( +def test_select_gt_features_is_deterministic( rng: np.random.Generator, ) -> None: - """The mask is a function of GT alone: repeated calls agree, and no pred enters. - - ``select_features`` pools GT with a prediction, so two models get two masks; the - GT-only selector is what makes the CP feature space model-independent. - """ + """Repeated calls on the same GT agree exactly, and the redundant column is the one dropped.""" gt = rng.standard_normal((150, 10)) gt[:, 7] = gt[:, 2] * 3.0 + 1e-6 * rng.standard_normal(150) @@ -203,8 +138,6 @@ def test_select_gt_features_is_deterministic_and_ignores_predictions( second = select_gt_features(gt.copy()) np.testing.assert_array_equal(first, second) - assert not (first[2] and first[7]) - # A prediction that decorrelates column 7 would flip the pooled mask; GT-only cannot see it. - pred = rng.standard_normal((150, 10)) * 50.0 - _, _, pooled_mask = select_features(gt, pred) - assert not np.array_equal(pooled_mask, first) + expected = np.ones(10, dtype=bool) + expected[7] = False # tie on connectivity -> the higher index of the (2, 7) pair goes + np.testing.assert_array_equal(first, expected) diff --git a/applications/dynacell/tests/test_cross_condition_probe.py b/applications/dynacell/tests/test_cross_condition_probe.py index e97869b65..4ba7f3a8f 100644 --- a/applications/dynacell/tests/test_cross_condition_probe.py +++ b/applications/dynacell/tests/test_cross_condition_probe.py @@ -5,10 +5,14 @@ """ import csv +import json from pathlib import Path import numpy as np +import pytest +import dynacell.evaluation.cross_condition_probe as probe +from dynacell.evaluation.cp_reference import DatasetFit, fit_cp_reference, write_cp_reference from dynacell.evaluation.cross_condition_probe import ( _FEATURE_TYPES, GROUP_PROBE_FILENAME, @@ -16,8 +20,30 @@ run_for_group, ) - -def _write_group_embeddings(eval_dir: Path, *, seed: int, n_fovs: int = 4, n_per_fov: int = 25, dim: int = 16) -> None: +_DIM = 16 +_DROPPED = 3 # constant in the reference GT, so the reference mask drops it + + +def _write_reference(path: Path, seed: int = 0) -> str: + """Fit a real CP reference over ``_DIM`` features whose mask drops column ``_DROPPED``; return its sha256.""" + gt = np.random.default_rng(seed).normal(size=(200, _DIM)) + gt[:, _DROPPED] = 1.0 + fit = DatasetFit( + dataset="ds", + cells=gt, + record={"positions": [], "gt_cache_dir": None, "cp_cache_built_at": None}, + in_mask_fit=True, + ) + payload = fit_cp_reference( + [fit], target_name="membrane", feature_names=tuple(f"f{i}" for i in range(_DIM)), cp_identity={}, lite={} + ) + write_cp_reference(payload, path, force=True) + return payload["sha256"] + + +def _write_group_embeddings( + eval_dir: Path, *, seed: int, n_fovs: int = 4, n_per_fov: int = 25, dim: int = _DIM +) -> None: """Write gt+pred NPZ for every feature space into ``eval_dir/embeddings``. Cells are spread over ``n_fovs`` FOVs so the FOV-stratified GroupKFold has @@ -36,6 +62,12 @@ def _write_group_embeddings(eval_dir: Path, *, seed: int, n_fovs: int = 4, n_per for feat in _FEATURE_TYPES: x = rng.normal(size=(n, dim)).astype(np.float32) np.savez(emb / f"{source}_{feat}_single_cell_embeddings.npz", embeddings=x, fov=fov, timepoint=tp) + # The CP sidecar every eval writes, naming the reference it scored in. + reference = eval_dir.parent / "cp_reference.json" + sha256 = _write_reference(reference) if not reference.exists() else json.loads(reference.read_text())["sha256"] + (eval_dir / "cp_selected_feature_mask.json").write_text( + json.dumps({"reference_path": str(reference), "reference_sha256": sha256, "dataset": "ds"}) + ) def _read_rows(csv_path: Path) -> list[dict]: @@ -129,3 +161,38 @@ def test_run_longform_covers_all_pairs_and_sources(tmp_path): # 4 features x 2 pairs x 2 sources. assert len(rows) == len(_FEATURE_TYPES) * 2 * 2 assert {r["pair"] for r in rows} == {"mock_vs_denv", "mock_vs_zikv"} + + +def test_cp_probe_uses_the_reference_mask(tmp_path, monkeypatch): + """CP columns reaching the classifier are the reference's kept set, for GT and pred alike.""" + mock = tmp_path / "eval_demo_membrane_mock" + denv = tmp_path / "eval_demo_membrane_denv" + _write_group_embeddings(mock, seed=1) + _write_group_embeddings(denv, seed=2) + widths = [] + real = probe.paired_auroc + + def _spy(x0, x1, *args, **kwargs): + widths.append((x0.shape[1], x1.shape[1])) + return real(x0, x1, *args, **kwargs) + + monkeypatch.setattr(probe, "paired_auroc", _spy) + rows = probe._probe_one_group({"mock": mock, "denv": denv}, n_splits=2, rng_seed=0) + assert rows + cp_calls = widths[_FEATURE_TYPES.index("cp") * 2 : _FEATURE_TYPES.index("cp") * 2 + 2] + assert cp_calls == [(_DIM - 1, _DIM - 1)] * 2 + assert all(w == (_DIM, _DIM) for i, w in enumerate(widths) if i // 2 != _FEATURE_TYPES.index("cp")) + + +def test_cp_probe_refuses_evals_scored_in_different_references(tmp_path): + """Two conditions scored in different CP references are ambiguous and raise.""" + mock = tmp_path / "eval_demo_membrane_mock" + denv = tmp_path / "eval_demo_membrane_denv" + _write_group_embeddings(mock, seed=1) + _write_group_embeddings(denv, seed=2) + other = tmp_path / "other.json" + sidecar = json.loads((denv / "cp_selected_feature_mask.json").read_text()) + sidecar.update(reference_path=str(other), reference_sha256=_write_reference(other, seed=9)) + (denv / "cp_selected_feature_mask.json").write_text(json.dumps(sidecar)) + with pytest.raises(ValueError, match="different CP references"): + run([mock, denv], tmp_path / "probe.csv") diff --git a/applications/dynacell/tests/test_evaluation_pipeline.py b/applications/dynacell/tests/test_evaluation_pipeline.py index a700681b0..9a604f6d9 100644 --- a/applications/dynacell/tests/test_evaluation_pipeline.py +++ b/applications/dynacell/tests/test_evaluation_pipeline.py @@ -44,9 +44,7 @@ def _import_pipeline_with_stubs(monkeypatch): feature_metrics_module.compute_feature_similarity_pairwise = lambda *args, **kwargs: {} feature_select_module = types.ModuleType("dynacell.evaluation.feature_select") - # ``cross_condition_probe`` imports ``select_features``; ``cp_reference`` imports - # ``select_gt_features``. Neither is exercised by these cache-reuse tests. - feature_select_module.select_features = lambda gt, pred, **kw: (gt, pred, np.ones(gt.shape[1], dtype=bool)) + # ``cp_reference`` imports ``select_gt_features``; not exercised by these cache-reuse tests. feature_select_module.select_gt_features = lambda gt, **kw: np.ones(gt.shape[1], dtype=bool) feature_select_module.DEFAULT_FREQ_CUT = 0.05 feature_select_module.DEFAULT_UNIQUE_CUT = 0.01 From 307f9deddde3ec94bec9021750b36fdd9074bcee Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 18:30:20 -0700 Subject: [PATCH 07/41] fix(dynacell/tools): score lite-sim CP in the pipeline's reference space lite_checks (check E) and lite_subset_sim re-implemented the retired CP transform: the sidecar keep_mask, then a per-side z-score over each cell subset. Both now read the reference path, hash and dataset from cp_selected_feature_mask.json and apply the pipeline's transform: the reference mask plus that dataset's GT scaler (the parent's for lite). The transform no longer depends on the cell subset, so the --cp-frozen-norm variant is gone. The tests fit a real reference whose scaler differs from the eval cells' own statistics, and assert the result differs from the per-side z-score. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- applications/dynacell/tools/lite_checks.py | 13 ++- .../dynacell/tools/lite_checks_test.py | 54 +++++++++++- .../dynacell/tools/lite_subset_sim.py | 75 ++++++++--------- .../dynacell/tools/lite_subset_sim_test.py | 82 ++++++++++++++----- 4 files changed, 155 insertions(+), 69 deletions(-) diff --git a/applications/dynacell/tools/lite_checks.py b/applications/dynacell/tools/lite_checks.py index ef4dbff94..57764de16 100644 --- a/applications/dynacell/tools/lite_checks.py +++ b/applications/dynacell/tools/lite_checks.py @@ -25,7 +25,6 @@ import argparse import itertools -import json from pathlib import Path import numpy as np @@ -33,6 +32,7 @@ from lite_subset_sim import ( EXTRACTORS, MODELS_3D, + cp_space_of, designs_for, load_systems, mmd2_from_features, @@ -205,9 +205,9 @@ def check_e( Y = np.asarray(np.load(g, allow_pickle=True)["embeddings"], dtype=np.float64) keep = np.isfinite(X).all(1) & np.isfinite(Y).all(1) X, Y = X[keep], Y[keep] - if tok == "cp": - mask = np.array(json.loads((path / "cp_selected_feature_mask.json").read_text())["keep_mask"], bool) - X, Y = X[:, mask], Y[:, mask] + if tok == "cp": # the pipeline's CP space: reference mask + this dataset's GT scaler + space = cp_space_of(path) + X, Y = space.transform(X), space.transform(Y) n_all = X.shape[0] if tok != "cp": # deep extractors: precompute cell-level kernels once, slice per draw K_XX, K_YY, K_XY = poly_kernel(X, X), poly_kernel(Y, Y), poly_kernel(X, Y) @@ -215,10 +215,7 @@ def check_e( def kid(idx: np.ndarray) -> float: m = len(idx) if tok == "cp": - Xs, Ys = X[idx], Y[idx] - Xs = (Xs - Xs.mean(0)) / (Xs.std(0) + 1e-8) - Ys = (Ys - Ys.mean(0)) / (Ys.std(0) + 1e-8) - return mmd2_from_features(poly3_features(Xs), poly3_features(Ys)) + return mmd2_from_features(poly3_features(X[idx]), poly3_features(Y[idx])) kxx, kyy, kxy = K_XX[np.ix_(idx, idx)], K_YY[np.ix_(idx, idx)], K_XY[np.ix_(idx, idx)] return (kxx.sum() - np.trace(kxx) + kyy.sum() - np.trace(kyy)) / (m * (m - 1)) - 2 * kxy.sum() / (m * m) diff --git a/applications/dynacell/tools/lite_checks_test.py b/applications/dynacell/tools/lite_checks_test.py index 311e63dea..445696644 100644 --- a/applications/dynacell/tools/lite_checks_test.py +++ b/applications/dynacell/tools/lite_checks_test.py @@ -16,11 +16,16 @@ from __future__ import annotations +import json from pathlib import Path import numpy as np import pandas as pd -from lite_checks import check_b2 +import pytest +from lite_checks import check_b2, check_e +from lite_subset_sim import mmd2_from_features, poly3_features + +from dynacell.evaluation.cp_reference import DatasetFit, fit_cp_reference, write_cp_reference _BUCKET = "a549__mock" _FOVS = [f"0/0/fov{i:04d}" for i in range(12)] @@ -85,3 +90,50 @@ def test_lite_subset_that_reverses_the_gap_loses_the_pair(tmp_path: Path) -> Non assert row.retention == 0.0 assert row.confidently_wrong == 1 assert row.n_rows == 3 + + +def test_check_e_scores_cp_in_the_reference_space(tmp_path: Path) -> None: + """Check E's full-pool GLCM+ KID is the pipeline's: reference mask + the eval dataset's GT scaler. + + The reference scaler is far from the eval cells' own statistics, so the retired + per-side z-score gives a clearly different KID on the same cells. + """ + rng = np.random.default_rng(5) + d, n = 4, 120 + fit = DatasetFit( + dataset="ds", + cells=rng.normal(size=(300, d)) * 3.0 + 5.0, + record={"positions": ["0/0/fov0000"], "gt_cache_dir": None, "cp_cache_built_at": None}, + in_mask_fit=True, + ) + payload = fit_cp_reference( + [fit], target_name="nucleus", feature_names=tuple(f"f{i}" for i in range(d)), cp_identity={}, lite={} + ) + reference = tmp_path / "nucleus.json" + write_cp_reference(payload, reference) + path = tmp_path / "nucleus" / "fnet3d_paper" / "ipsc" / "ipsc" + (path / "embeddings").mkdir(parents=True) + gt = rng.normal(size=(n, d)) + pred = 0.5 * gt + 0.2 * rng.normal(size=gt.shape) + fov = np.array([f"0/0/fov{i % 6:04d}" for i in range(n)]) + for side, arr in (("gt", gt), ("pred", pred)): + np.savez(path / "embeddings" / f"{side}_cp_single_cell_embeddings.npz", embeddings=arr, fov=fov) + (path / "cp_selected_feature_mask.json").write_text( + json.dumps({"reference_path": str(reference), "reference_sha256": payload["sha256"], "dataset": "ds"}) + ) + + result = check_e("nucleus", "ipsc", ["fnet3d_paper/ipsc"], tmp_path, sizes=(50,), draws=2, data_root=tmp_path) + + mask = np.array(payload["keep_mask"]) + mean, std = np.array(payload["scalers"]["ds"]["mean"]), np.array(payload["scalers"]["ds"]["std"]) + expected = mmd2_from_features( + poly3_features((pred[:, mask] - mean) / std), poly3_features((gt[:, mask] - mean) / std) + ) + full = result.loc[result.extractor == "CP", "full"].iloc[0] + assert full == pytest.approx(expected, rel=1e-10) + + def zscore(x: np.ndarray) -> np.ndarray: + return (x - x.mean(0)) / (x.std(0) + 1e-8) + + old = mmd2_from_features(poly3_features(zscore(pred[:, mask])), poly3_features(zscore(gt[:, mask]))) + assert full != pytest.approx(old, rel=0.1) diff --git a/applications/dynacell/tools/lite_subset_sim.py b/applications/dynacell/tools/lite_subset_sim.py index 1dba70d76..da97436a8 100644 --- a/applications/dynacell/tools/lite_subset_sim.py +++ b/applications/dynacell/tools/lite_subset_sim.py @@ -14,9 +14,11 @@ KID estimator: a single unbiased MMD^2 over all selected cells (poly kernel degree 3, gamma 1/d, coef0 1) -- identical to torch-fidelity's per-subset estimator, so it equals the pipeline's ``100 x min(1000, n)`` subset mean exactly when n <= 1000 and -in expectation above that. GLCM+ (``cp``) is z-scored over the scored cells after -the run's ``select_features`` mask, so it depends on the cell subset and is scored -directly through an exact degree-3 feature map instead of block sums. +in expectation above that. GLCM+ (``cp``) is scored in the pipeline's CP space: +the target's CP reference mask plus the eval dataset's GT scaler (the parent's for a +lite dataset), both read via the run's ``cp_selected_feature_mask.json`` sidecar. +That transform is fixed, so it does not depend on the cell subset; CP is still +scored directly through an exact degree-3 feature map (22-dim: cheap). Run (on a compute node; the login node has one CPU):: @@ -39,6 +41,7 @@ from build_temporal_subset_zarr import spread_timepoints from scipy.stats import kendalltau, spearmanr +from dynacell.evaluation.cp_reference import DatasetCPSpace, load_cp_reference from dynacell.evaluation.paths import DATA_ROOT EXTRACTORS = {"cp": "CP", "dinov3": "DINOv3", "dynaclr": "DynaCLR", "celldino": "CellDINO", "morphem": "MorphEm"} @@ -63,6 +66,26 @@ _EVAL_CSVS = ("pixel_metrics.csv", "mask_metrics.csv", "feature_metrics.csv") +def cp_space_of(eval_dir: Path) -> DatasetCPSpace: + """Return the CP space an eval dir was scored in, from its ``cp_selected_feature_mask.json``. + + The sidecar names the reference (path + hash) and the eval's dataset; the + reference is loaded, hash-checked and bound to that dataset, so a lite eval + gets its parent's scaler exactly as the pipeline did. + + Raises + ------ + ValueError + If the reference on disk no longer has the hash the eval recorded. + """ + sidecar = json.loads((eval_dir / "cp_selected_feature_mask.json").read_text()) + path = Path(sidecar["reference_path"]) + ref = load_cp_reference(path, target_name=json.loads(path.read_text())["target_name"]) + if ref.sha256 != sidecar["reference_sha256"]: + raise ValueError(f"{eval_dir}: CP reference {path} changed since the eval ({ref.sha256[:12]})") + return ref.for_dataset(sidecar["dataset"]) + + def poly_kernel(X: np.ndarray, Y: np.ndarray) -> np.ndarray: """Return torch-fidelity's KID kernel ``(x.y / d + 1)^3`` between the rows of X and Y. @@ -141,9 +164,6 @@ class System: Eval dir ``////``. blocks : list of tuple of (str, int) The shared ``(FOV, Timepoint)`` universe, in block-index order. - cp_frozen_norm : bool - Z-score GLCM+ subsets with the full pool's statistics instead of the subset's - own (a candidate lite estimator; the pipeline uses the subset's own). Raises ------ @@ -151,10 +171,9 @@ class System: If an extractor's pred and GT embeddings are not cell-aligned. """ - def __init__(self, path: Path, blocks: list[tuple[str, int]], cp_frozen_norm: bool = False): + def __init__(self, path: Path, blocks: list[tuple[str, int]]): self.path = path self.name = f"{path.parts[-3]}/{path.parts[-2]}" - self.cp_frozen_norm = cp_frozen_norm self.block_index = {b: i for i, b in enumerate(blocks)} nb = len(blocks) px = pd.read_csv(path / "pixel_metrics.csv") @@ -195,20 +214,9 @@ def __init__(self, path: Path, blocks: list[tuple[str, int]], cp_frozen_norm: bo ok = cell_block >= 0 X, Y, cell_block = X[ok], Y[ok], cell_block[ok] if tok == "cp": - # pipeline: select_features() mask (read from the run's sidecar), then per-side - # z-score over the scored cells, then KID / median cosine. Both depend on the - # cell subset, so CP is scored directly (22-dim: cheap) instead of via block sums. - mask = np.array(json.loads((path / "cp_selected_feature_mask.json").read_text())["keep_mask"], bool) - Xm, Ym = X[:, mask], Y[:, mask] - self.cp = dict( - X=Xm, - Y=Ym, - cell_block=cell_block, - mu_x=Xm.mean(0), - sd_x=Xm.std(0) + 1e-8, - mu_y=Ym.mean(0), - sd_y=Ym.std(0) + 1e-8, - ) + # pipeline: the CP reference mask + this dataset's GT scaler, on both sides. + space = cp_space_of(path) + self.cp = dict(X=space.transform(X), Y=space.transform(Y), cell_block=cell_block) continue # one-hot block membership (n x nb) -> block sums via M^T K M M = np.zeros((len(cell_block), nb)) @@ -281,14 +289,8 @@ def metrics(self, sel: np.ndarray) -> dict[str, float]: X, Y = np.repeat(self.cp["X"], reps, axis=0), np.repeat(self.cp["Y"], reps, axis=0) m = X.shape[0] if m >= KID_MIN: - if self.cp_frozen_norm: - Xz = (X - self.cp["mu_x"]) / self.cp["sd_x"] - Yz = (Y - self.cp["mu_y"]) / self.cp["sd_y"] - else: - Xz = (X - X.mean(0)) / (X.std(0) + 1e-8) - Yz = (Y - Y.mean(0)) / (Y.std(0) + 1e-8) - out["CP_KID"] = mmd2_from_features(poly3_features(Xz), poly3_features(Yz)) - cos = np.einsum("ij,ij->i", Xz, Yz) / (np.linalg.norm(Xz, axis=1) * np.linalg.norm(Yz, axis=1)) + out["CP_KID"] = mmd2_from_features(poly3_features(X), poly3_features(Y)) + cos = np.einsum("ij,ij->i", X, Y) / (np.linalg.norm(X, axis=1) * np.linalg.norm(Y, axis=1)) out["CP_MedCos"] = float(np.nanmedian(cos)) else: out["CP_KID"] = out["CP_MedCos"] = np.nan @@ -300,7 +302,6 @@ def load_systems( bucket: str, models: list[str], data_root: Path = DATA_ROOT, - cp_frozen_norm: bool = False, ) -> tuple[list[System], list[tuple[str, int]]]: """Load every (model, train_set) eval dir of one bucket that has all three eval CSVs. @@ -314,8 +315,6 @@ def load_systems( Model code names to look for, crossed with :data:`TRAIN_SETS`. data_root : Path Benchmark root laid out as ``///``. - cp_frozen_norm : bool - Forwarded to :class:`System`. Returns ------- @@ -338,7 +337,7 @@ def load_systems( "using common" ) blocks = sorted(common) - return [System(p, blocks, cp_frozen_norm) for p in paths], blocks + return [System(p, blocks) for p in paths], blocks def t_rules() -> dict[str, Callable[[int], list[int]] | None]: @@ -445,7 +444,6 @@ def run( out: Path, designs: list[str] | None = None, data_root: Path = DATA_ROOT, - cp_frozen_norm: bool = False, ) -> None: """Score every design of one bucket over ``reps`` random FOV draws and write the records. @@ -466,11 +464,9 @@ def run( Restrict to these design names. data_root : Path Benchmark root. - cp_frozen_norm : bool - Forwarded to :class:`System`. """ models = MODELS_3D + (MODELS_2D if include_2d else []) - systems, blocks = load_systems(org, bucket, models, data_root, cp_frozen_norm) + systems, blocks = load_systems(org, bucket, models, data_root) if len(systems) < 3: print(f"{org}/{bucket}: only {len(systems)} systems, skipping") return @@ -604,7 +600,6 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--include-2d", action="store_true") ap.add_argument("--summarize-only", action="store_true") ap.add_argument("--designs", nargs="*", default=None, help="restrict to these design names, e.g. fov12_t_ord") - ap.add_argument("--cp-frozen-norm", action="store_true", help="z-score GLCM+ subsets with full-pool statistics") ap.add_argument("--data-root", type=Path, default=DATA_ROOT) ap.add_argument("--out", type=Path, required=True) a = ap.parse_args(argv) @@ -612,7 +607,7 @@ def main(argv: list[str] | None = None) -> int: if not a.summarize_only: for org in a.org: for bucket in a.bucket: - run(org, bucket, a.reps, a.include_2d, a.out, a.designs, a.data_root, a.cp_frozen_norm) + run(org, bucket, a.reps, a.include_2d, a.out, a.designs, a.data_root) summary = summarize(a.out) pd.set_option("display.width", 250) print(summary.round(3).to_string()) diff --git a/applications/dynacell/tools/lite_subset_sim_test.py b/applications/dynacell/tools/lite_subset_sim_test.py index 93e65f92d..c0c27c5fc 100644 --- a/applications/dynacell/tools/lite_subset_sim_test.py +++ b/applications/dynacell/tools/lite_subset_sim_test.py @@ -22,13 +22,42 @@ import pytest from lite_subset_sim import System, load_systems, mmd2_from_features, poly3_features, poly_kernel +from dynacell.evaluation.cp_reference import DatasetFit, fit_cp_reference, write_cp_reference from dynacell.evaluation.feature_metrics import _kid, _median_cosine_similarity _BLOCKS = [("A/1/0", 0), ("A/1/0", 1), ("A/1/1", 0), ("A/1/1", 1), ("A/1/2", 0), ("A/1/2", 1)] _CELLS_PER_BLOCK = [7, 12, 9, 15, 5, 11] _D_DEEP = 6 _D_CP = 5 -_CP_KEEP = [True, False, True, True, True] +_DATASET = "synthetic-set" + + +def _write_reference(path: Path) -> dict: + """Fit a real CP reference whose mask drops column 1 and whose scaler is far from the eval's cells. + + The reference GT is offset and rescaled relative to the eval dir's cells, so the + reference transform and the retired per-side z-score give clearly different KIDs. + """ + rng = np.random.default_rng(99) + gt = rng.normal(size=(200, _D_CP)) * 2.0 + 3.0 + gt[:, 1] = 0.0 # constant -> dropped by the GT-only variance filter + fit = DatasetFit( + dataset=_DATASET, + cells=gt, + record={"positions": sorted({f for f, _ in _BLOCKS}), "gt_cache_dir": None, "cp_cache_built_at": None}, + in_mask_fit=True, + ) + payload = fit_cp_reference( + [fit], target_name="nucleus", feature_names=tuple(f"f{i}" for i in range(_D_CP)), cp_identity={}, lite={} + ) + write_cp_reference(payload, path) + return payload + + +def _reference_transform(payload: dict, x: np.ndarray) -> np.ndarray: + """Apply the pipeline's CP transform spelled out from the payload: mask, then this dataset's scaler.""" + scaler = payload["scalers"][_DATASET] + return (x[:, np.array(payload["keep_mask"])] - np.array(scaler["mean"])) / np.array(scaler["std"]) def _pipeline_kid(pred: np.ndarray, gt: np.ndarray) -> float: @@ -36,8 +65,8 @@ def _pipeline_kid(pred: np.ndarray, gt: np.ndarray) -> float: return _kid(pred, gt, 100, 1000, 2020)[0] -def _write_eval_dir(path: Path, seed: int) -> dict[str, dict[str, np.ndarray]]: - """Write a synthetic eval dir and return its per-cell arrays keyed by extractor token. +def _write_eval_dir(path: Path, seed: int, reference: Path) -> dict[str, dict[str, np.ndarray]]: + """Write a synthetic eval dir scored in ``reference`` and return its per-cell arrays by extractor token. Pred embeddings are GT shifted and rescaled, so every KID is well away from zero. """ @@ -61,7 +90,10 @@ def _write_eval_dir(path: Path, seed: int) -> dict[str, dict[str, np.ndarray]]: path / "embeddings" / f"{side}_{tok}_single_cell_embeddings.npz", embeddings=arr, fov=fov, timepoint=tp ) cells[tok] = {"pred": pred, "gt": gt, "fov": fov, "timepoint": tp} - (path / "cp_selected_feature_mask.json").write_text(json.dumps({"keep_mask": _CP_KEEP})) + sha256 = json.loads(reference.read_text())["sha256"] + (path / "cp_selected_feature_mask.json").write_text( + json.dumps({"reference_path": str(reference), "reference_sha256": sha256, "dataset": _DATASET}) + ) return cells @@ -74,17 +106,19 @@ def _cells_of(cells: dict[str, np.ndarray], sel: np.ndarray) -> tuple[np.ndarray return cells["pred"][idx], cells["gt"][idx] -def _zscore(x: np.ndarray) -> np.ndarray: - """Z-score each column as the pipeline does for GLCM+ features.""" +def _per_side_zscore(x: np.ndarray) -> np.ndarray: + """Apply the retired GLCM+ transform (each side by its own stats); used only to show discrimination.""" return (x - x.mean(0)) / (x.std(0) + 1e-8) @pytest.fixture -def system(tmp_path: Path) -> tuple[System, dict[str, dict[str, np.ndarray]]]: - """One synthetic system over :data:`_BLOCKS` plus its explicit cell arrays.""" +def system(tmp_path: Path) -> tuple[System, dict[str, dict[str, np.ndarray]], dict]: + """One synthetic system over :data:`_BLOCKS`, its explicit cell arrays, and its CP reference payload.""" + reference = tmp_path / "nucleus.json" + payload = _write_reference(reference) path = tmp_path / "nucleus" / "fnet3d_paper" / "ipsc" / "ipsc" - cells = _write_eval_dir(path, seed=0) - return System(path, _BLOCKS), cells + cells = _write_eval_dir(path, seed=0, reference=reference) + return System(path, _BLOCKS), cells, payload def test_poly3_feature_map_reproduces_the_kernel() -> None: @@ -102,22 +136,26 @@ def test_poly3_feature_map_reproduces_the_kernel() -> None: @pytest.mark.parametrize("sel", [np.arange(len(_BLOCKS)), np.array([1, 2, 5])], ids=["all", "subset"]) def test_block_sum_kid_matches_pipeline_kid(system, sel: np.ndarray) -> None: """Block-sum deep KID and direct GLCM+ KID equal ``feature_metrics._kid`` on the same cells.""" - sysobj, cells = system + sysobj, cells, payload = system out = sysobj.metrics(sel) pred, gt = _cells_of(cells["dinov3"], sel) assert out["DINOv3_KID"] == pytest.approx(_pipeline_kid(pred, gt), rel=1e-4) assert out["DINOv3_MedCos"] == pytest.approx(_median_cosine_similarity(pred, gt), rel=1e-10) assert out["n_cells"] == len(pred) - pred, gt = _cells_of(cells["cp"], sel) - pred, gt = _zscore(pred[:, _CP_KEEP]), _zscore(gt[:, _CP_KEEP]) - assert out["CP_KID"] == pytest.approx(_pipeline_kid(pred, gt), rel=1e-4) - # Guard against a vacuous pass: both estimates are far from zero (z-scored GLCM+ KID can be negative). + raw_pred, raw_gt = _cells_of(cells["cp"], sel) + expected = _pipeline_kid(_reference_transform(payload, raw_pred), _reference_transform(payload, raw_gt)) + assert out["CP_KID"] == pytest.approx(expected, rel=1e-4) + # Guard against a vacuous pass: both estimates are far from zero, and the retired per-side + # z-score on the same cells gives a clearly different CP KID. assert out["DINOv3_KID"] > 0.1 and abs(out["CP_KID"]) > 0.01 + mask = np.array(payload["keep_mask"]) + old = _pipeline_kid(_per_side_zscore(raw_pred[:, mask]), _per_side_zscore(raw_gt[:, mask])) + assert out["CP_KID"] != pytest.approx(old, rel=0.1) def test_multiset_weights_equal_brute_force_duplication(system) -> None: """A block drawn twice scores exactly as if its cells were physically duplicated.""" - sysobj, cells = system + sysobj, cells, payload = system sel = np.array([0, 0, 1, 3, 3, 3, 5]) out = sysobj.metrics(sel) pred, gt = _cells_of(cells["dinov3"], sel) @@ -126,7 +164,7 @@ def test_multiset_weights_equal_brute_force_duplication(system) -> None: assert out["DINOv3_MedCos"] == pytest.approx(float(np.median(cos)), rel=1e-12) cp_pred, cp_gt = _cells_of(cells["cp"], sel) assert out["CP_KID"] == pytest.approx( - _pipeline_kid(_zscore(cp_pred[:, _CP_KEEP]), _zscore(cp_gt[:, _CP_KEEP])), rel=1e-4 + _pipeline_kid(_reference_transform(payload, cp_pred), _reference_transform(payload, cp_gt)), rel=1e-4 ) assert out["SI_SSIM"] == pytest.approx(float(np.mean(sysobj.row_vals["SI_SSIM"][sel])), rel=1e-12) # Guard against a vacuous pass: the multiset differs from the plain set of its blocks. @@ -135,7 +173,7 @@ def test_multiset_weights_equal_brute_force_duplication(system) -> None: def test_kid_is_nan_below_the_pipeline_minimum(system) -> None: """Under 16 cells both the block-sum KID and the pipeline KID are NaN; at 16+ neither is.""" - sysobj, cells = system + sysobj, cells, _ = system small = np.array([4]) # 5 cells out = sysobj.metrics(small) assert np.isnan(out["DINOv3_KID"]) and np.isnan(out["CP_KID"]) @@ -147,8 +185,10 @@ def test_kid_is_nan_below_the_pipeline_minimum(system) -> None: def test_load_systems_keeps_only_the_common_rows(tmp_path: Path) -> None: """Systems whose row sets differ are scored on the intersection.""" + reference = tmp_path / "nucleus.json" + _write_reference(reference) for model in ("fnet3d_paper", "unetvit3d"): - _write_eval_dir(tmp_path / "nucleus" / model / "ipsc" / "ipsc", seed=2) + _write_eval_dir(tmp_path / "nucleus" / model / "ipsc" / "ipsc", seed=2, reference=reference) px = tmp_path / "nucleus" / "unetvit3d" / "ipsc" / "ipsc" / "pixel_metrics.csv" pd.read_csv(px).iloc[:-1].to_csv(px, index=False) systems, blocks = load_systems("nucleus", "ipsc", ["fnet3d_paper", "unetvit3d"], data_root=tmp_path) @@ -159,7 +199,9 @@ def test_load_systems_keeps_only_the_common_rows(tmp_path: Path) -> None: def test_misaligned_embeddings_raise(tmp_path: Path) -> None: """Pred and GT embeddings from different cells are refused, not skipped.""" path = tmp_path / "nucleus" / "fnet3d_paper" / "ipsc" / "ipsc" - _write_eval_dir(path, seed=3) + reference = tmp_path / "nucleus.json" + _write_reference(reference) + _write_eval_dir(path, seed=3, reference=reference) npz = path / "embeddings" / "gt_dinov3_single_cell_embeddings.npz" with np.load(npz) as z: arrays = dict(z) From 4be7b5351384bef8ac8c924263761a1685d90199 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 18:56:12 -0700 Subject: [PATCH 08/41] test(dynacell/eval): pin CP scaler selection to each condition's dataset Nothing checked which scaler an eval was bound to: making eval_cp_space return the first scaler passed every test. The new tests fit a reference with three sets whose scalers differ plus one lite set. They drive the real eval_cp_space and the grouped per-condition driver, and assert that each condition gets its own dataset's mean/std, a lite set gets its parent's, the stamp is the scoring reference, and an unknown dataset is refused. The first-scaler mutation now fails all three. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- applications/dynacell/tests/_eval_fixtures.py | 54 ++++++--- .../test_evaluation_cp_scaler_selection.py | 112 ++++++++++++++++++ 2 files changed, 147 insertions(+), 19 deletions(-) create mode 100644 applications/dynacell/tests/test_evaluation_cp_scaler_selection.py diff --git a/applications/dynacell/tests/_eval_fixtures.py b/applications/dynacell/tests/_eval_fixtures.py index 25a0e0f11..9b465778c 100644 --- a/applications/dynacell/tests/_eval_fixtures.py +++ b/applications/dynacell/tests/_eval_fixtures.py @@ -211,17 +211,25 @@ def build_fixture(root: Path): CP_TEST_DATASET = "synthetic-set" -def make_cp_reference(config, path: Path, seed: int = 0, n_cells: int = 64) -> str: +def make_cp_reference( + config, + path: Path, + seed: int = 0, + n_cells: int = 64, + datasets: tuple[str, ...] = (CP_TEST_DATASET,), + lite: dict[str, str] | None = None, +) -> str: """Fit a real CP reference matching ``config``'s CP recipe and point ``config`` at it. - The GT matrix is synthetic, but the reference is built by the production + The GT matrices are synthetic, but the reference is built by the production :func:`~dynacell.evaluation.cp_reference.fit_cp_reference` with the identity and column names :func:`~dynacell.evaluation.cp_reference.cp_space` derives from the - same config. It fits one scaler, for :data:`CP_TEST_DATASET` over the plate's - ``A/1/{i}`` positions, and records no GT cache stamp (so the re-cache check is - skipped). Sets ``feature_metrics.cp`` (GLCM off unless already configured), - ``feature_metrics.cp.reference_path`` and ``benchmark.dataset_ref.dataset``, and - returns the reference's sha256. + same config. It fits one scaler per entry of ``datasets`` over the plate's + ``A/1/{i}`` positions; dataset ``i`` is drawn with offset ``10 * i`` and scale + ``1 + i``, so every scaler differs. ``lite`` maps lite dataset -> parent. Sets + ``feature_metrics.cp`` (GLCM off unless already configured), + ``feature_metrics.cp.reference_path`` and ``benchmark.dataset_ref.dataset`` + (the first dataset), and returns the reference's sha256. """ # Imported here like the other helpers in this file: live_pipeline_module() drops # every dynacell.evaluation.* module, so a top-level binding would go stale. @@ -234,20 +242,28 @@ def make_cp_reference(config, path: Path, seed: int = 0, n_cells: int = 64) -> s {"norm": {"p_lo": 1.0, "p_hi": 99.0}, "glcm": {"enabled": False}, "reference_path": None}, merge=True, ) - OmegaConf.update(config, "benchmark", {"dataset_ref": {"dataset": CP_TEST_DATASET, "target": "sec61b"}}, merge=True) + OmegaConf.update(config, "benchmark", {"dataset_ref": {"dataset": datasets[0], "target": "sec61b"}}, merge=True) identity, names = cp_space(config) - fit = DatasetFit( - dataset=CP_TEST_DATASET, - cells=np.random.default_rng(seed).standard_normal((n_cells, len(names))), - record={ - "positions": [f"A/1/{i}" for i in range(N_POSITIONS)], - "gt_cache_dir": None, - "cp_cache_built_at": None, - }, - in_mask_fit=True, - ) + rng = np.random.default_rng(seed) + fits = [ + DatasetFit( + dataset=name, + cells=rng.standard_normal((n_cells, len(names))) * (1 + i) + 10.0 * i, + record={ + "positions": [f"A/1/{p}" for p in range(N_POSITIONS)], + "gt_cache_dir": None, + "cp_cache_built_at": None, + }, + in_mask_fit=True, + ) + for i, name in enumerate(datasets) + ] + lite_entries = { + name: {"parent": parent, "gt_cache_dir": None, "cp_cache_built_at": None} + for name, parent in (lite or {}).items() + } payload = fit_cp_reference( - [fit], target_name=config.target_name, feature_names=names, cp_identity=identity, lite={} + fits, target_name=config.target_name, feature_names=names, cp_identity=identity, lite=lite_entries ) write_cp_reference(payload, path, force=True) config.feature_metrics.cp.reference_path = str(path) diff --git a/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py b/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py new file mode 100644 index 000000000..c52046866 --- /dev/null +++ b/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py @@ -0,0 +1,112 @@ +"""Each eval condition is scored with its own dataset's CP scaler. + +Drives the real :func:`~dynacell.evaluation.cp_reference.eval_cp_space` and the real +grouped per-condition driver (:func:`~dynacell.evaluation.pipeline.evaluate_predictions_grouped`) +against a reference holding three datasets with different scalers plus one lite +dataset. Only the heavy per-condition work (``evaluate_predictions``, +``save_metrics``, model loading, the manifest splice) is replaced by recorders. +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest +from omegaconf import OmegaConf + +from ._eval_fixtures import build_eval_config, live_pipeline_module, make_cp_reference + +_SETS = ("set-a", "set-b", "set-c") +_LITE = {"set-b-lite": "set-b"} + + +def _config(tmp_path: Path): + """Feature-metrics eval config pointed at a three-set + one-lite reference.""" + config = build_eval_config( + tmp_path / "pred.zarr", + tmp_path / "gt.zarr", + tmp_path / "g", + tmp_path / "p", + tmp_path / "out", + executor="serial", + fov_workers=1, + ) + config.compute_feature_metrics = True + make_cp_reference(config, tmp_path / "er.json", datasets=_SETS, lite=_LITE) + return config + + +def test_eval_cp_space_binds_the_config_dataset(tmp_path: Path) -> None: + """``eval_cp_space`` returns the scaler of ``benchmark.dataset_ref.dataset``; lite gets its parent's.""" + pipeline = live_pipeline_module() + config = _config(tmp_path) + means = {} + for dataset in (*_SETS, *_LITE): + config.benchmark.dataset_ref.dataset = dataset + space = pipeline.eval_cp_space(config) + assert space.dataset == dataset + means[dataset] = space.mean + for a in _SETS: + for b in _SETS: + if a != b: + assert not np.allclose(means[a], means[b]) + assert pipeline.eval_cp_space(config).scaler_dataset == "set-b" + np.testing.assert_array_equal(means["set-b-lite"], means["set-b"]) + + +def test_grouped_conditions_each_get_their_own_scaler(tmp_path: Path, monkeypatch) -> None: + """The grouped driver binds every condition to its own dataset, and stamps that reference.""" + pipeline = live_pipeline_module() + base = _config(tmp_path) + expected = { + d: pipeline.eval_cp_space(OmegaConf.merge(base, {"benchmark": {"dataset_ref": {"dataset": d}}})) + for d in (*_SETS, *_LITE) + } + conditions = [ + {"name": d, "benchmark": {"dataset_ref": {"dataset": d}}, "save": {"save_dir": str(tmp_path / d)}} + for d in (*_SETS, *_LITE) + ] + config = OmegaConf.merge( + base, + { + "conditions": conditions, + "force_recompute": {"final_metrics": True}, + "cross_condition_probe": {"enabled": False}, + }, + ) + seen: dict[str, object] = {} + stamped: dict[str, str | None] = {} + + def _fake_evaluate_predictions(cfg, *, models, cp_space): + seen[cfg.benchmark.dataset_ref.dataset] = cp_space + return [], [], [] + + def _fake_save_metrics(cfg, *args, cp_reference_sha256, **kwargs): + stamped[cfg.benchmark.dataset_ref.dataset] = cp_reference_sha256 + + monkeypatch.setattr(pipeline, "apply_dataset_ref", lambda cfg: None) + monkeypatch.setattr(pipeline, "load_eval_models", lambda cfg: object()) + monkeypatch.setattr(pipeline, "evaluate_predictions", _fake_evaluate_predictions) + monkeypatch.setattr(pipeline, "save_metrics", _fake_save_metrics) + + pipeline.evaluate_predictions_grouped(config) + + assert sorted(seen) == sorted(expected) + for dataset, space in seen.items(): + assert space.dataset == dataset + assert space.scaler_dataset == _LITE.get(dataset, dataset) + np.testing.assert_array_equal(space.mean, expected[dataset].mean) + np.testing.assert_array_equal(space.std, expected[dataset].std) + assert stamped[dataset] == space.reference_sha256 + np.testing.assert_array_equal(seen["set-b-lite"].mean, seen["set-b"].mean) + assert not np.allclose(seen["set-a"].mean, seen["set-c"].mean) + + +def test_unknown_dataset_is_refused(tmp_path: Path) -> None: + """A condition whose dataset has no scaler fails instead of borrowing another set's.""" + pipeline = live_pipeline_module() + config = _config(tmp_path) + config.benchmark.dataset_ref.dataset = "unknown-set" + with pytest.raises(KeyError, match="no scaler for dataset 'unknown-set'"): + pipeline.eval_cp_space(config) From f4b4153cb351ddd2e82daff9b0e196f68f70037b Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 18:58:29 -0700 Subject: [PATCH 09/41] fix(dynacell/eval): hash only what moves CP numbers in the reference The content hash covered fit provenance too (GT-cache built_at, cache dirs, positions, cell counts, GT-matrix hashes), so rebuilding after a harmless re-cache changed the hash and invalidated every stamped final-metrics cache although no scaler moved. Schema 3 splits the payload. The hashed part is recipe identity, criteria, feature names, keep-mask, per-dataset mean/std/floored features, and the lite -> parent map. Provenance moves to an unhashed `fit` section, which the eval checks still read. write_cp_reference rewrites same-hash, new-provenance references without --force, so the checks see the new built_at. It stays a no-op when both are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/cp_reference.py | 70 ++++++++++++------- .../dynacell/evaluation/cp_reference_test.py | 53 ++++++++++++-- .../tests/test_evaluation_timepoint_guard.py | 5 +- .../dynacell/tools/build_cp_reference.py | 18 +++-- .../dynacell/tools/build_cp_reference_test.py | 15 ++-- 5 files changed, 114 insertions(+), 47 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index 2088ed0d6..5b11b54cd 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -66,7 +66,7 @@ from dynacell.evaluation.pipeline_cache import cp_recipe_identity #: Schema of the reference JSON. Bump on any change to its keys or their meaning. -CP_REFERENCE_SCHEMA = 2 +CP_REFERENCE_SCHEMA = 3 #: A per-dataset std below this fraction of the feature's pooled GT std is floored to it. STD_FLOOR_FRACTION = 1e-3 @@ -101,10 +101,13 @@ ), } -#: Keys excluded from the content hash: the hash itself, and the build timestamp, -#: so rebuilding from identical GT caches yields an identical hash and does not -#: needlessly invalidate every final-metrics cache stamped with it. -_UNHASHED_KEYS = frozenset({"sha256", "created_at"}) +#: Keys excluded from the content hash. The hash covers only what changes CP numbers +#: (recipe identity, criteria, feature names, keep-mask, every scaler's mean/std and +#: floored features, the lite -> parent map). ``fit`` holds the fit provenance the eval +#: checks read (positions, cell counts, GT-cache ``built_at``, GT-matrix hashes), so a +#: rebuild after a harmless GT re-cache that leaves every scaler unchanged keeps the +#: hash and invalidates no final-metrics cache stamped with it. +_UNHASHED_KEYS = frozenset({"sha256", "created_at", "fit"}) def _build_command(target_name: str) -> str: @@ -210,7 +213,8 @@ def fit_cp_reference( Dataset-independent CP recipe identity (from :func:`cp_space`). lite : dict ``{lite dataset: {"parent": ..., "gt_cache_dir": ..., "cp_cache_built_at": ...}}``. - Each parent must be one of ``fits``. + Each parent must be one of ``fits``. Only ``parent`` enters the hash; the + rest is fit provenance. Returns ------- @@ -252,16 +256,19 @@ def fit_cp_reference( floor = STD_FLOOR_FRACTION * pooled_std scalers: dict[str, dict[str, Any]] = {} + provenance: dict[str, dict[str, Any]] = {} for fit in fits: kept = np.asarray(fit.cells, dtype=np.float64)[:, keep_mask] std = kept.std(axis=0) floored = std < floor scalers[fit.dataset] = { - **fit.record, - "in_mask_fit": fit.in_mask_fit, "mean": [float(v) for v in kept.mean(axis=0)], "std": [float(v) for v in np.where(floored, floor, std)], "floored_features": [n for n, f in zip(kept_names, floored, strict=True) if f], + } + provenance[fit.dataset] = { + **fit.record, + "in_mask_fit": fit.in_mask_fit, "n_cells": int(kept.shape[0]), "gt_matrix_sha256": gt_matrix_sha256(fit.cells), } @@ -281,14 +288,18 @@ def fit_cp_reference( "std_floor_fraction_of_pooled": STD_FLOOR_FRACTION, }, "cp_identity": cp_identity, - "mask_fit": { - "datasets": [f.dataset for f in fits if f.in_mask_fit], - "n_cells": int(pooled.shape[0]), - "gt_matrix_sha256": gt_matrix_sha256(pooled), - "pooled_std": [float(v) for v in pooled_std], - }, "scalers": scalers, - "lite": lite, + "lite": {name: {"parent": entry["parent"]} for name, entry in lite.items()}, + "fit": { + "mask_fit": { + "datasets": [f.dataset for f in fits if f.in_mask_fit], + "n_cells": int(pooled.shape[0]), + "gt_matrix_sha256": gt_matrix_sha256(pooled), + "pooled_std": [float(v) for v in pooled_std], + }, + "datasets": provenance, + "lite": {name: {k: v for k, v in entry.items() if k != "parent"} for name, entry in lite.items()}, + }, "created_at": datetime.now(UTC).isoformat(), } payload["sha256"] = payload_sha256(payload) @@ -310,8 +321,10 @@ def write_cp_reference(payload: dict[str, Any], path: Path, *, force: bool = Fal Returns ------- bool - ``True`` if the file was written, ``False`` if an identical reference was - already there (no-op). + ``True`` if the file was written, ``False`` if an identical reference (same + hash AND same fit provenance) was already there (no-op). Same hash with new + provenance -- a harmless GT re-cache -- is written without ``force``: no CP + number changes, and the eval checks need the new ``built_at``. Raises ------ @@ -320,13 +333,15 @@ def write_cp_reference(payload: dict[str, Any], path: Path, *, force: bool = Fal replacing it changes CP values and invalidates every eval stamped with it. """ if path.exists(): - existing = json.loads(path.read_text())["sha256"] - if existing == payload["sha256"]: - return False - if not force: + existing = json.loads(path.read_text()) + if existing["sha256"] == payload["sha256"]: + if existing["fit"] == payload["fit"]: + return False + elif not force: raise FileExistsError( - f"{path} holds a different CP reference (sha256 {existing[:12]} vs new {payload['sha256'][:12]}); " - "replacing it invalidates every eval scored with it. Pass --force to replace it." + f"{path} holds a different CP reference (sha256 {existing['sha256'][:12]} vs new " + f"{payload['sha256'][:12]}); replacing it invalidates every eval scored with it. " + "Pass --force to replace it." ) path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") @@ -474,6 +489,7 @@ class CPReference: cp_identity: dict[str, Any] scalers: dict[str, dict[str, Any]] lite: dict[str, dict[str, Any]] + fit: dict[str, Any] def for_dataset(self, dataset: str) -> DatasetCPSpace: """Bind the reference to one eval dataset (a lite dataset gets its parent's scaler). @@ -492,7 +508,8 @@ def for_dataset(self, dataset: str) -> DatasetCPSpace: ) scaler_dataset = self.lite[dataset]["parent"] if is_lite else dataset scaler = self.scalers[scaler_dataset] - own = self.lite[dataset] if is_lite else scaler + scaler_fit = self.fit["datasets"][scaler_dataset] + own = self.fit["lite"][dataset] if is_lite else scaler_fit return DatasetCPSpace( reference_path=self.path, reference_sha256=self.sha256, @@ -504,8 +521,8 @@ def for_dataset(self, dataset: str) -> DatasetCPSpace: keep_mask=self.keep_mask, mean=np.asarray(scaler["mean"], dtype=np.float64), std=np.asarray(scaler["std"], dtype=np.float64), - fit_positions=frozenset(scaler["positions"]), - fit_n_cells=int(scaler["n_cells"]), + fit_positions=frozenset(scaler_fit["positions"]), + fit_n_cells=int(scaler_fit["n_cells"]), gt_cache_dir=own["gt_cache_dir"], cp_cache_built_at=own["cp_cache_built_at"], ) @@ -556,6 +573,7 @@ def load_cp_reference(path: Path, *, target_name: str) -> CPReference: cp_identity=payload["cp_identity"], scalers=payload["scalers"], lite=payload["lite"], + fit=payload["fit"], ) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py index 2244f68a1..3eeb53504 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -93,9 +93,9 @@ def test_mask_is_pooled_over_the_mask_fit_sets_only(two_sets) -> None: # HEK entering the fit would change the mask: its offset reshapes the pooled correlations. with_hek = select_gt_features(np.vstack([cells["set-a"], cells["set-b"], cells["hek"]])) assert not np.array_equal(with_hek, expected) - assert payload["mask_fit"]["datasets"] == ["set-a", "set-b"] - assert payload["mask_fit"]["n_cells"] == 600 - assert payload["scalers"]["hek"]["in_mask_fit"] is False + assert payload["fit"]["mask_fit"]["datasets"] == ["set-a", "set-b"] + assert payload["fit"]["mask_fit"]["n_cells"] == 600 + assert payload["fit"]["datasets"]["hek"]["in_mask_fit"] is False def test_each_set_gets_its_own_gt_scaler(two_sets) -> None: @@ -106,7 +106,7 @@ def test_each_set_gets_its_own_gt_scaler(two_sets) -> None: z = space.transform(cells[name]) np.testing.assert_allclose(z.mean(axis=0), 0.0, atol=1e-9) np.testing.assert_allclose(z.std(axis=0), 1.0, atol=1e-9) - assert payload["scalers"][name]["n_cells"] == 300 + assert payload["fit"]["datasets"][name]["n_cells"] == 300 # set-b's GT in set-a's space is far from standardized: the scalers really differ. assert np.abs(ref.for_dataset("set-a").transform(cells["set-b"]).mean(axis=0)).max() > 5 lite = ref.for_dataset("set-a-lite") @@ -165,6 +165,51 @@ def test_hash_ignores_build_time_and_detects_edits(tmp_path: Path) -> None: load_cp_reference(path, target_name="er") +def test_harmless_recache_keeps_the_hash(tmp_path: Path) -> None: + """A rebuild whose scalers are identical but whose fit provenance moved keeps the hash. + + The new provenance (e.g. a GT cache re-stamped with the same values) is written + without ``force``, so the eval checks see the new ``built_at`` while no + final-metrics cache stamped with the old hash is invalidated. + """ + cells = _gt() + first = fit_cp_reference( + [_fit("set-a", cells, built_at="t0")], target_name="er", feature_names=_NAMES, cp_identity={}, lite={} + ) + fit_b = _fit("set-a", cells, built_at="t1") + fit_b.record["positions"] = [*_POSITIONS] + fit_b.record["gt_cache_dir"] = "/moved/cache" + second = fit_cp_reference([fit_b], target_name="er", feature_names=_NAMES, cp_identity={}, lite={}) + assert first["sha256"] == second["sha256"] + assert first["fit"] != second["fit"] + + path = tmp_path / "er.json" + write_cp_reference(first, path) + assert write_cp_reference(second, path) is True # same hash, new provenance: no --force needed + assert json.loads(path.read_text())["fit"]["datasets"]["set-a"]["cp_cache_built_at"] == "t1" + assert write_cp_reference(second, path) is False + + +def test_changed_scaler_changes_the_hash() -> None: + """Anything that moves a CP number -- here one dataset's mean -- changes the hash.""" + payload = fit_cp_reference( + [_fit("set-a", _gt()), _fit("set-b", _gt(seed=1))], + target_name="er", + feature_names=_NAMES, + cp_identity={}, + lite={"set-a-lite": {"parent": "set-a", "gt_cache_dir": None, "cp_cache_built_at": None}}, + ) + moved = json.loads(json.dumps(payload)) + moved["scalers"]["set-b"]["mean"][0] += 1e-9 + assert payload_sha256(moved) != payload["sha256"] + relinked = json.loads(json.dumps(payload)) + relinked["lite"]["set-a-lite"]["parent"] = "set-b" + assert payload_sha256(relinked) != payload["sha256"] + provenance_only = json.loads(json.dumps(payload)) + provenance_only["fit"]["datasets"]["set-b"]["n_cells"] = 1 + assert payload_sha256(provenance_only) == payload["sha256"] + + def test_write_is_atomic_and_refuses_a_different_reference(tmp_path: Path) -> None: """Identical hash is a no-op; a different one needs force; no tmp file is left behind.""" first = fit_cp_reference([_fit("set-a", _gt())], target_name="er", feature_names=_NAMES, cp_identity={}, lite={}) diff --git a/applications/dynacell/tests/test_evaluation_timepoint_guard.py b/applications/dynacell/tests/test_evaluation_timepoint_guard.py index 5cccbd4fc..0da98425e 100644 --- a/applications/dynacell/tests/test_evaluation_timepoint_guard.py +++ b/applications/dynacell/tests/test_evaluation_timepoint_guard.py @@ -18,7 +18,6 @@ from iohub.ngff import open_ome_zarr from dynacell.evaluation.cache import cache_paths, save_manifest -from dynacell.evaluation.cp_reference import payload_sha256 from dynacell.evaluation.model_loader import EvalModels from dynacell.evaluation.provenance import PROVENANCE_FILENAME, write_metrics_provenance @@ -293,10 +292,10 @@ def test_gt_recache_after_the_reference_is_refused_up_front(tmp_path: Path, monk {"artifacts": {"cp_features": {"path": "features/cp.zarr", "built_at": "2026-09-26T00:00:00+00:00"}}}, ) reference = json.loads((tmp_path / "cp_reference.json").read_text()) - reference["scalers"][CP_TEST_DATASET].update( + # Fit provenance is outside the content hash, so it can be edited without re-hashing. + reference["fit"]["datasets"][CP_TEST_DATASET].update( gt_cache_dir=str(tmp_path / "gt_cache"), cp_cache_built_at="2026-09-01T00:00:00+00:00" ) - reference["sha256"] = payload_sha256(reference) (tmp_path / "cp_reference.json").write_text(json.dumps(reference)) monkeypatch.setattr(pipeline, "load_eval_models", lambda *a, **k: pytest.fail("models loaded")) cp_space = pipeline.eval_cp_space(config) diff --git a/applications/dynacell/tools/build_cp_reference.py b/applications/dynacell/tools/build_cp_reference.py index d29afaa9c..b21ce05da 100644 --- a/applications/dynacell/tools/build_cp_reference.py +++ b/applications/dynacell/tools/build_cp_reference.py @@ -231,15 +231,18 @@ def _summary(payload: dict[str, Any]) -> str: lines = [ f"kept {len(payload['kept_feature_names'])}/{len(payload['feature_names'])}: {payload['kept_feature_names']}" ] - lines.append(f"mask fit: {payload['mask_fit']['n_cells']} cells from {payload['mask_fit']['datasets']}") - for name, s in sorted(payload["scalers"].items()): + fit = payload["fit"] + lines.append(f"mask fit: {fit['mask_fit']['n_cells']} cells from {fit['mask_fit']['datasets']}") + for name, scaler in sorted(payload["scalers"].items()): + d = fit["datasets"][name] lines.append( - f" {name}{' [mask]' if s['in_mask_fit'] else ''}: {len(s['positions'])} positions, " - f"{s['n_timepoints']} timepoints, {s['n_cells']} cells ({s['n_cells_dropped_nonfinite']} non-finite " - f"dropped), floored {s['floored_features'] or 'none'}" + f" {name}{' [mask]' if d['in_mask_fit'] else ''}: {len(d['positions'])} positions, " + f"{d['n_timepoints']} timepoints, {d['n_cells']} cells ({d['n_cells_dropped_nonfinite']} non-finite " + f"dropped), floored {scaler['floored_features'] or 'none'}" ) for name, entry in sorted(payload["lite"].items()): - lines.append(f" {name} -> scaler of {entry['parent']} (own cache built_at {entry['cp_cache_built_at']})") + built_at = fit["lite"][name]["cp_cache_built_at"] + lines.append(f" {name} -> scaler of {entry['parent']} (own cache built_at {built_at})") return "\n".join(lines) @@ -262,7 +265,8 @@ def main(argv: list[str] | None = None) -> None: print(f"dry run: would write {out} sha256={payload['sha256']} ({elapsed:.1f} s)") return written = write_cp_reference(payload, out, force=args.force) - print(f"{'wrote' if written else 'unchanged (identical hash)'} {out} sha256={payload['sha256']} ({elapsed:.1f} s)") + status = "wrote" if written else "unchanged (identical reference)" + print(f"{status} {out} sha256={payload['sha256']} ({elapsed:.1f} s)") if __name__ == "__main__": diff --git a/applications/dynacell/tools/build_cp_reference_test.py b/applications/dynacell/tools/build_cp_reference_test.py index 48517d34e..a6eeb54a7 100644 --- a/applications/dynacell/tools/build_cp_reference_test.py +++ b/applications/dynacell/tools/build_cp_reference_test.py @@ -153,23 +153,24 @@ def test_build_writes_a_loadable_reference(registry: Path) -> None: main(["--target", "er", "--leaves-root", str(leaves), "--out", str(out)]) payload = json.loads(out.read_text()) - assert payload["mask_fit"]["datasets"] == ["ds-a", "ds-b"] + fit = payload["fit"] + assert fit["mask_fit"]["datasets"] == ["ds-a", "ds-b"] assert sorted(payload["scalers"]) == ["ds-a", "ds-b", "ds-h"] - assert payload["scalers"]["ds-h"]["in_mask_fit"] is False - assert payload["lite"] == { + assert fit["datasets"]["ds-h"]["in_mask_fit"] is False + assert payload["lite"] == {"ds-a-lite": {"parent": "ds-a"}} + assert fit["lite"] == { "ds-a-lite": { - "parent": "ds-a", "target": "sec61b", "gt_cache_dir": str(registry / "caches" / "ds-a-lite"), "cp_cache_built_at": None, } } - a = payload["scalers"]["ds-a"] + a = {**payload["scalers"]["ds-a"], **fit["datasets"]["ds-a"]} assert a["n_cells"] == 2 * _T * 5 - 1 # one non-finite cell dropped assert a["n_cells_dropped_nonfinite"] == 1 assert a["positions"] == list(_POSITIONS) assert a["cp_cache_built_at"] == _BUILT_AT - assert payload["mask_fit"]["n_cells"] == 38 + assert fit["mask_fit"]["n_cells"] == 38 ref = load_cp_reference(out, target_name="er") assert ref.for_dataset("ds-a-lite").scaler_dataset == "ds-a" # Scalers differ per set: ds-b was drawn with a different offset and scale. @@ -183,7 +184,7 @@ def test_rebuild_is_a_no_op_and_a_changed_reference_needs_force(registry: Path, args = ["--target", "er", "--leaves-root", str(leaves), "--out", str(out)] main(args) main(args) - assert "unchanged (identical hash)" in capsys.readouterr().out + assert "unchanged (identical reference)" in capsys.readouterr().out with open_features_group(cache_paths(registry / "caches" / "ds-b"), "cp", mode="a") as group: write_features_to_group(group, "A/1/0", 0, np.ones((5, _N_FEATURES)) * 9.0) with pytest.raises(FileExistsError, match="--force"): From 409f4cf016a3ea7c4e45f7574a3edd327bdd0ee3 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 18:59:58 -0700 Subject: [PATCH 10/41] fix(dynacell/eval): refuse a non-lite CP scaler with no GT-cache stamp check_gt_cache skipped whenever no built_at was recorded, so a non-lite reference built without one silently disabled GT re-cache detection. The skip now applies only to lite datasets (their scaler is the parent's); a non-lite dataset without a recorded built_at raises. The test fixture now writes a real cp_features entry (full CP identity plus built_at) into the eval's GT cache and records it in the reference, as the builder does for real caches. The re-cache test moves that stamp instead of hand-editing the reference. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/cp_reference.py | 19 +++++++++++----- .../dynacell/evaluation/cp_reference_test.py | 3 +++ applications/dynacell/tests/_eval_fixtures.py | 21 +++++++++++++++--- .../tests/test_evaluation_timepoint_guard.py | 22 +++++++------------ 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index 5b11b54cd..724921ac7 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -396,10 +396,10 @@ def check_gt_cache(self, gt_cache_dir: str | None) -> None: """Refuse an eval whose GT CP cache was re-cached after the reference was fit. Compares the eval's GT cache manifest ``cp_features.built_at`` with the one - recorded at build time. A lite dataset is checked against its own cache - only when the build recorded one (its scaler is the parent's, so a lite - cache is otherwise not what the scaler describes); when nothing is - recorded the check is skipped. + recorded at build time. A non-lite dataset must have one recorded. A lite + dataset is checked against its own cache only when the build recorded one + (its scaler is the parent's, so a lite cache is otherwise not what the + scaler describes); when nothing is recorded the check is skipped. Parameters ---------- @@ -409,12 +409,19 @@ def check_gt_cache(self, gt_cache_dir: str | None) -> None: Raises ------ ValueError - If the eval reads a different GT cache dir than the recorded one. + If the eval reads a different GT cache dir than the recorded one, or a + non-lite dataset's reference records no ``built_at``. StaleCacheError If the recorded and current ``built_at`` differ. """ if self.cp_cache_built_at is None: - return + if self.is_lite: + return + raise ValueError( + f"{self.dataset}: the CP reference {self.reference_path} records no GT-cache built_at for this " + f"non-lite dataset, so a GT re-cache could not be detected. Rebuild it:\n" + f" {_build_command(self.target_name)}" + ) if gt_cache_dir is None or Path(gt_cache_dir) != Path(self.gt_cache_dir): raise ValueError( f"{self.dataset}: the CP reference was fit on GT cache {self.gt_cache_dir}, " diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py index 3eeb53504..84af50ec7 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -283,8 +283,11 @@ def test_gt_recache_is_detected(tmp_path: Path) -> None: def test_lite_without_a_recorded_cache_skips_the_recache_check(two_sets) -> None: + """A lite dataset with no recorded cache stamp skips the check; a non-lite one refuses.""" ref, _, _ = two_sets ref.for_dataset("set-a-lite").check_gt_cache(None) + with pytest.raises(ValueError, match="records no GT-cache built_at for this non-lite dataset"): + ref.for_dataset("set-b").check_gt_cache("/c/set-b") def test_staged_dataset_arrays_are_the_dataset_scaler_transform(two_sets, tmp_path: Path) -> None: diff --git a/applications/dynacell/tests/_eval_fixtures.py b/applications/dynacell/tests/_eval_fixtures.py index 9b465778c..116f4f0f5 100644 --- a/applications/dynacell/tests/_eval_fixtures.py +++ b/applications/dynacell/tests/_eval_fixtures.py @@ -226,13 +226,17 @@ def make_cp_reference( column names :func:`~dynacell.evaluation.cp_reference.cp_space` derives from the same config. It fits one scaler per entry of ``datasets`` over the plate's ``A/1/{i}`` positions; dataset ``i`` is drawn with offset ``10 * i`` and scale - ``1 + i``, so every scaler differs. ``lite`` maps lite dataset -> parent. Sets + ``1 + i``, so every scaler differs. ``lite`` maps lite dataset -> parent. Every + non-lite dataset records ``config.io.gt_cache_dir`` and a real ``built_at``, and a + matching ``cp_features`` entry (full CP identity) is written into that cache's + manifest, so the eval's re-cache check passes the way it does on real data. Sets ``feature_metrics.cp`` (GLCM off unless already configured), ``feature_metrics.cp.reference_path`` and ``benchmark.dataset_ref.dataset`` (the first dataset), and returns the reference's sha256. """ # Imported here like the other helpers in this file: live_pipeline_module() drops # every dynacell.evaluation.* module, so a top-level binding would go stale. + from dynacell.evaluation.cache import built_at_now, cache_paths, load_manifest, save_manifest from dynacell.evaluation.cp_reference import DatasetFit, cp_space, fit_cp_reference, write_cp_reference if OmegaConf.select(config, "feature_metrics.cp", default=None) is None: @@ -244,6 +248,17 @@ def make_cp_reference( ) OmegaConf.update(config, "benchmark", {"dataset_ref": {"dataset": datasets[0], "target": "sec61b"}}, merge=True) identity, names = cp_space(config) + gt_cache_dir = str(config.io.gt_cache_dir) + paths = cache_paths(gt_cache_dir) + manifest = load_manifest(paths) + built_at = built_at_now() + manifest["artifacts"]["cp_features"] = { + "path": "features/cp.zarr", + "built_at": built_at, + "spacing": list(config.pixel_metrics.spacing), + **identity, + } + save_manifest(paths, manifest) rng = np.random.default_rng(seed) fits = [ DatasetFit( @@ -251,8 +266,8 @@ def make_cp_reference( cells=rng.standard_normal((n_cells, len(names))) * (1 + i) + 10.0 * i, record={ "positions": [f"A/1/{p}" for p in range(N_POSITIONS)], - "gt_cache_dir": None, - "cp_cache_built_at": None, + "gt_cache_dir": gt_cache_dir, + "cp_cache_built_at": built_at, }, in_mask_fit=True, ) diff --git a/applications/dynacell/tests/test_evaluation_timepoint_guard.py b/applications/dynacell/tests/test_evaluation_timepoint_guard.py index 0da98425e..c4ec3b29a 100644 --- a/applications/dynacell/tests/test_evaluation_timepoint_guard.py +++ b/applications/dynacell/tests/test_evaluation_timepoint_guard.py @@ -17,11 +17,11 @@ import pytest from iohub.ngff import open_ome_zarr -from dynacell.evaluation.cache import cache_paths, save_manifest +from dynacell.evaluation.cache import cache_paths, load_manifest, save_manifest from dynacell.evaluation.model_loader import EvalModels from dynacell.evaluation.provenance import PROVENANCE_FILENAME, write_metrics_provenance -from ._eval_fixtures import CP_TEST_DATASET, build_eval_config, live_pipeline_module, make_cp_reference +from ._eval_fixtures import build_eval_config, live_pipeline_module, make_cp_reference D, H, W = 3, 8, 8 @@ -286,19 +286,13 @@ def test_partial_position_walk_is_refused_for_cp(tmp_path: Path, monkeypatch) -> def test_gt_recache_after_the_reference_is_refused_up_front(tmp_path: Path, monkeypatch) -> None: """A GT CP cache whose ``built_at`` moved since the build fails before any model load.""" pipeline = live_pipeline_module() - config, _ = _feature_cache_config(tmp_path) - save_manifest( - cache_paths(tmp_path / "gt_cache"), - {"artifacts": {"cp_features": {"path": "features/cp.zarr", "built_at": "2026-09-26T00:00:00+00:00"}}}, - ) - reference = json.loads((tmp_path / "cp_reference.json").read_text()) - # Fit provenance is outside the content hash, so it can be edited without re-hashing. - reference["fit"]["datasets"][CP_TEST_DATASET].update( - gt_cache_dir=str(tmp_path / "gt_cache"), cp_cache_built_at="2026-09-01T00:00:00+00:00" - ) - (tmp_path / "cp_reference.json").write_text(json.dumps(reference)) + config, _ = _feature_cache_config(tmp_path) # records the cache's real built_at + paths = cache_paths(tmp_path / "gt_cache") + manifest = load_manifest(paths) + manifest["artifacts"]["cp_features"]["built_at"] = "2099-01-01T00:00:00+00:00" # re-cached after the build + save_manifest(paths, manifest) monkeypatch.setattr(pipeline, "load_eval_models", lambda *a, **k: pytest.fail("models loaded")) cp_space = pipeline.eval_cp_space(config) - with pytest.raises(Exception, match="was built at 2026-09-26") as err: + with pytest.raises(Exception, match="was built at 2099-01-01") as err: pipeline.evaluate_predictions(config, cp_space=cp_space) assert type(err.value).__name__ == "StaleCacheError" From 659fa00c00db1680373a94872fa55b2e99a69a78 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 19:01:37 -0700 Subject: [PATCH 11/41] feat(dynacell/tools): verify a CP reference's GT-matrix hashes against the caches The docstrings claimed the per-dataset gt_matrix_sha256 made a GT re-cache "detectable against the artifact", but nothing ever recomputed it. build_cp_reference.py --verify now re-reads each non-lite dataset's GT CP cells, compares sha256 and cell count with the reference's fit section, and exits 1 on a mismatch. check_gt_cache now documents its limit. built_at is a best-effort re-cache signal: _merge_manifests takes it from the last flusher, and cache merging is unchanged here. check_n_cells catches count changes; a value-only change is caught only by --verify. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/cp_reference.py | 38 ++++++++++-- .../dynacell/tools/build_cp_reference.py | 60 +++++++++++++++++-- .../dynacell/tools/build_cp_reference_test.py | 18 ++++++ 3 files changed, 106 insertions(+), 10 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index 724921ac7..eee677764 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -25,9 +25,12 @@ Each scaler is fit on exactly the GT cells an eval of that dataset scores: every position and timepoint of the GT store, read from the GT CP cache, with the non-finite rows dropped. The reference records each dataset's position list, cell -count, GT-matrix hash and GT-cache ``cp_features.built_at``, and an eval refuses -to score if its GT side differs (:meth:`DatasetCPSpace.check_gt_cache`, -:meth:`DatasetCPSpace.check_positions`, :meth:`DatasetCPSpace.check_n_cells`). +count and GT-cache ``cp_features.built_at``, and an eval refuses to score if its +positions or cell count differ, or the cache stamp moved +(:meth:`DatasetCPSpace.check_positions`, :meth:`DatasetCPSpace.check_n_cells`, +:meth:`DatasetCPSpace.check_gt_cache`). The per-dataset GT-matrix sha256 is +recorded for audit and checked by ``build_cp_reference.py --verify``, not at eval +time. A per-dataset std can collapse on a feature the pooled mask keeps (a feature that is near-constant within one test set). Such a std is floored at @@ -156,10 +159,23 @@ def payload_sha256(payload: dict[str, Any]) -> str: def gt_matrix_sha256(gt: np.ndarray) -> str: - """Return a sha256 over a stacked GT cell matrix (shape + float64 C-order bytes). + """Return a sha256 over a stacked GT cell matrix. - Recorded per dataset so a later GT re-cache that changes any fitted value, or - the cell count, is detectable against the artifact. + The digest covers the shape and the float64 C-order bytes, so it changes with + any cell value or the cell count. It is recorded per dataset in the reference's + ``fit`` section for audit; the eval does not recompute it (that would mean + re-reading every GT cell). ``build_cp_reference.py --verify`` recomputes it from + the current caches and fails on a mismatch. + + Parameters + ---------- + gt : np.ndarray + ``(n_cells, n_features)`` GT CP matrix. + + Returns + ------- + str + Hex sha256 digest. """ arr = np.ascontiguousarray(gt, dtype=np.float64) digest = hashlib.sha256(f"{arr.shape[0]}x{arr.shape[1]}:".encode()) @@ -401,6 +417,16 @@ def check_gt_cache(self, gt_cache_dir: str | None) -> None: (its scaler is the parent's, so a lite cache is otherwise not what the scaler describes); when nothing is recorded the check is skipped. + This is a best-effort re-cache signal, not a content check. ``built_at`` + moves whenever any slot of the cache's CP artifact is rewritten, and the + cache-manifest merge (``pipeline_cache._merge_manifests``) takes + ``built_at`` from whichever writer flushes last, so it can also move + without a value change (a false alarm) or be restored by a concurrent + flush (a miss). :meth:`check_n_cells` catches a change in the GT cell + count; a value-only change with the same count is caught by neither, only + by ``build_cp_reference.py --verify``, which recomputes each dataset's + GT-matrix sha256 from the caches. + Parameters ---------- gt_cache_dir : str or None diff --git a/applications/dynacell/tools/build_cp_reference.py b/applications/dynacell/tools/build_cp_reference.py index b21ce05da..832f39064 100644 --- a/applications/dynacell/tools/build_cp_reference.py +++ b/applications/dynacell/tools/build_cp_reference.py @@ -22,11 +22,18 @@ The write is atomic, and an existing reference with a different content hash is only replaced under ``--force`` (an identical one is left untouched). +``--verify`` re-reads each non-lite dataset's GT CP cells from the current caches +and compares their sha256 and count with the ones recorded in the reference's +``fit`` section; it exits 1 on any mismatch. That is the only check that catches a +value-only GT re-cache (same cell count), which the eval's ``built_at`` and +cell-count checks cannot see. + Run:: uv run --no-sync python applications/dynacell/tools/build_cp_reference.py --target er --dry-run uv run --no-sync python applications/dynacell/tools/build_cp_reference.py --target er # registry path uv run --no-sync python applications/dynacell/tools/build_cp_reference.py --target er --out /tmp/er.json + uv run --no-sync python applications/dynacell/tools/build_cp_reference.py --target er --verify """ from __future__ import annotations @@ -58,6 +65,8 @@ DatasetFit, cp_space, fit_cp_reference, + gt_matrix_sha256, + load_cp_reference, write_cp_reference, ) from dynacell.evaluation.metrics import drop_paired_nonfinite_rows @@ -226,6 +235,37 @@ def build(target_name: str, leaves_root: Path) -> dict[str, Any]: return fit_cp_reference(fits, target_name=target_name, feature_names=names, cp_identity=identity, lite=lite) +def verify(target_name: str, path: Path) -> list[str]: + """Recompute every non-lite dataset's GT-matrix sha256 from the caches and compare. + + Parameters + ---------- + target_name : str + Eval target of the reference. + path : pathlib.Path + Reference JSON to verify. + + Returns + ------- + list of str + One message per dataset whose recomputed sha256 or cell count differs from + the recorded one; empty when all match. + """ + ref = load_cp_reference(path, target_name=target_name) + mismatches = [] + for name, record in sorted(ref.fit["datasets"].items()): + fit = read_dataset_fit(target_name, (name, record["target"]), len(ref.feature_names), record["in_mask_fit"]) + sha256 = gt_matrix_sha256(fit.cells) + ok = sha256 == record["gt_matrix_sha256"] and fit.cells.shape[0] == record["n_cells"] + print(f" {name}: {'OK' if ok else 'MISMATCH'} ({fit.cells.shape[0]} cells, sha256 {sha256[:12]})") + if not ok: + mismatches.append( + f"{name}: recorded {record['n_cells']} cells sha256 {record['gt_matrix_sha256'][:12]}, " + f"caches now give {fit.cells.shape[0]} cells sha256 {sha256[:12]}" + ) + return mismatches + + def _summary(payload: dict[str, Any]) -> str: """Human-readable per-dataset summary of a payload.""" lines = [ @@ -246,28 +286,40 @@ def _summary(payload: dict[str, Any]) -> str: return "\n".join(lines) -def main(argv: list[str] | None = None) -> None: - """CLI entry point.""" +def main(argv: list[str] | None = None) -> int: + """CLI entry point; returns the process exit code (1 when ``--verify`` finds a mismatch).""" parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--target", required=True, choices=sorted(MASK_FIT_DATASETS)) parser.add_argument("--leaves-root", type=Path, default=_LEAVES_ROOT, help="grouped eval leaves to scan") parser.add_argument("--out", type=Path, default=None, help="output JSON (default: the registry path)") parser.add_argument("--dry-run", action="store_true", help="read and fit; write nothing") parser.add_argument("--force", action="store_true", help="replace an existing reference with a different hash") + parser.add_argument( + "--verify", + action="store_true", + help="recompute each dataset's GT-matrix sha256 from the caches; exit 1 on mismatch", + ) args = parser.parse_args(argv) out = args.out if args.out is not None else cp_reference_path(args.target) + if args.verify: + mismatches = verify(args.target, out) + for line in mismatches: + print(f"MISMATCH {line}") + print(f"verify {out}: {'FAILED' if mismatches else 'OK'}") + return 1 if mismatches else 0 start = time.perf_counter() payload = build(args.target, args.leaves_root) print(_summary(payload)) elapsed = time.perf_counter() - start if args.dry_run: print(f"dry run: would write {out} sha256={payload['sha256']} ({elapsed:.1f} s)") - return + return 0 written = write_cp_reference(payload, out, force=args.force) status = "wrote" if written else "unchanged (identical reference)" print(f"{status} {out} sha256={payload['sha256']} ({elapsed:.1f} s)") + return 0 if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/applications/dynacell/tools/build_cp_reference_test.py b/applications/dynacell/tools/build_cp_reference_test.py index a6eeb54a7..0d0e42b24 100644 --- a/applications/dynacell/tools/build_cp_reference_test.py +++ b/applications/dynacell/tools/build_cp_reference_test.py @@ -241,3 +241,21 @@ def test_2d_store_is_refused(registry: Path) -> None: leaves = _leaves(registry, ["ds-a", "ds-b"]) with pytest.raises(ValueError, match="CP regionprops are 3-D"): main(["--target", "er", "--leaves-root", str(leaves), "--out", str(registry / "x.json")]) + + +def test_verify_passes_on_unchanged_caches_and_fails_on_a_value_only_recache(registry: Path, capsys) -> None: + """``--verify`` exits 0 on the caches the reference was built from, 1 after a same-count value change. + + The value-only re-cache keeps every cell count and the manifest's built_at, so the + eval's checks cannot see it; only the recomputed GT-matrix sha256 does. + """ + leaves = _build_all(registry) + out = registry / "er.json" + main(["--target", "er", "--leaves-root", str(leaves), "--out", str(out)]) + assert main(["--target", "er", "--out", str(out), "--verify"]) == 0 + + with open_features_group(cache_paths(registry / "caches" / "ds-h"), "cp", mode="a") as group: + write_features_to_group(group, "A/1/1", 1, np.full((5, _N_FEATURES), 2.0)) # same 5 cells, new values + assert main(["--target", "er", "--out", str(out), "--verify"]) == 1 + printed = capsys.readouterr().out + assert "MISMATCH ds-h" in printed and "verify" in printed and "FAILED" in printed From ca2de5592ecaf612a225714009b012a7346c7feb Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 19:03:13 -0700 Subject: [PATCH 12/41] refactor(dynacell/eval): resolve an eval dir's CP space from its sidecar in one place cross_condition_probe and lite_subset_sim each re-implemented "sidecar -> reference". They both read the reference file once just to learn its target_name and passed it back to load_cp_reference, which then checked the target against itself. cp_reference.sidecar_cp_space reads the reference path and sha256 from the sidecar, loads the reference, and verifies that sha256 still matches. It binds the space to the recorded dataset, so lite gets its parent's scaler. All three consumers use it, and the sidecar filename is a single constant shared with the pipeline. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/cp_reference.py | 92 ++++++++++++++----- .../dynacell/evaluation/cp_reference_test.py | 18 ++++ .../evaluation/cross_condition_probe.py | 30 ++---- .../src/dynacell/evaluation/pipeline.py | 4 +- applications/dynacell/tools/lite_checks.py | 4 +- .../dynacell/tools/lite_subset_sim.py | 25 +---- 6 files changed, 103 insertions(+), 70 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index eee677764..78b5a1c69 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -71,6 +71,9 @@ #: Schema of the reference JSON. Bump on any change to its keys or their meaning. CP_REFERENCE_SCHEMA = 3 +#: Sidecar each eval writes beside its metrics, naming the CP reference it scored in. +CP_SIDECAR_FILENAME = "cp_selected_feature_mask.json" + #: A per-dataset std below this fraction of the feature's pooled GT std is floored to it. STD_FLOOR_FRACTION = 1e-3 @@ -561,6 +564,32 @@ def for_dataset(self, dataset: str) -> DatasetCPSpace: ) +def _read_cp_reference(path: Path, *, build_hint: str) -> CPReference: + """Load a CP reference and verify its content hash and schema (no target check).""" + if not path.is_file(): + raise FileNotFoundError( + f"CP reference {path} not found. CP (GLCM+) feature metrics are scored in a per-target " + f"GT feature space; build it with:\n {build_hint}\n" + "(or point feature_metrics.cp.reference_path at an existing reference)." + ) + payload = json.loads(path.read_text()) + if payload["sha256"] != payload_sha256(payload): + raise ValueError(f"CP reference {path} content does not match its recorded sha256 (edited by hand?)") + if payload["schema"] != CP_REFERENCE_SCHEMA: + raise ValueError(f"CP reference {path} has schema {payload['schema']}, expected {CP_REFERENCE_SCHEMA}") + return CPReference( + path=str(path), + sha256=payload["sha256"], + target_name=payload["target_name"], + feature_names=tuple(payload["feature_names"]), + keep_mask=np.asarray(payload["keep_mask"], dtype=bool), + cp_identity=payload["cp_identity"], + scalers=payload["scalers"], + lite=payload["lite"], + fit=payload["fit"], + ) + + def load_cp_reference(path: Path, *, target_name: str) -> CPReference: """Load and hash-verify a CP reference. @@ -584,30 +613,47 @@ def load_cp_reference(path: Path, *, target_name: str) -> CPReference: If the stored hash disagrees with the content, or the reference was fit for another target or schema. """ - if not path.is_file(): - raise FileNotFoundError( - f"CP reference {path} not found. CP (GLCM+) feature metrics are scored in a per-target " - f"GT feature space; build it with:\n {_build_command(target_name)}\n" - "(or point feature_metrics.cp.reference_path at an existing reference)." + ref = _read_cp_reference(path, build_hint=_build_command(target_name)) + if ref.target_name != target_name: + raise ValueError(f"CP reference {path} was fit for {ref.target_name!r}, this eval is {target_name!r}") + return ref + + +def sidecar_cp_space(eval_dir: Path) -> DatasetCPSpace: + """Return the CP space a finished eval dir was scored in, from its :data:`CP_SIDECAR_FILENAME`. + + The sidecar records the reference path, its sha256 and the eval's dataset. The + reference is loaded from that path and must still carry that sha256; it is then + bound to the recorded dataset, so a lite eval gets its parent's scaler exactly as + the pipeline did. Used by post-hoc consumers of eval dirs (the cross-condition + probe, the lite subset tools) so they score CP the way the eval did. + + Parameters + ---------- + eval_dir : pathlib.Path + Eval save dir holding the sidecar. + + Returns + ------- + DatasetCPSpace + The reference bound to the eval's dataset. + + Raises + ------ + FileNotFoundError + If the sidecar or the reference it names is missing. + ValueError + If the reference on disk no longer has the sha256 the eval recorded. + """ + sidecar = json.loads((eval_dir / CP_SIDECAR_FILENAME).read_text()) + path = Path(sidecar["reference_path"]) + ref = _read_cp_reference(path, build_hint="the reference this eval was scored in") + if ref.sha256 != sidecar["reference_sha256"]: + raise ValueError( + f"{eval_dir}: CP reference {path} changed since the eval " + f"(sha256 {ref.sha256[:12]} now, {sidecar['reference_sha256'][:12]} at eval time)" ) - payload = json.loads(path.read_text()) - if payload["sha256"] != payload_sha256(payload): - raise ValueError(f"CP reference {path} content does not match its recorded sha256 (edited by hand?)") - if payload["schema"] != CP_REFERENCE_SCHEMA: - raise ValueError(f"CP reference {path} has schema {payload['schema']}, expected {CP_REFERENCE_SCHEMA}") - if payload["target_name"] != target_name: - raise ValueError(f"CP reference {path} was fit for {payload['target_name']!r}, this eval is {target_name!r}") - return CPReference( - path=str(path), - sha256=payload["sha256"], - target_name=payload["target_name"], - feature_names=tuple(payload["feature_names"]), - keep_mask=np.asarray(payload["keep_mask"], dtype=bool), - cp_identity=payload["cp_identity"], - scalers=payload["scalers"], - lite=payload["lite"], - fit=payload["fit"], - ) + return ref.for_dataset(sidecar["dataset"]) def resolve_cp_reference_path(config: DictConfig) -> Path: diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py index 84af50ec7..d55f31ff4 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -18,6 +18,7 @@ load_cp_reference, payload_sha256, resolve_cp_reference_path, + sidecar_cp_space, write_cp_reference, ) from dynacell.evaluation.feature_metrics import compute_feature_similarity_pairwise @@ -391,3 +392,20 @@ def test_over_smoothing_registers_in_the_shared_space(two_sets) -> None: assert new["CP_Median_Cosine_Similarity"] < 0.9 assert new["CP_KID"] > 1.0 assert new["CP_KID"] > 20 * abs(floor["CP_KID"]) + + +def test_sidecar_binds_the_recorded_reference_and_dataset(two_sets, tmp_path: Path) -> None: + """An eval dir's sidecar resolves to its reference + dataset; a changed reference is refused.""" + ref, _, cells = two_sets + eval_dir = tmp_path / "eval" + eval_dir.mkdir() + _stage_cp_dataset_inputs( + _lists(cells["set-a"][:40], cells["set-a"][:40]), ref.for_dataset("set-a-lite"), 40, eval_dir + ) + space = sidecar_cp_space(eval_dir) + assert (space.dataset, space.scaler_dataset, space.reference_sha256) == ("set-a-lite", "set-a", ref.sha256) + np.testing.assert_array_equal(space.mean, ref.for_dataset("set-a").mean) + + _reference(tmp_path, [_fit("set-a", _gt(seed=8))], lite={"set-a-lite": {"parent": "set-a"}}) # rebuilt in place + with pytest.raises(ValueError, match="changed since the eval"): + sidecar_cp_space(eval_dir) diff --git a/applications/dynacell/src/dynacell/evaluation/cross_condition_probe.py b/applications/dynacell/src/dynacell/evaluation/cross_condition_probe.py index 42c081881..462e05b32 100644 --- a/applications/dynacell/src/dynacell/evaluation/cross_condition_probe.py +++ b/applications/dynacell/src/dynacell/evaluation/cross_condition_probe.py @@ -26,7 +26,6 @@ import argparse import csv -import json import sys from pathlib import Path from typing import get_args @@ -34,7 +33,7 @@ import numpy as np from dynacell.evaluation.cache import FeatureKind -from dynacell.evaluation.cp_reference import load_cp_reference +from dynacell.evaluation.cp_reference import sidecar_cp_space from dynacell.evaluation.linear_probe import MADScaler, paired_auroc _FEATURE_TYPES: tuple[str, ...] = get_args(FeatureKind) @@ -118,36 +117,27 @@ def _load_embeddings( return result -#: Sidecar the eval writes beside its CP embeddings, naming the CP reference it scored in. -_CP_SIDECAR = "cp_selected_feature_mask.json" - - def _cp_reference_mask(eval_dirs: list[Path]) -> np.ndarray: """Return the CP reference keep-mask shared by ``eval_dirs``. - Read from each eval dir's ``cp_selected_feature_mask.json``, which names the - target's CP reference (path + hash). The mask is a property of the target, not - of the model or condition, so the probe compares conditions on the same feature - subset every eval scored. + Each eval dir's sidecar names the target's CP reference (path + hash); see + :func:`~dynacell.evaluation.cp_reference.sidecar_cp_space`. The mask is a + property of the target, not of the model or condition, so the probe compares + conditions on the same feature subset every eval scored. Raises ------ FileNotFoundError If an eval dir has no sidecar. ValueError - If the dirs name different references (ambiguous), or the reference on - disk no longer has the recorded hash. + If the dirs name different references (ambiguous), or a reference on disk + no longer has the recorded hash. """ - sidecars = [json.loads((d / _CP_SIDECAR).read_text()) for d in eval_dirs] - refs = {(s["reference_path"], s["reference_sha256"]) for s in sidecars} + spaces = [sidecar_cp_space(d) for d in eval_dirs] + refs = {(s.reference_path, s.reference_sha256) for s in spaces} if len(refs) != 1: raise ValueError(f"eval dirs {[str(d) for d in eval_dirs]} were scored in different CP references: {refs}") - ((path, sha256),) = refs - ref_payload = json.loads(Path(path).read_text()) - ref = load_cp_reference(Path(path), target_name=ref_payload["target_name"]) - if ref.sha256 != sha256: - raise ValueError(f"CP reference {path} changed since these evals (sha256 {ref.sha256[:12]} vs {sha256[:12]})") - return ref.keep_mask + return spaces[0].keep_mask def _probe_pair( diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline.py b/applications/dynacell/src/dynacell/evaluation/pipeline.py index 7b682d651..562da40bd 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline.py @@ -18,7 +18,7 @@ from dynacell.evaluation._ref_hook import apply_dataset_ref from dynacell.evaluation.cache import FeatureKind -from dynacell.evaluation.cp_reference import DatasetCPSpace, eval_cp_space +from dynacell.evaluation.cp_reference import CP_SIDECAR_FILENAME, DatasetCPSpace, eval_cp_space from dynacell.evaluation.cross_condition_probe import run_for_group as _cross_condition_run_for_group from dynacell.evaluation.feature_metrics import ( compute_feature_similarity, @@ -367,7 +367,7 @@ def _stage_cp_dataset_inputs( "n_kept": int(cp_space.keep_mask.sum()), "n_total": int(cp_space.keep_mask.size), } - (save_dir / "cp_selected_feature_mask.json").write_text(json.dumps(mask_payload, indent=2)) + (save_dir / CP_SIDECAR_FILENAME).write_text(json.dumps(mask_payload, indent=2)) return ( "CP", cp_space.transform(pred_cp_raw), diff --git a/applications/dynacell/tools/lite_checks.py b/applications/dynacell/tools/lite_checks.py index 57764de16..a03cad7f4 100644 --- a/applications/dynacell/tools/lite_checks.py +++ b/applications/dynacell/tools/lite_checks.py @@ -32,7 +32,6 @@ from lite_subset_sim import ( EXTRACTORS, MODELS_3D, - cp_space_of, designs_for, load_systems, mmd2_from_features, @@ -41,6 +40,7 @@ subset_keys, ) +from dynacell.evaluation.cp_reference import sidecar_cp_space from dynacell.evaluation.paths import DATA_ROOT E_SYSTEMS = [ @@ -206,7 +206,7 @@ def check_e( keep = np.isfinite(X).all(1) & np.isfinite(Y).all(1) X, Y = X[keep], Y[keep] if tok == "cp": # the pipeline's CP space: reference mask + this dataset's GT scaler - space = cp_space_of(path) + space = sidecar_cp_space(path) X, Y = space.transform(X), space.transform(Y) n_all = X.shape[0] if tok != "cp": # deep extractors: precompute cell-level kernels once, slice per draw diff --git a/applications/dynacell/tools/lite_subset_sim.py b/applications/dynacell/tools/lite_subset_sim.py index da97436a8..cdfdc5ff8 100644 --- a/applications/dynacell/tools/lite_subset_sim.py +++ b/applications/dynacell/tools/lite_subset_sim.py @@ -30,7 +30,6 @@ import argparse import itertools -import json from collections import Counter from collections.abc import Callable, Iterator from math import comb, factorial @@ -41,7 +40,7 @@ from build_temporal_subset_zarr import spread_timepoints from scipy.stats import kendalltau, spearmanr -from dynacell.evaluation.cp_reference import DatasetCPSpace, load_cp_reference +from dynacell.evaluation.cp_reference import sidecar_cp_space from dynacell.evaluation.paths import DATA_ROOT EXTRACTORS = {"cp": "CP", "dinov3": "DINOv3", "dynaclr": "DynaCLR", "celldino": "CellDINO", "morphem": "MorphEm"} @@ -66,26 +65,6 @@ _EVAL_CSVS = ("pixel_metrics.csv", "mask_metrics.csv", "feature_metrics.csv") -def cp_space_of(eval_dir: Path) -> DatasetCPSpace: - """Return the CP space an eval dir was scored in, from its ``cp_selected_feature_mask.json``. - - The sidecar names the reference (path + hash) and the eval's dataset; the - reference is loaded, hash-checked and bound to that dataset, so a lite eval - gets its parent's scaler exactly as the pipeline did. - - Raises - ------ - ValueError - If the reference on disk no longer has the hash the eval recorded. - """ - sidecar = json.loads((eval_dir / "cp_selected_feature_mask.json").read_text()) - path = Path(sidecar["reference_path"]) - ref = load_cp_reference(path, target_name=json.loads(path.read_text())["target_name"]) - if ref.sha256 != sidecar["reference_sha256"]: - raise ValueError(f"{eval_dir}: CP reference {path} changed since the eval ({ref.sha256[:12]})") - return ref.for_dataset(sidecar["dataset"]) - - def poly_kernel(X: np.ndarray, Y: np.ndarray) -> np.ndarray: """Return torch-fidelity's KID kernel ``(x.y / d + 1)^3`` between the rows of X and Y. @@ -215,7 +194,7 @@ def __init__(self, path: Path, blocks: list[tuple[str, int]]): X, Y, cell_block = X[ok], Y[ok], cell_block[ok] if tok == "cp": # pipeline: the CP reference mask + this dataset's GT scaler, on both sides. - space = cp_space_of(path) + space = sidecar_cp_space(path) self.cp = dict(X=space.transform(X), Y=space.transform(Y), cell_block=cell_block) continue # one-hot block membership (n x nb) -> block sums via M^T K M From 3cbd4f1c4afcd30e7e48a49be2c02f92ec65a3a6 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 19:03:35 -0700 Subject: [PATCH 13/41] docs(dynacell/eval): point the CP criteria comments at the reference feature_select's constants comment and active_cp_feature_names still described the per-eval cp_selected_feature_mask sidecar as where the criteria and column schema must not drift. Both now live in the CP reference (the criteria block and the recipe identity). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../dynacell/src/dynacell/evaluation/feature_select.py | 10 +++++----- .../dynacell/src/dynacell/evaluation/metrics.py | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/feature_select.py b/applications/dynacell/src/dynacell/evaluation/feature_select.py index 1004b508e..eb1d1fd3a 100644 --- a/applications/dynacell/src/dynacell/evaluation/feature_select.py +++ b/applications/dynacell/src/dynacell/evaluation/feature_select.py @@ -15,11 +15,11 @@ import numpy as np -# Defaults shared with downstream consumers (e.g. the cp_selected_feature_mask -# JSON sidecar emitted by the evaluation pipeline, and the CP reference). Keep -# `select_gt_features`, `variance_threshold`, and `correlation_threshold` -# keyword defaults aligned -# with these constants so the sidecar cannot drift from the actual call. +# Selection criteria of the per-target CP reference. `fit_cp_reference` passes +# these explicitly and records them in the reference's `criteria` block (inside +# its content hash), so the reference is where the applied values live. Keep the +# `select_gt_features`, `variance_threshold` and `correlation_threshold` keyword +# defaults aligned with them. DEFAULT_FREQ_CUT = 0.05 DEFAULT_UNIQUE_CUT = 0.01 DEFAULT_CORR_THRESHOLD = 0.9 diff --git a/applications/dynacell/src/dynacell/evaluation/metrics.py b/applications/dynacell/src/dynacell/evaluation/metrics.py index 80319f13c..440b446b4 100644 --- a/applications/dynacell/src/dynacell/evaluation/metrics.py +++ b/applications/dynacell/src/dynacell/evaluation/metrics.py @@ -458,8 +458,9 @@ def active_cp_feature_names(glcm_enabled: bool) -> tuple[str, ...]: The schema is GLCM-dependent: the base distribution/texture columns are always emitted; the seven ``glcm_*`` columns are appended only when GLCM is - enabled. Used by both the matrix assembly and the - ``cp_selected_feature_mask.json`` sidecar so they never drift. + enabled. Used by both the matrix assembly and the CP reference's recipe + identity (``cp_reference.cp_space``), so a reference built for another + column set is refused instead of silently misaligned. """ if glcm_enabled: return _CP_BASE_FEATURE_NAMES + _CP_GLCM_FEATURE_NAMES From add4af7a167a7f12fe5bbc8bef6a5735680441cc Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 19:03:36 -0700 Subject: [PATCH 14/41] docs(dynacell/eval): numpy docstrings for check_n_cells and save_metrics Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/cp_reference.py | 7 ++++++ .../src/dynacell/evaluation/pipeline.py | 24 +++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index 78b5a1c69..9779fb6ed 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -500,6 +500,13 @@ def check_n_cells(self, n_gt_cells: int) -> None: Lite datasets are exempt: they score a subset of the parent's cells, and :meth:`CPReference.for_dataset` already required the parent to be recorded. + A value-only GT change with the same count passes; see :meth:`check_gt_cache`. + + Parameters + ---------- + n_gt_cells : int + Finite GT CP rows the eval scored, summed over every position and + timepoint, before the pairwise pred/GT non-finite drop. Raises ------ diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline.py b/applications/dynacell/src/dynacell/evaluation/pipeline.py index 562da40bd..ea6914b67 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline.py @@ -1833,12 +1833,26 @@ def save_metrics( *, cp_reference_sha256: str | None, ): - """Save metrics to files. + """Save metric rows as CSV + NPY (and plots), then stamp ``metrics_provenance.json``. - ``cp_reference_sha256`` is the hash of the CP reference the run SCORED with - (``DatasetCPSpace.reference_sha256``; ``None`` without feature metrics). It is - passed in rather than re-read here, so a reference rebuilt mid-run cannot be - stamped on values it did not produce. + Parameters + ---------- + config : DictConfig + Eval config (``save.*`` filenames and ``save.save_dir``, + ``compute_feature_metrics``). + pixel_metrics, mask_metrics, feature_metrics : list of dict, optional + Per-(FOV, timepoint) rows; an empty or ``None`` family is skipped. + cp_reference_sha256 : str or None + Hash of the CP reference the run SCORED with + (``DatasetCPSpace.reference_sha256``), ``None`` without feature metrics. + Passed in rather than re-read here, so a reference rebuilt mid-run cannot + be stamped on values it did not produce. + + Raises + ------ + ValueError + If ``cp_reference_sha256`` is given without feature metrics, or missing + with them. """ if config.compute_feature_metrics != (cp_reference_sha256 is not None): raise ValueError( From cb52c00ae998eb2f07816771a24e9644b0d0f8f3 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 19:22:24 -0700 Subject: [PATCH 15/41] fix(dynacell/eval): refuse final-metrics cache reuse after a GT re-cache The cache-reuse path compared only the CP reference hash, so a GT CP cache re-cached after the reference was built still let the next eval reuse CP rows scored against the old GT. Run the same check_gt_cache guard the scoring path runs before accepting the cache. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../dynacell/src/dynacell/evaluation/pipeline.py | 8 +++++++- .../tests/test_evaluation_timepoint_guard.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline.py b/applications/dynacell/src/dynacell/evaluation/pipeline.py index ea6914b67..c5d1499e3 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline.py @@ -1912,7 +1912,13 @@ def _final_metrics_cache_valid(config: DictConfig) -> bool: # An unstamped dir has nothing to reuse, so it needs no reference to be rejected. if not (save_dir / PROVENANCE_FILENAME).is_file(): return False - current_sha256 = eval_cp_space(config).reference_sha256 if config.compute_feature_metrics else None + current_sha256 = None + if config.compute_feature_metrics: + space = eval_cp_space(config) + # Same GT re-cache guard as the scoring path: cached CP rows scored against a + # GT cache that has since been re-cached must not be reused silently. + space.check_gt_cache(OmegaConf.select(config, "io.gt_cache_dir", default=None)) + current_sha256 = space.reference_sha256 if not metrics_provenance_matches(save_dir, cp_reference_sha256=current_sha256): return False pixel_ok = (save_dir / config.save.pixel_metrics_filename).exists() diff --git a/applications/dynacell/tests/test_evaluation_timepoint_guard.py b/applications/dynacell/tests/test_evaluation_timepoint_guard.py index c4ec3b29a..17441c795 100644 --- a/applications/dynacell/tests/test_evaluation_timepoint_guard.py +++ b/applications/dynacell/tests/test_evaluation_timepoint_guard.py @@ -174,6 +174,22 @@ def test_cache_scored_in_another_cp_reference_is_invalid(tmp_path: Path) -> None assert not pipeline._final_metrics_cache_valid(config) +def test_cache_reuse_refuses_a_gt_recache(tmp_path: Path) -> None: + """A reusable final-metrics cache is refused once the GT CP cache was re-cached after the build.""" + pipeline = live_pipeline_module() + config, sha256 = _feature_cache_config(tmp_path) # records the cache's real built_at + _write_final_caches(tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES), sha256) + assert pipeline._final_metrics_cache_valid(config) + + paths = cache_paths(tmp_path / "gt_cache") + manifest = load_manifest(paths) + manifest["artifacts"]["cp_features"]["built_at"] = "2099-01-01T00:00:00+00:00" # re-cached after the build + save_manifest(paths, manifest) + with pytest.raises(Exception, match="was built at 2099-01-01") as err: + pipeline._final_metrics_cache_valid(config) + assert type(err.value).__name__ == "StaleCacheError" + + def test_feature_less_cache_ignores_the_cp_reference(tmp_path: Path) -> None: """A compute_feature_metrics=false dir with no CP hash in its stamp stays reusable, with no reference.""" pipeline = live_pipeline_module() From babe080f6cfbb68237d4a0e3cc8f3057cffb81b5 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 19:47:13 -0700 Subject: [PATCH 16/41] fix(dynacell/eval): bind the final-metrics stamp to the dataset and its GT cells The stamp held only the whole-reference hash, which deliberately leaves fit provenance unhashed. Two stale caches could still be reused (both from Copilot review on #499). A GT re-cache that changed the cell matrix while mean/std/mask stayed the same reused CP rows scored on the old cells. A save_dir scored for dataset A was also accepted for dataset B of the same reference. One mechanism fixes both. DatasetCPSpace.binding_sha256 hashes the reference sha256, dataset, scaler dataset, lite flag and the GT-matrix sha256 recorded in the fit section. That is the dataset's own for a non-lite set; a lite set uses its own if recorded, else the parent's plus its recorded cache built_at. metrics_provenance.json stamps it as cp_space_sha256 and keeps cp_reference_sha256 for audit. metrics_provenance_matches compares cp_space_sha256 when feature metrics are on. save_metrics takes the scoring DatasetCPSpace itself, so both hashes come from the space that scored the run. A rebuild that moves only built_at keeps the binding, so harmless re-caches still invalidate nothing. Existing registry references need no rebuild: their fit section already records every GT-matrix sha256. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/cp_reference.py | 37 +++++++++ .../dynacell/evaluation/cp_reference_test.py | 25 ++++++ .../evaluation/instance_metrics_test.py | 2 +- .../src/dynacell/evaluation/pipeline.py | 38 +++++---- .../src/dynacell/evaluation/provenance.py | 43 ++++++---- .../dynacell/evaluation/provenance_test.py | 43 ++++++---- .../test_evaluation_cp_scaler_selection.py | 6 +- .../tests/test_evaluation_pipeline.py | 2 +- .../tests/test_evaluation_timepoint_guard.py | 78 +++++++++++++++---- .../dynacell/tests/test_pipeline_cache.py | 4 +- 10 files changed, 212 insertions(+), 66 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index 9779fb6ed..35ae26ece 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -390,6 +390,40 @@ class DatasetCPSpace: fit_n_cells: int gt_cache_dir: str | None cp_cache_built_at: str | None + fit_gt_matrix_sha256: str + own_gt_matrix_sha256: str | None + + @property + def binding_sha256(self) -> str: + """Return a sha256 identifying this eval's CP space AND the GT cells it describes. + + The final-metrics stamp needs more than :attr:`reference_sha256`, which + deliberately leaves fit provenance unhashed: a cache scored for dataset A + must not be reused for dataset B, and a GT re-cache that changes the cells + while leaving mean/std/mask unchanged must not reuse CP rows scored on the + old cells. This hash covers the reference hash, the dataset, its scaler + dataset, the lite flag, and the recorded GT-matrix sha256. That is the + dataset's own for a non-lite set. A lite set uses its own when one is + recorded, else the parent's plus the lite cache's recorded ``built_at``. + A harmless rebuild that only moves ``built_at`` on a non-lite set keeps it. + + Returns + ------- + str + Hex sha256 over the canonical JSON of those fields. + """ + if self.is_lite and self.own_gt_matrix_sha256 is None: + gt = {"parent_gt_matrix_sha256": self.fit_gt_matrix_sha256, "cp_cache_built_at": self.cp_cache_built_at} + else: + gt = {"gt_matrix_sha256": self.own_gt_matrix_sha256 or self.fit_gt_matrix_sha256} + body = { + "reference_sha256": self.reference_sha256, + "dataset": self.dataset, + "scaler_dataset": self.scaler_dataset, + "is_lite": self.is_lite, + **gt, + } + return hashlib.sha256(json.dumps(body, sort_keys=True, separators=(",", ":")).encode()).hexdigest() def _check_width(self, x: np.ndarray) -> None: """Raise when a CP matrix does not have the reference's column count.""" @@ -568,6 +602,9 @@ def for_dataset(self, dataset: str) -> DatasetCPSpace: fit_n_cells=int(scaler_fit["n_cells"]), gt_cache_dir=own["gt_cache_dir"], cp_cache_built_at=own["cp_cache_built_at"], + fit_gt_matrix_sha256=scaler_fit["gt_matrix_sha256"], + # A lite entry records no GT matrix today; the key is optional by design. + own_gt_matrix_sha256=own.get("gt_matrix_sha256") if is_lite else scaler_fit["gt_matrix_sha256"], ) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py index d55f31ff4..449ed610f 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -409,3 +409,28 @@ def test_sidecar_binds_the_recorded_reference_and_dataset(two_sets, tmp_path: Pa _reference(tmp_path, [_fit("set-a", _gt(seed=8))], lite={"set-a-lite": {"parent": "set-a"}}) # rebuilt in place with pytest.raises(ValueError, match="changed since the eval"): sidecar_cp_space(eval_dir) + + +def test_binding_separates_datasets_and_tracks_the_gt_matrix(tmp_path: Path) -> None: + """The binding differs per dataset (lite vs parent too) and moves with GT matrix or lite stamp, not built_at.""" + fits = [_fit("set-a", _gt(), built_at="t0"), _fit("set-b", _gt(seed=1), built_at="t0")] + lite = {"set-a-lite": {"parent": "set-a", "gt_cache_dir": "/lite", "cp_cache_built_at": "L0"}} + ref, payload = _reference(tmp_path, fits, lite=lite) + bindings = {d: ref.for_dataset(d).binding_sha256 for d in ("set-a", "set-b", "set-a-lite")} + assert len(set(bindings.values())) == 3 + + def rebound(edit) -> dict[str, str]: + moved = json.loads(json.dumps(payload)) + edit(moved) + (tmp_path / "er.json").write_text(json.dumps(moved)) + again = load_cp_reference(tmp_path / "er.json", target_name="er") + assert again.sha256 == ref.sha256 # provenance only: numeric hash unchanged + return {d: again.for_dataset(d).binding_sha256 for d in bindings} + + built_at = rebound(lambda p: p["fit"]["datasets"]["set-a"].update(cp_cache_built_at="t1")) + assert built_at == bindings + gt = rebound(lambda p: p["fit"]["datasets"]["set-a"].update(gt_matrix_sha256="0" * 64)) + assert gt["set-a"] != bindings["set-a"] and gt["set-a-lite"] != bindings["set-a-lite"] + assert gt["set-b"] == bindings["set-b"] + lite_stamp = rebound(lambda p: p["fit"]["lite"]["set-a-lite"].update(cp_cache_built_at="L1")) + assert lite_stamp["set-a-lite"] != bindings["set-a-lite"] and lite_stamp["set-a"] == bindings["set-a"] diff --git a/applications/dynacell/src/dynacell/evaluation/instance_metrics_test.py b/applications/dynacell/src/dynacell/evaluation/instance_metrics_test.py index 11e28f843..0082da991 100644 --- a/applications/dynacell/src/dynacell/evaluation/instance_metrics_test.py +++ b/applications/dynacell/src/dynacell/evaluation/instance_metrics_test.py @@ -148,7 +148,7 @@ def test_instance_ap_is_written_to_mask_metrics_csv(tmp_path): }, } ) - save_metrics(config, mask_metrics=[{"FOV": "0/0/0", "Timepoint": 0, **row}], cp_reference_sha256=None) + save_metrics(config, mask_metrics=[{"FOV": "0/0/0", "Timepoint": 0, **row}], cp_space=None) assert not list(tmp_path.glob("instance_ap*.csv")) written = tmp_path / "mask_metrics.csv" diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline.py b/applications/dynacell/src/dynacell/evaluation/pipeline.py index c5d1499e3..439fce8f3 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline.py @@ -1831,7 +1831,7 @@ def save_metrics( mask_metrics=None, feature_metrics=None, *, - cp_reference_sha256: str | None, + cp_space: DatasetCPSpace | None, ): """Save metric rows as CSV + NPY (and plots), then stamp ``metrics_provenance.json``. @@ -1842,21 +1842,20 @@ def save_metrics( ``compute_feature_metrics``). pixel_metrics, mask_metrics, feature_metrics : list of dict, optional Per-(FOV, timepoint) rows; an empty or ``None`` family is skipped. - cp_reference_sha256 : str or None - Hash of the CP reference the run SCORED with - (``DatasetCPSpace.reference_sha256``), ``None`` without feature metrics. - Passed in rather than re-read here, so a reference rebuilt mid-run cannot - be stamped on values it did not produce. + cp_space : DatasetCPSpace or None + The bound CP space the run SCORED with, ``None`` without feature metrics. + Its ``reference_sha256`` (audit) and ``binding_sha256`` (compared on cache + reuse) are stamped. It is passed in rather than re-read here, so a reference + rebuilt mid-run cannot be stamped on values it did not produce. Raises ------ ValueError - If ``cp_reference_sha256`` is given without feature metrics, or missing - with them. + If ``cp_space`` is given without feature metrics, or missing with them. """ - if config.compute_feature_metrics != (cp_reference_sha256 is not None): + if config.compute_feature_metrics != (cp_space is not None): raise ValueError( - "cp_reference_sha256 must be given exactly when compute_feature_metrics=true " + "cp_space must be given exactly when compute_feature_metrics=true " f"(compute_feature_metrics={config.compute_feature_metrics})" ) save_dir = Path(config.save.save_dir) @@ -1881,7 +1880,11 @@ def save_metrics( # whether the cache is comparable instead of assuming it is. Stamping first meant a # crash partway through the loop above left a fresh stamp certifying a previous # run's rows -- the exact misattribution the sidecar exists to make detectable. - write_metrics_provenance(save_dir, cp_reference_sha256=cp_reference_sha256) + write_metrics_provenance( + save_dir, + cp_reference_sha256=cp_space.reference_sha256 if cp_space is not None else None, + cp_space_sha256=cp_space.binding_sha256 if cp_space is not None else None, + ) #: Pixel columns that only a dual-scaling run writes. Their absence marks a pixel @@ -1907,8 +1910,9 @@ def _final_metrics_cache_valid(config: DictConfig) -> bool: # Metric values are not comparable across cubic versions — FSC/FRC/Spectral_PCC # move (see dynacell.evaluation.provenance). A cache with no stamp, or one # stamped with a different cubic, is not reusable regardless of its columns. - # Likewise for the CP reference: CP KID/FID/cosine were scored in the reference - # stamped beside them, so a rebuilt (or never-stamped) reference forces a recompute. + # Likewise for CP: KID/FID/cosine were scored in the CP space stamped beside them + # (reference + dataset + the GT cells it was fit on), so another dataset's space, a + # changed GT matrix, or a rebuilt (or never-stamped) reference forces a recompute. # An unstamped dir has nothing to reuse, so it needs no reference to be rejected. if not (save_dir / PROVENANCE_FILENAME).is_file(): return False @@ -1918,8 +1922,8 @@ def _final_metrics_cache_valid(config: DictConfig) -> bool: # Same GT re-cache guard as the scoring path: cached CP rows scored against a # GT cache that has since been re-cached must not be reused silently. space.check_gt_cache(OmegaConf.select(config, "io.gt_cache_dir", default=None)) - current_sha256 = space.reference_sha256 - if not metrics_provenance_matches(save_dir, cp_reference_sha256=current_sha256): + current_sha256 = space.binding_sha256 + if not metrics_provenance_matches(save_dir, cp_space_sha256=current_sha256): return False pixel_ok = (save_dir / config.save.pixel_metrics_filename).exists() mask_path = save_dir / config.save.mask_metrics_filename @@ -2239,7 +2243,7 @@ def get_models() -> EvalModels: pixel_metrics=pixel_metrics, mask_metrics=mask_metrics, feature_metrics=feature_metrics, - cp_reference_sha256=cp_space.reference_sha256 if cp_space is not None else None, + cp_space=cp_space, ) results.append((name, (pixel_metrics, mask_metrics, feature_metrics))) condition_save_dirs.append(Path(merged.save.save_dir)) @@ -2289,7 +2293,7 @@ def evaluate_model(config: DictConfig): pixel_metrics=pixel_metrics, mask_metrics=mask_metrics, feature_metrics=feature_metrics, - cp_reference_sha256=cp_space.reference_sha256 if cp_space is not None else None, + cp_space=cp_space, ) # Re-dump so save_metrics_csvs lands in eval_timing.csv. evaluate_predictions # dumps once before save_metrics runs; this second dump overwrites with the diff --git a/applications/dynacell/src/dynacell/evaluation/provenance.py b/applications/dynacell/src/dynacell/evaluation/provenance.py index 4cd03c539..3710d141f 100644 --- a/applications/dynacell/src/dynacell/evaluation/provenance.py +++ b/applications/dynacell/src/dynacell/evaluation/provenance.py @@ -89,7 +89,7 @@ def check_cubic_pin() -> None: ) -def write_metrics_provenance(save_dir: Path, *, cp_reference_sha256: str | None) -> None: +def write_metrics_provenance(save_dir: Path, *, cp_reference_sha256: str | None, cp_space_sha256: str | None) -> None: """Write the numeric-provenance sidecar into ``save_dir``. Parameters @@ -97,14 +97,29 @@ def write_metrics_provenance(save_dir: Path, *, cp_reference_sha256: str | None) save_dir : pathlib.Path Directory that receives the metric CSV/NPY files. cp_reference_sha256 : str or None - Content hash of the CP reference the feature metrics were scored in; - ``None`` when the run computed no feature metrics. + Content hash of the CP reference the feature metrics were scored in + (recorded for audit); ``None`` when the run computed no feature metrics. + cp_space_sha256 : str or None + ``DatasetCPSpace.binding_sha256`` of the space the run scored in: the + reference bound to one dataset and the GT cells it was fit on. This is the + value cache reuse compares. ``None`` exactly when ``cp_reference_sha256`` is. + + Raises + ------ + ValueError + If only one of the two hashes is given. """ - payload = {"versions": installed_versions(), "cp_reference_sha256": cp_reference_sha256} + if (cp_reference_sha256 is None) != (cp_space_sha256 is None): + raise ValueError("cp_reference_sha256 and cp_space_sha256 must both be given or both be None") + payload = { + "versions": installed_versions(), + "cp_reference_sha256": cp_reference_sha256, + "cp_space_sha256": cp_space_sha256, + } (save_dir / PROVENANCE_FILENAME).write_text(json.dumps(payload, indent=1, sort_keys=True) + "\n") -def metrics_provenance_matches(save_dir: Path, *, cp_reference_sha256: str | None) -> bool: +def metrics_provenance_matches(save_dir: Path, *, cp_space_sha256: str | None) -> bool: """Return True when ``save_dir``'s metrics were built by the running ``cubic`` and CP reference. A missing sidecar returns False. Every cache written before this stamp @@ -117,18 +132,18 @@ def metrics_provenance_matches(save_dir: Path, *, cp_reference_sha256: str | Non ---------- save_dir : pathlib.Path Directory holding the metric CSV/NPY files. - cp_reference_sha256 : str or None - Content hash of the CP reference the current run would score in, or - ``None`` when it computes no feature metrics -- then no CP value is - reused and the recorded hash (or its absence) is irrelevant. When given, - a sidecar written before the CP reference existed carries no such key - and never matches. + cp_space_sha256 : str or None + ``DatasetCPSpace.binding_sha256`` of the space the current run would score + in, or ``None`` when it computes no feature metrics -- then no CP value is + reused and the recorded hash (or its absence) is irrelevant. When given, a + sidecar without a ``cp_space_sha256`` (written before the binding existed) + never matches. Returns ------- bool True when the recorded ``cubic`` version equals the installed one and, - if ``cp_reference_sha256`` is given, the recorded hash equals it. + if ``cp_space_sha256`` is given, the recorded binding equals it. """ path = save_dir / PROVENANCE_FILENAME if not path.is_file(): @@ -137,6 +152,6 @@ def metrics_provenance_matches(save_dir: Path, *, cp_reference_sha256: str | Non recorded = payload.get("versions", {}).get("cubic") if recorded != version("cubic"): return False - if cp_reference_sha256 is None: + if cp_space_sha256 is None: return True - return "cp_reference_sha256" in payload and payload["cp_reference_sha256"] == cp_reference_sha256 + return "cp_space_sha256" in payload and payload["cp_space_sha256"] == cp_space_sha256 diff --git a/applications/dynacell/src/dynacell/evaluation/provenance_test.py b/applications/dynacell/src/dynacell/evaluation/provenance_test.py index 229837832..7ae32db37 100644 --- a/applications/dynacell/src/dynacell/evaluation/provenance_test.py +++ b/applications/dynacell/src/dynacell/evaluation/provenance_test.py @@ -38,38 +38,39 @@ def test_installed_versions_records_cubic(): def test_roundtrip_matches_running_environment(tmp_path): - write_metrics_provenance(tmp_path, cp_reference_sha256="abc") + write_metrics_provenance(tmp_path, cp_reference_sha256="ref-abc", cp_space_sha256="abc") payload = json.loads((tmp_path / PROVENANCE_FILENAME).read_text()) assert payload["versions"]["cubic"] == version("cubic") - assert payload["cp_reference_sha256"] == "abc" - assert metrics_provenance_matches(tmp_path, cp_reference_sha256="abc") + assert payload["cp_reference_sha256"] == "ref-abc" + assert payload["cp_space_sha256"] == "abc" + assert metrics_provenance_matches(tmp_path, cp_space_sha256="abc") def test_feature_less_stamp_matches_only_a_feature_less_run(tmp_path): """A run without feature metrics stamps ``None``; a CP-scoring run must not reuse it.""" - write_metrics_provenance(tmp_path, cp_reference_sha256=None) - assert metrics_provenance_matches(tmp_path, cp_reference_sha256=None) - assert not metrics_provenance_matches(tmp_path, cp_reference_sha256="abc") + write_metrics_provenance(tmp_path, cp_reference_sha256=None, cp_space_sha256=None) + assert metrics_provenance_matches(tmp_path, cp_space_sha256=None) + assert not metrics_provenance_matches(tmp_path, cp_space_sha256="abc") def test_feature_less_run_ignores_the_cp_key(tmp_path): """compute_feature_metrics=false reuses no CP value, so a missing or foreign hash is irrelevant.""" (tmp_path / PROVENANCE_FILENAME).write_text(json.dumps({"versions": {"cubic": version("cubic")}})) - assert metrics_provenance_matches(tmp_path, cp_reference_sha256=None) - write_metrics_provenance(tmp_path, cp_reference_sha256="abc") - assert metrics_provenance_matches(tmp_path, cp_reference_sha256=None) + assert metrics_provenance_matches(tmp_path, cp_space_sha256=None) + write_metrics_provenance(tmp_path, cp_reference_sha256="ref-abc", cp_space_sha256="abc") + assert metrics_provenance_matches(tmp_path, cp_space_sha256=None) def test_other_cp_reference_is_not_a_match(tmp_path): """CP values scored in another reference are not reusable after a rebuild.""" - write_metrics_provenance(tmp_path, cp_reference_sha256="old") - assert not metrics_provenance_matches(tmp_path, cp_reference_sha256="new") + write_metrics_provenance(tmp_path, cp_reference_sha256="ref-old", cp_space_sha256="old") + assert not metrics_provenance_matches(tmp_path, cp_space_sha256="new") def test_stamp_predating_cp_reference_is_not_a_match(tmp_path): """A sidecar written before the CP reference existed carries no hash; a CP-scoring run refuses it.""" (tmp_path / PROVENANCE_FILENAME).write_text(json.dumps({"versions": {"cubic": version("cubic")}})) - assert not metrics_provenance_matches(tmp_path, cp_reference_sha256="abc") + assert not metrics_provenance_matches(tmp_path, cp_space_sha256="abc") def test_missing_sidecar_is_not_a_match(tmp_path): @@ -78,16 +79,16 @@ def test_missing_sidecar_is_not_a_match(tmp_path): Every cache written before the stamp existed is exactly the ambiguous 0.8.0a2-or-0.9.0a1 case that has to be recomputed. """ - assert not metrics_provenance_matches(tmp_path, cp_reference_sha256=None) + assert not metrics_provenance_matches(tmp_path, cp_space_sha256=None) def test_foreign_cubic_version_is_not_a_match(tmp_path): - write_metrics_provenance(tmp_path, cp_reference_sha256=None) + write_metrics_provenance(tmp_path, cp_reference_sha256=None, cp_space_sha256=None) path = tmp_path / PROVENANCE_FILENAME payload = json.loads(path.read_text()) payload["versions"]["cubic"] = "0.8.0a2" path.write_text(json.dumps(payload)) - assert not metrics_provenance_matches(tmp_path, cp_reference_sha256=None) + assert not metrics_provenance_matches(tmp_path, cp_space_sha256=None) def test_check_cubic_pin_accepts_the_declared_version(): @@ -101,3 +102,15 @@ def test_check_cubic_pin_rejects_a_mismatch(monkeypatch): monkeypatch.setattr("dynacell.evaluation.provenance.version", lambda _name: "0.8.0a2") with pytest.raises(RuntimeError, match="not comparable across"): check_cubic_pin() + + +def test_stamp_without_a_cp_space_binding_is_not_a_match(tmp_path): + """A stamp carrying only the whole-reference hash (no dataset binding) is refused by a CP-scoring run.""" + payload = {"versions": {"cubic": version("cubic")}, "cp_reference_sha256": "abc"} + (tmp_path / PROVENANCE_FILENAME).write_text(json.dumps(payload)) + assert not metrics_provenance_matches(tmp_path, cp_space_sha256="abc") + + +def test_half_a_cp_stamp_is_refused(tmp_path): + with pytest.raises(ValueError, match="both be given or both be None"): + write_metrics_provenance(tmp_path, cp_reference_sha256="abc", cp_space_sha256=None) diff --git a/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py b/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py index c52046866..5e3e003cf 100644 --- a/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py +++ b/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py @@ -82,8 +82,8 @@ def _fake_evaluate_predictions(cfg, *, models, cp_space): seen[cfg.benchmark.dataset_ref.dataset] = cp_space return [], [], [] - def _fake_save_metrics(cfg, *args, cp_reference_sha256, **kwargs): - stamped[cfg.benchmark.dataset_ref.dataset] = cp_reference_sha256 + def _fake_save_metrics(cfg, *args, cp_space, **kwargs): + stamped[cfg.benchmark.dataset_ref.dataset] = cp_space.binding_sha256 monkeypatch.setattr(pipeline, "apply_dataset_ref", lambda cfg: None) monkeypatch.setattr(pipeline, "load_eval_models", lambda cfg: object()) @@ -98,7 +98,7 @@ def _fake_save_metrics(cfg, *args, cp_reference_sha256, **kwargs): assert space.scaler_dataset == _LITE.get(dataset, dataset) np.testing.assert_array_equal(space.mean, expected[dataset].mean) np.testing.assert_array_equal(space.std, expected[dataset].std) - assert stamped[dataset] == space.reference_sha256 + assert stamped[dataset] == space.binding_sha256 np.testing.assert_array_equal(seen["set-b-lite"].mean, seen["set-b"].mean) assert not np.allclose(seen["set-a"].mean, seen["set-c"].mean) diff --git a/applications/dynacell/tests/test_evaluation_pipeline.py b/applications/dynacell/tests/test_evaluation_pipeline.py index 9a604f6d9..5e9b6f37a 100644 --- a/applications/dynacell/tests/test_evaluation_pipeline.py +++ b/applications/dynacell/tests/test_evaluation_pipeline.py @@ -116,7 +116,7 @@ def test_evaluate_model_reuses_cache_without_feature_metrics( _write_metrics(tmp_path / config.save.mask_metrics_filename, expected_mask_metrics) # A reusable cache is one this code could have written, which includes the # numeric-provenance stamp save_metrics emits. - write_metrics_provenance(tmp_path, cp_reference_sha256=None) + write_metrics_provenance(tmp_path, cp_reference_sha256=None, cp_space_sha256=None) def fail_if_recomputed(_config): raise AssertionError("evaluate_predictions should not run when cache is valid") diff --git a/applications/dynacell/tests/test_evaluation_timepoint_guard.py b/applications/dynacell/tests/test_evaluation_timepoint_guard.py index 17441c795..45ed723e9 100644 --- a/applications/dynacell/tests/test_evaluation_timepoint_guard.py +++ b/applications/dynacell/tests/test_evaluation_timepoint_guard.py @@ -111,12 +111,13 @@ def _feature_cache_config(tmp_path: Path, **flags: bool): return config, make_cp_reference(config, tmp_path / "cp_reference.json") -def _write_final_caches(save_dir: Path, feature_row: dict, reference_sha256: str) -> None: - """Write a pixel/mask/feature NPY cache set holding one feature row, stamped with ``reference_sha256``.""" +def _write_final_caches(pipeline, save_dir: Path, feature_row: dict, config) -> None: + """Write a pixel/mask/feature NPY cache set holding one feature row, stamped as ``config`` would score it.""" np.save(save_dir / "pixel_metrics.npy", [{"SI_PSNR": 1.0, "SI_SSIM": 1.0, "SI_NRMSE": 1.0}]) np.save(save_dir / "mask_metrics.npy", [{"metric": "mask"}]) np.save(save_dir / "feature_metrics.npy", [feature_row]) - write_metrics_provenance(save_dir, cp_reference_sha256=reference_sha256) + space = pipeline.eval_cp_space(config) + write_metrics_provenance(save_dir, cp_reference_sha256=space.reference_sha256, cp_space_sha256=space.binding_sha256) def _dataset_row(prefixes: tuple[str, ...], families: tuple[str, ...]) -> dict: @@ -134,8 +135,10 @@ def _dataset_row(prefixes: tuple[str, ...], families: tuple[str, ...]) -> dict: def test_cache_without_fid_is_invalid_once_fid_is_enabled(tmp_path: Path) -> None: """A dir written with FID off is recomputed when FID is on, reused while it stays off.""" pipeline = live_pipeline_module() - config, sha256 = _feature_cache_config(tmp_path) - _write_final_caches(tmp_path, _dataset_row(("CP", "DINOv3"), tuple(f for f in _ALL_FAMILIES if f != "FID")), sha256) + config, _ = _feature_cache_config(tmp_path) + _write_final_caches( + pipeline, tmp_path, _dataset_row(("CP", "DINOv3"), tuple(f for f in _ALL_FAMILIES if f != "FID")), config + ) assert not pipeline._final_metrics_cache_valid(config) config.feature_metrics.compute_fid = False @@ -145,8 +148,8 @@ def test_cache_without_fid_is_invalid_once_fid_is_enabled(tmp_path: Path) -> Non def test_full_cache_stays_valid(tmp_path: Path) -> None: """A dir carrying every family for every prefix it scored remains reusable.""" pipeline = live_pipeline_module() - config, sha256 = _feature_cache_config(tmp_path) - _write_final_caches(tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES), sha256) + config, _ = _feature_cache_config(tmp_path) + _write_final_caches(pipeline, tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES), config) assert pipeline._final_metrics_cache_valid(config) @@ -158,16 +161,16 @@ def test_real_full_benchmark_columns_stay_valid(tmp_path: Path) -> None: with real.open() as f: header = f.readline().rstrip("\n").split(",") pipeline = live_pipeline_module() - config, sha256 = _feature_cache_config(tmp_path) - _write_final_caches(tmp_path, dict.fromkeys(header, 0.0), sha256) + config, _ = _feature_cache_config(tmp_path) + _write_final_caches(pipeline, tmp_path, dict.fromkeys(header, 0.0), config) assert pipeline._final_metrics_cache_valid(config) def test_cache_scored_in_another_cp_reference_is_invalid(tmp_path: Path) -> None: """Rebuilding the CP reference invalidates every final-metrics cache stamped with the old one.""" pipeline = live_pipeline_module() - config, sha256 = _feature_cache_config(tmp_path) - _write_final_caches(tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES), sha256) + config, _ = _feature_cache_config(tmp_path) + _write_final_caches(pipeline, tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES), config) assert pipeline._final_metrics_cache_valid(config) make_cp_reference(config, tmp_path / "cp_reference.json", seed=1) # same path, new content @@ -177,8 +180,8 @@ def test_cache_scored_in_another_cp_reference_is_invalid(tmp_path: Path) -> None def test_cache_reuse_refuses_a_gt_recache(tmp_path: Path) -> None: """A reusable final-metrics cache is refused once the GT CP cache was re-cached after the build.""" pipeline = live_pipeline_module() - config, sha256 = _feature_cache_config(tmp_path) # records the cache's real built_at - _write_final_caches(tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES), sha256) + config, _ = _feature_cache_config(tmp_path) # records the cache's real built_at + _write_final_caches(pipeline, tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES), config) assert pipeline._final_metrics_cache_valid(config) paths = cache_paths(tmp_path / "gt_cache") @@ -190,6 +193,54 @@ def test_cache_reuse_refuses_a_gt_recache(tmp_path: Path) -> None: assert type(err.value).__name__ == "StaleCacheError" +def _two_set_cache(pipeline, tmp_path: Path): + """A final-metrics cache scored for set-a of a two-set reference; returns ``(config, reference path)``.""" + config, _ = _feature_cache_config(tmp_path) + reference = tmp_path / "cp_reference.json" + make_cp_reference(config, reference, datasets=("set-a", "set-b")) # binds config to set-a + _write_final_caches(pipeline, tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES), config) + assert pipeline._final_metrics_cache_valid(config) + return config, reference + + +def test_cache_scored_for_another_dataset_is_invalid(tmp_path: Path) -> None: + """A save_dir scored with set-a's scaler is not reusable by a config bound to set-b (same reference).""" + pipeline = live_pipeline_module() + config, _ = _two_set_cache(pipeline, tmp_path) + config.benchmark.dataset_ref.dataset = "set-b" + assert not pipeline._final_metrics_cache_valid(config) + + +def test_cache_is_invalid_once_the_gt_matrix_changed(tmp_path: Path) -> None: + """Same numeric reference hash, but the dataset's recorded GT matrix changed: CP rows are stale.""" + pipeline = live_pipeline_module() + config, reference = _two_set_cache(pipeline, tmp_path) + payload = json.loads(reference.read_text()) + payload["fit"]["datasets"]["set-a"]["gt_matrix_sha256"] = "0" * 64 # unhashed fit provenance + reference.write_text(json.dumps(payload)) + assert pipeline.eval_cp_space(config).reference_sha256 == payload["sha256"] # numeric hash unchanged + assert not pipeline._final_metrics_cache_valid(config) + + +def test_cache_stays_valid_after_a_harmless_rebuild(tmp_path: Path) -> None: + """A rebuild that only moves built_at (same GT matrix) keeps the cache reusable. + + check_gt_cache passes because the rebuild records the cache's new built_at, and + the binding is unchanged because the GT-matrix sha256 is. + """ + pipeline = live_pipeline_module() + config, reference = _two_set_cache(pipeline, tmp_path) + paths = cache_paths(tmp_path / "gt_cache") + manifest = load_manifest(paths) + manifest["artifacts"]["cp_features"]["built_at"] = "2099-01-01T00:00:00+00:00" + save_manifest(paths, manifest) + payload = json.loads(reference.read_text()) + for record in payload["fit"]["datasets"].values(): + record["cp_cache_built_at"] = "2099-01-01T00:00:00+00:00" + reference.write_text(json.dumps(payload)) + assert pipeline._final_metrics_cache_valid(config) + + def test_feature_less_cache_ignores_the_cp_reference(tmp_path: Path) -> None: """A compute_feature_metrics=false dir with no CP hash in its stamp stays reusable, with no reference.""" pipeline = live_pipeline_module() @@ -261,6 +312,7 @@ def _fake_evaluate_predictions(cfg, *, cp_space): assert seen["cp_space"].reference_sha256 == sha256 stamp = json.loads((tmp_path / PROVENANCE_FILENAME).read_text()) assert stamp["cp_reference_sha256"] == sha256 + assert stamp["cp_space_sha256"] == seen["cp_space"].binding_sha256 def _feature_plates(tmp_path: Path) -> None: diff --git a/applications/dynacell/tests/test_pipeline_cache.py b/applications/dynacell/tests/test_pipeline_cache.py index 084c6a473..2f0aa4475 100644 --- a/applications/dynacell/tests/test_pipeline_cache.py +++ b/applications/dynacell/tests/test_pipeline_cache.py @@ -2128,7 +2128,7 @@ def test_final_metrics_cache_gate_requires_ap_columns(tmp_path: Path) -> None: save_dir = tmp_path / "out" save_dir.mkdir() # Reuse also requires the numeric-provenance stamp save_metrics writes. - write_metrics_provenance(save_dir, cp_reference_sha256=None) + write_metrics_provenance(save_dir, cp_reference_sha256=None, cp_space_sha256=None) np.save(save_dir / "pixel_metrics.npy", np.array([dict(_DUAL_SCALING_PIXEL_ROW)], dtype=object)) cfg = _make_config( **{ @@ -2167,7 +2167,7 @@ def test_final_metrics_cache_gate_requires_both_pixel_scalings(tmp_path: Path) - save_dir = tmp_path / "out" save_dir.mkdir() - write_metrics_provenance(save_dir, cp_reference_sha256=None) + write_metrics_provenance(save_dir, cp_reference_sha256=None, cp_space_sha256=None) cfg = _make_config( **{ "compute_instance_ap": False, From b7b8615210d017495358a112410eca1bd244ec06 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 19:56:05 -0700 Subject: [PATCH 17/41] fix(dynacell/eval): bypass the final-metrics cache on a partial position walk The cache-hit path returned the cached CP rows without running check_positions, so an eval with limit_positions or io.exclude_fov_names could reuse metrics scored over a different GT position set. Treat a partial walk as a cache miss so the scoring path's position check runs. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../dynacell/src/dynacell/evaluation/pipeline.py | 4 ++++ .../tests/test_evaluation_timepoint_guard.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline.py b/applications/dynacell/src/dynacell/evaluation/pipeline.py index 439fce8f3..7fa429031 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline.py @@ -1922,6 +1922,10 @@ def _final_metrics_cache_valid(config: DictConfig) -> bool: # Same GT re-cache guard as the scoring path: cached CP rows scored against a # GT cache that has since been re-cached must not be reused silently. space.check_gt_cache(OmegaConf.select(config, "io.gt_cache_dir", default=None)) + # The cached rows cover the positions of the run that wrote them; a partial walk + # must go through the scoring path so ``check_positions`` sees its GT position set. + if config.limit_positions is not None or OmegaConf.select(config, "io.exclude_fov_names", default=None): + return False current_sha256 = space.binding_sha256 if not metrics_provenance_matches(save_dir, cp_space_sha256=current_sha256): return False diff --git a/applications/dynacell/tests/test_evaluation_timepoint_guard.py b/applications/dynacell/tests/test_evaluation_timepoint_guard.py index 45ed723e9..e6158c663 100644 --- a/applications/dynacell/tests/test_evaluation_timepoint_guard.py +++ b/applications/dynacell/tests/test_evaluation_timepoint_guard.py @@ -16,6 +16,7 @@ import numpy as np import pytest from iohub.ngff import open_ome_zarr +from omegaconf import OmegaConf from dynacell.evaluation.cache import cache_paths, load_manifest, save_manifest from dynacell.evaluation.model_loader import EvalModels @@ -177,6 +178,19 @@ def test_cache_scored_in_another_cp_reference_is_invalid(tmp_path: Path) -> None assert not pipeline._final_metrics_cache_valid(config) +@pytest.mark.parametrize( + ("key", "value"), [("limit_positions", 1), ("io.exclude_fov_names", ["A/1/0"])], ids=["limit", "exclude"] +) +def test_partial_walk_never_reuses_cp_rows(tmp_path: Path, key: str, value) -> None: + """A partial position walk bypasses the cache so ``check_positions`` sees its GT position set.""" + pipeline = live_pipeline_module() + config, _ = _feature_cache_config(tmp_path) + _write_final_caches(pipeline, tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES), config) + assert pipeline._final_metrics_cache_valid(config) + OmegaConf.update(config, key, value) + assert not pipeline._final_metrics_cache_valid(config) + + def test_cache_reuse_refuses_a_gt_recache(tmp_path: Path) -> None: """A reusable final-metrics cache is refused once the GT CP cache was re-cached after the build.""" pipeline = live_pipeline_module() From 6adf7a537f44a0e00487f2585f2d4a02cb47fd7f Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 20:14:52 -0700 Subject: [PATCH 18/41] fix(dynacell/eval): check the GT store's positions before reusing CP rows The cache-hit path accepted cached CP metrics without comparing the GT store's current positions with the reference's fit positions, so a GT store extended or replaced since the fit could hand back rows scored on another cell set. Run the scoring path's check_positions on the GT store before accepting the cache. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../dynacell/src/dynacell/evaluation/pipeline.py | 4 ++++ .../tests/test_evaluation_timepoint_guard.py | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline.py b/applications/dynacell/src/dynacell/evaluation/pipeline.py index 7fa429031..3ae5ca1d8 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline.py @@ -1926,6 +1926,10 @@ def _final_metrics_cache_valid(config: DictConfig) -> bool: # must go through the scoring path so ``check_positions`` sees its GT position set. if config.limit_positions is not None or OmegaConf.select(config, "io.exclude_fov_names", default=None): return False + # Same position guard as the scoring path, on the GT store as it is now: a GT + # store replaced or extended since the fit must not hand back cached CP rows. + with open_ome_zarr(Path(config.io.gt_path), mode="r") as gt_plate: + space.check_positions([name for name, _ in gt_plate.positions()]) current_sha256 = space.binding_sha256 if not metrics_provenance_matches(save_dir, cp_space_sha256=current_sha256): return False diff --git a/applications/dynacell/tests/test_evaluation_timepoint_guard.py b/applications/dynacell/tests/test_evaluation_timepoint_guard.py index e6158c663..33ac2554a 100644 --- a/applications/dynacell/tests/test_evaluation_timepoint_guard.py +++ b/applications/dynacell/tests/test_evaluation_timepoint_guard.py @@ -22,7 +22,7 @@ from dynacell.evaluation.model_loader import EvalModels from dynacell.evaluation.provenance import PROVENANCE_FILENAME, write_metrics_provenance -from ._eval_fixtures import build_eval_config, live_pipeline_module, make_cp_reference +from ._eval_fixtures import N_POSITIONS, build_eval_config, live_pipeline_module, make_cp_reference D, H, W = 3, 8, 8 @@ -107,6 +107,8 @@ def _feature_cache_config(tmp_path: Path, **flags: bool): fov_workers=1, ) config.compute_feature_metrics = True + if not (tmp_path / "gt.zarr").exists(): # the cache-hit path checks the GT store's positions + _make_plate(tmp_path / "gt.zarr", "target", [2] * N_POSITIONS) for name, value in flags.items(): config.feature_metrics[name] = value return config, make_cp_reference(config, tmp_path / "cp_reference.json") @@ -191,6 +193,18 @@ def test_partial_walk_never_reuses_cp_rows(tmp_path: Path, key: str, value) -> N assert not pipeline._final_metrics_cache_valid(config) +def test_cache_reuse_refuses_a_gt_store_extended_since_the_fit(tmp_path: Path) -> None: + """A GT store that gained a position the reference was not fit on is refused on the cache-hit path.""" + pipeline = live_pipeline_module() + config, _ = _feature_cache_config(tmp_path) + _write_final_caches(pipeline, tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES), config) + assert pipeline._final_metrics_cache_valid(config) + with open_ome_zarr(tmp_path / "gt.zarr", mode="r+") as plate: + plate.create_position("A", "1", str(N_POSITIONS)).create_image("0", np.zeros((2, 1, D, H, W), dtype=np.float32)) + with pytest.raises(ValueError): + pipeline._final_metrics_cache_valid(config) + + def test_cache_reuse_refuses_a_gt_recache(tmp_path: Path) -> None: """A reusable final-metrics cache is refused once the GT CP cache was re-cached after the build.""" pipeline = live_pipeline_module() From 9ae416d07e1f025ac89e3c11bb44331ac0cfe0c6 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 20:27:48 -0700 Subject: [PATCH 19/41] fix(dynacell/eval): gate CP scoring on the GT cells' content, not built_at The built_at staleness check broke every later eval of a dataset after one --overwrite / force_recompute.gt_cp|all re-cached its GT CP (new stamp, same cells). _merge_manifests could also restore an old stamp over a real value change. And it ran before GT CP was recomputed, so a same-count value change was scored with a stale scaler. (Blockers 1-3 of the #499 review.) The eval now hashes the GT CP cells it actually staged: GT-finite rows before the pred/GT paired drop, so independent of the prediction. It uses the builder's canonical order (sorted positions, t ascending, cell order; cp_reference.canonical_gt_matrix). The stage compares this hash with the dataset's recorded gt_matrix_sha256 before any sidecar, metric or stamp is written. A mismatch raises StaleCacheError naming `build_cp_reference.py --target X --force`. - check_gt_cache, check_n_cells and every built_at check are removed; built_at stays in the fit record as audit only. The cache-hit path keeps check_positions and the partial-walk miss. - The builder reads cells through the shared read_gt_cp_cells, and records positions, count and gt sha for lite datasets from their own GT caches. check_positions now requires exact equality for lite too. --verify covers lite. Schema 4. - fit_cp_reference takes lite datasets as DatasetFit(parent=...). - New end-to-end tests through the real evaluate_predictions: an identical force_recompute.gt_cp re-cache passes, and a same-count GT value change raises before anything is written. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/README.md | 2 +- .../src/dynacell/evaluation/cp_reference.py | 316 +++++++++--------- .../dynacell/evaluation/cp_reference_test.py | 151 +++++---- .../src/dynacell/evaluation/pipeline.py | 41 ++- applications/dynacell/tests/_eval_fixtures.py | 51 +-- .../tests/test_cross_condition_probe.py | 2 +- .../tests/test_evaluation_cp_end_to_end.py | 172 ++++++++++ .../test_evaluation_pipeline_parallel.py | 2 +- .../tests/test_evaluation_timepoint_guard.py | 44 +-- .../dynacell/tools/build_cp_reference.py | 129 +++---- .../dynacell/tools/build_cp_reference_test.py | 21 +- .../dynacell/tools/lite_checks_test.py | 2 +- .../dynacell/tools/lite_subset_sim_test.py | 2 +- 13 files changed, 505 insertions(+), 430 deletions(-) create mode 100644 applications/dynacell/tests/test_evaluation_cp_end_to_end.py diff --git a/applications/dynacell/src/dynacell/evaluation/README.md b/applications/dynacell/src/dynacell/evaluation/README.md index 4b3e2359d..89c46feeb 100644 --- a/applications/dynacell/src/dynacell/evaluation/README.md +++ b/applications/dynacell/src/dynacell/evaluation/README.md @@ -24,7 +24,7 @@ End-to-end evaluation pipeline for virtual staining predictions against fluoresc - `io.gt_path` — fluorescence ground truth (channel: `io.gt_channel_name`) - `io.cell_segmentation_path` — *optional* precomputed cell segmentation HCS OME-Zarr. Required when `compute_feature_metrics=true` or when building CP/DINOv3/DynaCLR/CELL-DINO cache entries. Position layout must match GT/pred 1:1. - `io.gt_cache_dir`, `io.pred_cache_dir` — *optional* artifact cache directories; must be distinct. See [Caches](#caches). -- `feature_metrics.cp.reference_path` — the target's CP (GLCM+) reference, required when `compute_feature_metrics=true`. `null` (default) resolves to `DATA_ROOT/cp_reference/.json` for full and lite evals alike. The reference holds one feature mask chosen on GT cells only, pooled over iPSC + A549 mock/denv/zikv, and one GT mean/std per test set (keyed by `benchmark.dataset_ref.dataset`; HEK gets its own, lite reuses its parent's). Every eval applies the mask and its dataset's scaler to pred AND GT. The eval refuses a partial position walk, a GT cell count or GT-cache `built_at` that differs from the fit. Build it from existing GT caches with `tools/build_cp_reference.py --target `. Its hash is stamped in `metrics_provenance.json`, and a rebuilt reference invalidates the final-metrics cache. +- `feature_metrics.cp.reference_path` — the target's CP (GLCM+) reference, required when `compute_feature_metrics=true`. `null` (default) resolves to `DATA_ROOT/cp_reference/.json` for full and lite evals alike. The reference holds one feature mask chosen on GT cells only, pooled over iPSC + A549 mock/denv/zikv, and one GT mean/std per test set (keyed by `benchmark.dataset_ref.dataset`; HEK gets its own, lite reuses its parent's). Every eval applies the mask and its dataset's scaler to pred AND GT. The eval refuses a partial position walk, and refuses to score unless its staged GT CP cells hash to the sha256 the reference recorded for that dataset (a content gate: an identical re-cache passes, any value change fails). Build it from existing GT caches with `tools/build_cp_reference.py --target `. Its hash is stamped in `metrics_provenance.json`, and a rebuilt reference invalidates the final-metrics cache. ## Quick start diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index 35ae26ece..6be424a31 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -24,13 +24,13 @@ Each scaler is fit on exactly the GT cells an eval of that dataset scores: every position and timepoint of the GT store, read from the GT CP cache, with the -non-finite rows dropped. The reference records each dataset's position list, cell -count and GT-cache ``cp_features.built_at``, and an eval refuses to score if its -positions or cell count differ, or the cache stamp moved -(:meth:`DatasetCPSpace.check_positions`, :meth:`DatasetCPSpace.check_n_cells`, -:meth:`DatasetCPSpace.check_gt_cache`). The per-dataset GT-matrix sha256 is -recorded for audit and checked by ``build_cp_reference.py --verify``, not at eval -time. +non-finite rows dropped. The reference records every dataset's (lite included) +position list and the sha256 of those cells in canonical order +(:func:`canonical_gt_matrix`). An eval refuses a different position set up front +(:meth:`DatasetCPSpace.check_positions`) and, once its GT CP cells are staged, +refuses to score unless they hash to the recorded value +(:meth:`DatasetCPSpace.check_gt_cells`). That content gate is independent of the +cache manifest's ``built_at``, which is kept in the fit record for audit only. A per-dataset std can collapse on a feature the pooled mask keeps (a feature that is near-constant within one test set). Such a std is floored at @@ -55,9 +55,10 @@ from typing import Any import numpy as np +from iohub.ngff import open_ome_zarr from omegaconf import DictConfig, OmegaConf -from dynacell.evaluation.cache import StaleCacheError, cache_paths, load_manifest +from dynacell.evaluation.cache import StaleCacheError, open_features_group, read_features_from_group from dynacell.evaluation.feature_select import ( DEFAULT_CORR_THRESHOLD, DEFAULT_FREQ_CUT, @@ -69,7 +70,7 @@ from dynacell.evaluation.pipeline_cache import cp_recipe_identity #: Schema of the reference JSON. Bump on any change to its keys or their meaning. -CP_REFERENCE_SCHEMA = 3 +CP_REFERENCE_SCHEMA = 4 #: Sidecar each eval writes beside its metrics, naming the CP reference it scored in. CP_SIDECAR_FILENAME = "cp_selected_feature_mask.json" @@ -110,7 +111,7 @@ #: Keys excluded from the content hash. The hash covers only what changes CP numbers #: (recipe identity, criteria, feature names, keep-mask, every scaler's mean/std and #: floored features, the lite -> parent map). ``fit`` holds the fit provenance the eval -#: checks read (positions, cell counts, GT-cache ``built_at``, GT-matrix hashes), so a +#: checks read (positions, cell counts, GT-matrix hashes; ``built_at`` for audit), so a #: rebuild after a harmless GT re-cache that leaves every scaler unchanged keeps the #: hash and invalidates no final-metrics cache stamped with it. _UNHASHED_KEYS = frozenset({"sha256", "created_at", "fit"}) @@ -195,18 +196,99 @@ class DatasetFit: dataset : str ``benchmark.dataset_ref.dataset`` of the test set. cells : np.ndarray - ``(n_cells, n_features)`` finite raw GT CP cells, every position and timepoint. + ``(n_cells, n_features)`` finite raw GT CP cells in canonical order + (:func:`canonical_gt_matrix`): every position and timepoint. record : dict - JSON-serializable provenance: at least ``positions`` (sorted names), - ``gt_cache_dir`` and ``cp_cache_built_at``. + JSON-serializable provenance: at least ``positions`` (sorted names). in_mask_fit : bool Whether these cells enter the pooled mask fit. + parent : str or None, optional + For a lite dataset, the dataset whose scaler it reuses; its cells are then + recorded (hash, count, positions) but never fit. """ dataset: str cells: np.ndarray record: dict[str, Any] in_mask_fit: bool + parent: str | None = None + + +def canonical_gt_matrix(blocks: dict[tuple[str, int], np.ndarray], n_features: int) -> np.ndarray: + """Stack per-``(position, timepoint)`` GT CP rows in the canonical order the fit is hashed in. + + Sorted position names, then ``t`` ascending, then cell (row) order within a + block, as float64. The builder and the eval both hash this matrix, so it is the + single definition of "the GT cells a scaler describes". Callers pass only + GT-finite rows (non-finite GT rows dropped, BEFORE the pred/GT paired drop), so + the matrix does not depend on the prediction. + + Parameters + ---------- + blocks : dict + ``{(position, t): (n_cells_t, n_features) array}``; empty blocks may be omitted. + n_features : int + Column count, for the empty result. + + Returns + ------- + np.ndarray + ``(n_cells, n_features)`` float64 matrix. + """ + rows = [np.asarray(blocks[key], dtype=np.float64) for key in sorted(blocks) if blocks[key].shape[0]] + return np.concatenate(rows, axis=0) if rows else np.empty((0, n_features)) + + +def read_gt_cp_cells(ctx, gt_path: Path, n_features: int) -> tuple[np.ndarray, dict[str, Any]]: + """Read every finite GT CP cell of one GT store from its cache, read-only, in canonical order. + + Parameters + ---------- + ctx : pipeline_cache._CacheContext + Enabled GT-side cache context of the store. + gt_path : pathlib.Path + GT HCS store; its positions and per-position ``T`` define what must be cached. + n_features : int + Expected CP column count. + + Returns + ------- + tuple + ``(cells, record)``: the :func:`canonical_gt_matrix` of the GT-finite rows, + and ``{"positions", "n_timepoints", "n_cells_dropped_nonfinite"}``. + + Raises + ------ + StaleCacheError + If the CP cache is missing, a ``(position, timepoint)`` of the GT store is + missing from it, or an entry has the wrong column count. + ValueError + If a GT position is not a 3-D volume. + """ + with open_ome_zarr(gt_path, mode="r") as plate: + shapes = {name: (int(pos.data.shape[0]), int(pos.data.shape[2])) for name, pos in plate.positions()} + blocks: dict[tuple[str, int], np.ndarray] = {} + n_timepoints = n_dropped = 0 + with open_features_group(ctx.paths, "cp", mode="r") as group: + if group is None: + raise StaleCacheError(f"no CP feature cache at {ctx.paths.cp_features()}") + for pos_name, (t_count, z_depth) in sorted(shapes.items()): + if z_depth < 2: + raise ValueError(f"{gt_path}/{pos_name} has Z={z_depth}; CP regionprops are 3-D") + for t in range(t_count): + feats = read_features_from_group(group, pos_name, t) + if feats is None: + raise StaleCacheError(f"CP cache miss at {pos_name}/t{t} in {ctx.paths.cp_features()}") + n_timepoints += 1 + if feats.shape[0] == 0: + continue + if feats.shape[1] != n_features: + raise StaleCacheError(f"{pos_name}/t{t} has {feats.shape[1]} CP columns, expected {n_features}") + finite = np.isfinite(feats).all(axis=1) + n_dropped += int((~finite).sum()) + blocks[(pos_name, t)] = feats[finite] + record = {"positions": sorted(shapes), "n_timepoints": n_timepoints, "n_cells_dropped_nonfinite": n_dropped} + return canonical_gt_matrix(blocks, n_features), record def fit_cp_reference( @@ -215,25 +297,22 @@ def fit_cp_reference( target_name: str, feature_names: tuple[str, ...], cp_identity: dict[str, Any], - lite: dict[str, dict[str, Any]], ) -> dict[str, Any]: """Fit a CP reference and return its JSON payload. Parameters ---------- fits : list of DatasetFit - Every non-lite test set that needs a scaler. Those with ``in_mask_fit`` - are pooled for the feature mask and the floor's pooled std. + Every test set of the target. A fit without ``parent`` gets a scaler; those + with ``in_mask_fit`` are pooled for the feature mask and the floor's pooled + std. A fit with ``parent`` (lite) is mapped to that parent's scaler, and + only its cell hash, count and positions are recorded. target_name : str Eval ``target_name`` (``nucleus``/``membrane``/``er``/``mitochondria``). feature_names : tuple of str Column names of every ``cells`` matrix. cp_identity : dict Dataset-independent CP recipe identity (from :func:`cp_space`). - lite : dict - ``{lite dataset: {"parent": ..., "gt_cache_dir": ..., "cp_cache_built_at": ...}}``. - Each parent must be one of ``fits``. Only ``parent`` enters the hash; the - rest is fit provenance. Returns ------- @@ -245,12 +324,13 @@ def fit_cp_reference( ValueError If a matrix does not match ``feature_names`` or holds a non-finite value, a dataset has no cells, a dataset appears twice, a lite parent has no - scaler, no dataset is in the mask fit, or the pooled fit keeps a - zero-variance feature. + scaler, a lite fit is in the mask fit, no dataset is in the mask fit, or + the pooled fit keeps a zero-variance feature. """ names = [f.dataset for f in fits] if len(set(names)) != len(names): raise ValueError(f"duplicate datasets: {names}") + scaled = [f for f in fits if f.parent is None] for fit in fits: if fit.cells.ndim != 2 or fit.cells.shape[1] != len(feature_names): raise ValueError(f"{fit.dataset}: GT matrix shape {fit.cells.shape} does not match {len(feature_names)}") @@ -258,11 +338,10 @@ def fit_cp_reference( raise ValueError(f"{fit.dataset}: no GT cells") if not np.isfinite(fit.cells).all(): raise ValueError(f"{fit.dataset}: GT matrix holds non-finite values; drop them before fitting") - for lite_name, entry in lite.items(): - if entry["parent"] not in names: - raise ValueError(f"lite dataset {lite_name} reuses {entry['parent']}, which has no scaler") + if fit.parent is not None and (fit.in_mask_fit or fit.parent not in {f.dataset for f in scaled}): + raise ValueError(f"lite dataset {fit.dataset} must reuse a scaled parent, not {fit.parent!r}") - pooled = np.concatenate([np.asarray(f.cells, dtype=np.float64) for f in fits if f.in_mask_fit], axis=0) + pooled = np.concatenate([np.asarray(f.cells, dtype=np.float64) for f in scaled if f.in_mask_fit], axis=0) if pooled.shape[0] == 0: raise ValueError("no dataset is in the mask fit") keep_mask = select_gt_features( @@ -274,9 +353,16 @@ def fit_cp_reference( raise ValueError(f"pooled fit kept zero-variance features: {np.array(kept_names)[pooled_std == 0]}") floor = STD_FLOOR_FRACTION * pooled_std + def _provenance(fit: DatasetFit) -> dict[str, Any]: + return { + **fit.record, + "in_mask_fit": fit.in_mask_fit, + "n_cells": int(fit.cells.shape[0]), + "gt_matrix_sha256": gt_matrix_sha256(fit.cells), + } + scalers: dict[str, dict[str, Any]] = {} - provenance: dict[str, dict[str, Any]] = {} - for fit in fits: + for fit in scaled: kept = np.asarray(fit.cells, dtype=np.float64)[:, keep_mask] std = kept.std(axis=0) floored = std < floor @@ -285,12 +371,7 @@ def fit_cp_reference( "std": [float(v) for v in np.where(floored, floor, std)], "floored_features": [n for n, f in zip(kept_names, floored, strict=True) if f], } - provenance[fit.dataset] = { - **fit.record, - "in_mask_fit": fit.in_mask_fit, - "n_cells": int(kept.shape[0]), - "gt_matrix_sha256": gt_matrix_sha256(fit.cells), - } + lite = [f for f in fits if f.parent is not None] payload: dict[str, Any] = { "schema": CP_REFERENCE_SCHEMA, @@ -308,16 +389,16 @@ def fit_cp_reference( }, "cp_identity": cp_identity, "scalers": scalers, - "lite": {name: {"parent": entry["parent"]} for name, entry in lite.items()}, + "lite": {f.dataset: {"parent": f.parent} for f in lite}, "fit": { "mask_fit": { - "datasets": [f.dataset for f in fits if f.in_mask_fit], + "datasets": [f.dataset for f in scaled if f.in_mask_fit], "n_cells": int(pooled.shape[0]), "gt_matrix_sha256": gt_matrix_sha256(pooled), "pooled_std": [float(v) for v in pooled_std], }, - "datasets": provenance, - "lite": {name: {k: v for k, v in entry.items() if k != "parent"} for name, entry in lite.items()}, + "datasets": {f.dataset: _provenance(f) for f in scaled}, + "lite": {f.dataset: _provenance(f) for f in lite}, }, "created_at": datetime.now(UTC).isoformat(), } @@ -342,8 +423,8 @@ def write_cp_reference(payload: dict[str, Any], path: Path, *, force: bool = Fal bool ``True`` if the file was written, ``False`` if an identical reference (same hash AND same fit provenance) was already there (no-op). Same hash with new - provenance -- a harmless GT re-cache -- is written without ``force``: no CP - number changes, and the eval checks need the new ``built_at``. + provenance (e.g. a re-cache that moved ``built_at``, or a lite dataset's + record) is written without ``force``: no scaler or mask changes. Raises ------ @@ -388,40 +469,28 @@ class DatasetCPSpace: std: np.ndarray fit_positions: frozenset[str] fit_n_cells: int - gt_cache_dir: str | None - cp_cache_built_at: str | None - fit_gt_matrix_sha256: str - own_gt_matrix_sha256: str | None + gt_matrix_sha256: str @property def binding_sha256(self) -> str: - """Return a sha256 identifying this eval's CP space AND the GT cells it describes. - - The final-metrics stamp needs more than :attr:`reference_sha256`, which - deliberately leaves fit provenance unhashed: a cache scored for dataset A - must not be reused for dataset B, and a GT re-cache that changes the cells - while leaving mean/std/mask unchanged must not reuse CP rows scored on the - old cells. This hash covers the reference hash, the dataset, its scaler - dataset, the lite flag, and the recorded GT-matrix sha256. That is the - dataset's own for a non-lite set. A lite set uses its own when one is - recorded, else the parent's plus the lite cache's recorded ``built_at``. - A harmless rebuild that only moves ``built_at`` on a non-lite set keeps it. + """Return a sha256 identifying this eval's CP space AND the GT cells it was fit on. + + Stamped as ``cp_space_sha256`` so cache reuse refuses another dataset's + space and GT cells that differ from the fit. Covers the reference hash, the + dataset, its scaler dataset, the lite flag, and this dataset's recorded + GT-matrix sha256 (a lite set's own). Returns ------- str Hex sha256 over the canonical JSON of those fields. """ - if self.is_lite and self.own_gt_matrix_sha256 is None: - gt = {"parent_gt_matrix_sha256": self.fit_gt_matrix_sha256, "cp_cache_built_at": self.cp_cache_built_at} - else: - gt = {"gt_matrix_sha256": self.own_gt_matrix_sha256 or self.fit_gt_matrix_sha256} body = { "reference_sha256": self.reference_sha256, "dataset": self.dataset, "scaler_dataset": self.scaler_dataset, "is_lite": self.is_lite, - **gt, + "gt_matrix_sha256": self.gt_matrix_sha256, } return hashlib.sha256(json.dumps(body, sort_keys=True, separators=(",", ":")).encode()).hexdigest() @@ -445,62 +514,8 @@ def transform(self, x: np.ndarray) -> np.ndarray: """Return the kept columns of a raw CP matrix, standardized by this dataset's GT scaler.""" return (self.select(x) - self.mean) / self.std - def check_gt_cache(self, gt_cache_dir: str | None) -> None: - """Refuse an eval whose GT CP cache was re-cached after the reference was fit. - - Compares the eval's GT cache manifest ``cp_features.built_at`` with the one - recorded at build time. A non-lite dataset must have one recorded. A lite - dataset is checked against its own cache only when the build recorded one - (its scaler is the parent's, so a lite cache is otherwise not what the - scaler describes); when nothing is recorded the check is skipped. - - This is a best-effort re-cache signal, not a content check. ``built_at`` - moves whenever any slot of the cache's CP artifact is rewritten, and the - cache-manifest merge (``pipeline_cache._merge_manifests``) takes - ``built_at`` from whichever writer flushes last, so it can also move - without a value change (a false alarm) or be restored by a concurrent - flush (a miss). :meth:`check_n_cells` catches a change in the GT cell - count; a value-only change with the same count is caught by neither, only - by ``build_cp_reference.py --verify``, which recomputes each dataset's - GT-matrix sha256 from the caches. - - Parameters - ---------- - gt_cache_dir : str or None - The eval's ``io.gt_cache_dir``. - - Raises - ------ - ValueError - If the eval reads a different GT cache dir than the recorded one, or a - non-lite dataset's reference records no ``built_at``. - StaleCacheError - If the recorded and current ``built_at`` differ. - """ - if self.cp_cache_built_at is None: - if self.is_lite: - return - raise ValueError( - f"{self.dataset}: the CP reference {self.reference_path} records no GT-cache built_at for this " - f"non-lite dataset, so a GT re-cache could not be detected. Rebuild it:\n" - f" {_build_command(self.target_name)}" - ) - if gt_cache_dir is None or Path(gt_cache_dir) != Path(self.gt_cache_dir): - raise ValueError( - f"{self.dataset}: the CP reference was fit on GT cache {self.gt_cache_dir}, " - f"this eval reads {gt_cache_dir}" - ) - entry = load_manifest(cache_paths(gt_cache_dir))["artifacts"].get("cp_features") - built_at = entry["built_at"] if entry is not None else None - if built_at != self.cp_cache_built_at: - raise StaleCacheError( - f"{self.dataset}: GT CP cache {gt_cache_dir} was built at {built_at}, but the CP reference " - f"{self.reference_path} was fit on the cache built at {self.cp_cache_built_at}. " - f"Rebuild the reference:\n {_build_command(self.target_name)}" - ) - def check_positions(self, positions: list[str]) -> None: - """Refuse an eval whose GT positions are not the ones the scaler was fit on. + """Refuse an eval whose GT positions are not exactly the ones the fit recorded. Parameters ---------- @@ -510,47 +525,49 @@ def check_positions(self, positions: list[str]) -> None: Raises ------ ValueError - If the positions are not a subset of the fit's, or, for a non-lite - dataset, not exactly the fit's (e.g. ``limit_positions`` or - ``io.exclude_fov_names``: a partial walk's CP values are not the - cells the scaler describes). + If the positions differ from the fit's (e.g. ``limit_positions`` or + ``io.exclude_fov_names``: a partial walk's CP cells are not the cells + the scaler was fit on and hashed over). """ - extra = set(positions) - self.fit_positions - if extra: - raise ValueError( - f"{self.dataset}: GT positions {sorted(extra)[:5]} are not in the CP reference fit of " - f"{self.scaler_dataset} ({self.reference_path})" - ) - if not self.is_lite and set(positions) != self.fit_positions: + if set(positions) != self.fit_positions: + extra = sorted(set(positions) - self.fit_positions) missing = sorted(self.fit_positions - set(positions)) raise ValueError( - f"{self.dataset}: CP metrics need every GT position the scaler was fit on; this eval " - f"skips {len(missing)} (e.g. {missing[:5]}). limit_positions / io.exclude_fov_names " - "cannot be combined with compute_feature_metrics=true." + f"{self.dataset}: CP metrics need exactly the GT positions the CP reference recorded; this eval " + f"adds {extra[:5]} and skips {len(missing)} (e.g. {missing[:5]}). limit_positions / " + "io.exclude_fov_names cannot be combined with CP feature metrics: pass " + "compute_feature_metrics=false for smoke runs." ) - def check_n_cells(self, n_gt_cells: int) -> None: - """Refuse a dataset-level CP stage whose GT cell count differs from the fit's. + def check_gt_cells(self, blocks: dict[tuple[str, int], np.ndarray]) -> None: + """Refuse to score unless the eval's GT CP cells are exactly the reference's fit cells. - Lite datasets are exempt: they score a subset of the parent's cells, and - :meth:`CPReference.for_dataset` already required the parent to be recorded. - A value-only GT change with the same count passes; see :meth:`check_gt_cache`. + The content gate. It hashes the GT cells this run actually staged -- after + any GT CP recompute -- in the builder's canonical form + (:func:`canonical_gt_matrix`) and compares with the recorded GT-matrix + sha256. So a GT re-cache that leaves the cells identical (``--overwrite``, + ``force_recompute.gt_cp``/``all``) passes, and any value or count change + fails, however the cache manifest was merged. Parameters ---------- - n_gt_cells : int - Finite GT CP rows the eval scored, summed over every position and - timepoint, before the pairwise pred/GT non-finite drop. + blocks : dict + ``{(position, t): GT-finite CP rows}`` over every scored FOV and + timepoint, before the pred/GT paired non-finite drop. Raises ------ - ValueError - If ``n_gt_cells`` (finite GT CP rows scored) differs from the recorded count. + StaleCacheError + If the hash differs from the recorded one. """ - if not self.is_lite and n_gt_cells != self.fit_n_cells: - raise ValueError( - f"{self.dataset}: this eval scores {n_gt_cells} finite GT CP cells, the CP reference scaler was " - f"fit on {self.fit_n_cells}. The GT CP cache changed since the reference was built." + cells = canonical_gt_matrix(blocks, len(self.feature_names)) + sha256 = gt_matrix_sha256(cells) + if sha256 != self.gt_matrix_sha256: + raise StaleCacheError( + f"{self.dataset}: the GT CP cells this eval scores ({cells.shape[0]} cells, sha256 {sha256[:12]}) " + f"differ from the CP reference fit ({self.fit_n_cells} cells, sha256 {self.gt_matrix_sha256[:12]}) " + f"in {self.reference_path}. The GT CP cache changed since the reference was built; rebuild it:\n" + f" {_build_command(self.target_name)} --force" ) @@ -585,8 +602,7 @@ def for_dataset(self, dataset: str) -> DatasetCPSpace: ) scaler_dataset = self.lite[dataset]["parent"] if is_lite else dataset scaler = self.scalers[scaler_dataset] - scaler_fit = self.fit["datasets"][scaler_dataset] - own = self.fit["lite"][dataset] if is_lite else scaler_fit + own = self.fit["lite"][dataset] if is_lite else self.fit["datasets"][dataset] return DatasetCPSpace( reference_path=self.path, reference_sha256=self.sha256, @@ -598,13 +614,9 @@ def for_dataset(self, dataset: str) -> DatasetCPSpace: keep_mask=self.keep_mask, mean=np.asarray(scaler["mean"], dtype=np.float64), std=np.asarray(scaler["std"], dtype=np.float64), - fit_positions=frozenset(scaler_fit["positions"]), - fit_n_cells=int(scaler_fit["n_cells"]), - gt_cache_dir=own["gt_cache_dir"], - cp_cache_built_at=own["cp_cache_built_at"], - fit_gt_matrix_sha256=scaler_fit["gt_matrix_sha256"], - # A lite entry records no GT matrix today; the key is optional by design. - own_gt_matrix_sha256=own.get("gt_matrix_sha256") if is_lite else scaler_fit["gt_matrix_sha256"], + fit_positions=frozenset(own["positions"]), + fit_n_cells=int(own["n_cells"]), + gt_matrix_sha256=own["gt_matrix_sha256"], ) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py index 449ed610f..6c33fea05 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -10,7 +10,7 @@ from omegaconf import OmegaConf from dynacell.evaluation import cp_reference -from dynacell.evaluation.cache import StaleCacheError, cache_paths, save_manifest +from dynacell.evaluation.cache import StaleCacheError from dynacell.evaluation.cp_reference import ( STD_FLOOR_FRACTION, DatasetFit, @@ -34,6 +34,13 @@ _N_FEATURES = 8 _NAMES = tuple(f"f{i}" for i in range(_N_FEATURES)) _POSITIONS = ["A/1/0", "A/1/1"] +_LITE_ROWS = 150 # the lite set's GT cells: set-a's first position + + +def _blocks(gt: np.ndarray) -> dict[tuple[str, int], np.ndarray]: + """GT cells as the eval stages them: the two halves as timepoints 0 and 1 of A/1/0.""" + half = gt.shape[0] // 2 + return {("A/1/0", 0): gt[:half], ("A/1/0", 1): gt[half:]} def _gt(n: int = 300, seed: int = 0, offset: float = 10.0, scale: float = 1.0) -> np.ndarray: @@ -45,20 +52,27 @@ def _gt(n: int = 300, seed: int = 0, offset: float = 10.0, scale: float = 1.0) - return x -def _fit(dataset: str, cells: np.ndarray, *, in_mask_fit: bool = True, built_at: str | None = None) -> DatasetFit: +def _fit( + dataset: str, + cells: np.ndarray, + *, + in_mask_fit: bool = True, + built_at: str | None = None, + parent: str | None = None, + positions: list[str] = _POSITIONS, +) -> DatasetFit: return DatasetFit( dataset=dataset, cells=cells, - record={"positions": _POSITIONS, "gt_cache_dir": "/c/" + dataset, "cp_cache_built_at": built_at}, + record={"positions": positions, "cp_cache_built_at": built_at}, in_mask_fit=in_mask_fit, + parent=parent, ) -def _reference(tmp_path: Path, fits: list[DatasetFit], lite: dict | None = None): +def _reference(tmp_path: Path, fits: list[DatasetFit]): """Fit, write and load a reference; returns ``(CPReference, payload)``.""" - payload = fit_cp_reference( - fits, target_name="er", feature_names=_NAMES, cp_identity={"cp_feature_version": "test"}, lite=lite or {} - ) + payload = fit_cp_reference(fits, target_name="er", feature_names=_NAMES, cp_identity={"cp_feature_version": "test"}) path = tmp_path / "er.json" write_cp_reference(payload, path, force=True) return load_cp_reference(path, target_name="er"), payload @@ -68,11 +82,8 @@ def _reference(tmp_path: Path, fits: list[DatasetFit], lite: dict | None = None) def two_sets(tmp_path: Path): """A reference over two test sets whose GT differ by a large offset and scale, plus HEK and one lite.""" a, b, hek = _gt(seed=0), _gt(seed=1, offset=40.0, scale=3.0), _gt(seed=2, offset=-5.0) - ref, payload = _reference( - tmp_path, - [_fit("set-a", a), _fit("set-b", b), _fit("hek", hek, in_mask_fit=False)], - lite={"set-a-lite": {"parent": "set-a", "gt_cache_dir": "/lite", "cp_cache_built_at": None}}, - ) + lite = _fit("set-a-lite", a[:_LITE_ROWS], in_mask_fit=False, parent="set-a", positions=_POSITIONS[:1]) + ref, payload = _reference(tmp_path, [_fit("set-a", a), _fit("set-b", b), _fit("hek", hek, in_mask_fit=False), lite]) return ref, payload, {"set-a": a, "set-b": b, "hek": hek} @@ -144,13 +155,13 @@ def test_pooled_zero_variance_is_refused(monkeypatch) -> None: # in sys.modules, so a dotted-path patch could land on a different module. monkeypatch.setattr(cp_reference, "select_gt_features", lambda gt, **kw: np.ones(gt.shape[1], dtype=bool)) with pytest.raises(ValueError, match="zero-variance"): - fit_cp_reference([_fit("set-a", _gt())], target_name="er", feature_names=_NAMES, cp_identity={}, lite={}) + fit_cp_reference([_fit("set-a", _gt())], target_name="er", feature_names=_NAMES, cp_identity={}) def test_empty_dataset_is_refused() -> None: with pytest.raises(ValueError, match="no GT cells"): fit_cp_reference( - [_fit("set-a", np.empty((0, _N_FEATURES)))], target_name="er", feature_names=_NAMES, cp_identity={}, lite={} + [_fit("set-a", np.empty((0, _N_FEATURES)))], target_name="er", feature_names=_NAMES, cp_identity={} ) @@ -175,12 +186,11 @@ def test_harmless_recache_keeps_the_hash(tmp_path: Path) -> None: """ cells = _gt() first = fit_cp_reference( - [_fit("set-a", cells, built_at="t0")], target_name="er", feature_names=_NAMES, cp_identity={}, lite={} + [_fit("set-a", cells, built_at="t0")], target_name="er", feature_names=_NAMES, cp_identity={} ) fit_b = _fit("set-a", cells, built_at="t1") fit_b.record["positions"] = [*_POSITIONS] - fit_b.record["gt_cache_dir"] = "/moved/cache" - second = fit_cp_reference([fit_b], target_name="er", feature_names=_NAMES, cp_identity={}, lite={}) + second = fit_cp_reference([fit_b], target_name="er", feature_names=_NAMES, cp_identity={}) assert first["sha256"] == second["sha256"] assert first["fit"] != second["fit"] @@ -193,12 +203,9 @@ def test_harmless_recache_keeps_the_hash(tmp_path: Path) -> None: def test_changed_scaler_changes_the_hash() -> None: """Anything that moves a CP number -- here one dataset's mean -- changes the hash.""" + lite = _fit("set-a-lite", _gt()[:10], in_mask_fit=False, parent="set-a") payload = fit_cp_reference( - [_fit("set-a", _gt()), _fit("set-b", _gt(seed=1))], - target_name="er", - feature_names=_NAMES, - cp_identity={}, - lite={"set-a-lite": {"parent": "set-a", "gt_cache_dir": None, "cp_cache_built_at": None}}, + [_fit("set-a", _gt()), _fit("set-b", _gt(seed=1)), lite], target_name="er", feature_names=_NAMES, cp_identity={} ) moved = json.loads(json.dumps(payload)) moved["scalers"]["set-b"]["mean"][0] += 1e-9 @@ -213,10 +220,8 @@ def test_changed_scaler_changes_the_hash() -> None: def test_write_is_atomic_and_refuses_a_different_reference(tmp_path: Path) -> None: """Identical hash is a no-op; a different one needs force; no tmp file is left behind.""" - first = fit_cp_reference([_fit("set-a", _gt())], target_name="er", feature_names=_NAMES, cp_identity={}, lite={}) - other = fit_cp_reference( - [_fit("set-a", _gt(seed=5))], target_name="er", feature_names=_NAMES, cp_identity={}, lite={} - ) + first = fit_cp_reference([_fit("set-a", _gt())], target_name="er", feature_names=_NAMES, cp_identity={}) + other = fit_cp_reference([_fit("set-a", _gt(seed=5))], target_name="er", feature_names=_NAMES, cp_identity={}) path = tmp_path / "er.json" assert write_cp_reference(first, path) is True assert write_cp_reference({**first, "created_at": "later"}, path) is False @@ -251,44 +256,35 @@ def test_registry_path_is_shared_by_lite_and_full() -> None: def test_positions_must_match_the_fit(two_sets) -> None: - """Non-lite: exactly the fit's positions. Lite: any subset of its parent's.""" + """Every dataset (lite included) must score exactly the positions its cells were recorded over.""" ref, _, _ = two_sets ref.for_dataset("set-a").check_positions(_POSITIONS) - with pytest.raises(ValueError, match="cannot be combined"): + with pytest.raises(ValueError, match="compute_feature_metrics=false for smoke runs"): ref.for_dataset("set-a").check_positions(_POSITIONS[:1]) - with pytest.raises(ValueError, match="not in the CP reference fit"): + with pytest.raises(ValueError, match="adds \\['B/1/0'\\]"): ref.for_dataset("set-a").check_positions([*_POSITIONS, "B/1/0"]) ref.for_dataset("set-a-lite").check_positions(_POSITIONS[:1]) - with pytest.raises(ValueError, match="not in the CP reference fit"): + with pytest.raises(ValueError, match="skips 1"): ref.for_dataset("set-a-lite").check_positions(["B/1/0"]) -def test_gt_recache_is_detected(tmp_path: Path) -> None: - """A GT cache whose cp_features.built_at moved after the build is refused.""" - cache_dir = tmp_path / "cache" - save_manifest(cache_paths(cache_dir), {"artifacts": {"cp_features": {"built_at": "t0"}}}) - fit = DatasetFit( - dataset="set-a", - cells=_gt(), - record={"positions": _POSITIONS, "gt_cache_dir": str(cache_dir), "cp_cache_built_at": "t0"}, - in_mask_fit=True, - ) - ref, _ = _reference(tmp_path, [fit]) +def test_content_gate_passes_identical_cells_and_refuses_a_value_change(two_sets) -> None: + """The gate hashes the staged GT cells: an identical re-cache passes, a same-count value change raises.""" + ref, _, cells = two_sets space = ref.for_dataset("set-a") - space.check_gt_cache(str(cache_dir)) - with pytest.raises(ValueError, match="this eval reads"): - space.check_gt_cache(str(tmp_path / "elsewhere")) - save_manifest(cache_paths(cache_dir), {"artifacts": {"cp_features": {"built_at": "t1"}}}) - with pytest.raises(StaleCacheError, match="was built at t1"): - space.check_gt_cache(str(cache_dir)) - - -def test_lite_without_a_recorded_cache_skips_the_recache_check(two_sets) -> None: - """A lite dataset with no recorded cache stamp skips the check; a non-lite one refuses.""" - ref, _, _ = two_sets - ref.for_dataset("set-a-lite").check_gt_cache(None) - with pytest.raises(ValueError, match="records no GT-cache built_at for this non-lite dataset"): - ref.for_dataset("set-b").check_gt_cache("/c/set-b") + gt = cells["set-a"] + space.check_gt_cells(_blocks(gt.copy())) # e.g. force_recompute.gt_cp reproducing the same cells + space.check_gt_cells(dict(reversed(list(_blocks(gt).items())))) # staging order does not matter + changed = gt.copy() + changed[5, 2] += 1e-6 # same count, one value moved + with pytest.raises(StaleCacheError, match="(?s)differ from the CP reference fit .*--target er --force"): + space.check_gt_cells(_blocks(changed)) + with pytest.raises(StaleCacheError, match="299 cells"): + space.check_gt_cells(_blocks(gt[:-1])) + lite = ref.for_dataset("set-a-lite") + lite.check_gt_cells(_blocks(gt[:_LITE_ROWS])) + with pytest.raises(StaleCacheError): + lite.check_gt_cells(_blocks(gt)) # the parent's cells are not the lite set's def test_staged_dataset_arrays_are_the_dataset_scaler_transform(two_sets, tmp_path: Path) -> None: @@ -301,7 +297,7 @@ def test_staged_dataset_arrays_are_the_dataset_scaler_transform(two_sets, tmp_pa gt = cells["set-b"] pred = 0.5 * gt + 7.0 space = ref.for_dataset("set-b") - staged = _stage_cp_dataset_inputs(_lists(pred, gt), space, gt.shape[0], tmp_path) + staged = _stage_cp_dataset_inputs(_lists(pred, gt), space, _blocks(gt), tmp_path) np.testing.assert_array_equal(staged[1], space.transform(pred)) np.testing.assert_array_equal(staged[2], space.transform(gt)) np.testing.assert_array_equal(staged[3], pred[:, ref.keep_mask]) # probe: masked, unscaled @@ -317,13 +313,13 @@ def test_staged_dataset_arrays_are_the_dataset_scaler_transform(two_sets, tmp_pa ) -def test_staged_cell_count_must_match_the_fit(two_sets, tmp_path: Path) -> None: - """A non-lite eval scoring a different number of GT cells than the fit is refused; lite is exempt.""" +def test_staging_refuses_gt_cells_that_differ_from_the_fit(two_sets, tmp_path: Path) -> None: + """The dataset-level stage runs the content gate before writing its sidecar.""" ref, _, cells = two_sets gt = cells["set-a"] - with pytest.raises(ValueError, match="scores 299 finite GT CP cells"): - _stage_cp_dataset_inputs(_lists(gt, gt), ref.for_dataset("set-a"), 299, tmp_path) - _stage_cp_dataset_inputs(_lists(gt[:40], gt[:40]), ref.for_dataset("set-a-lite"), 40, tmp_path) + with pytest.raises(StaleCacheError, match="differ from the CP reference fit"): + _stage_cp_dataset_inputs(_lists(gt[:-1], gt[:-1]), ref.for_dataset("set-a"), _blocks(gt[:-1]), tmp_path) + assert not (tmp_path / "cp_selected_feature_mask.json").exists() def test_row_features_use_the_dataset_scaler(two_sets) -> None: @@ -359,7 +355,7 @@ def test_two_models_share_one_mask(two_sets, tmp_path: Path) -> None: for i, pred in enumerate((pred_a, pred_b)): save_dir = tmp_path / f"model_{i}" save_dir.mkdir() - _stage_cp_dataset_inputs(_lists(pred, gt), ref.for_dataset("set-a"), gt.shape[0], save_dir) + _stage_cp_dataset_inputs(_lists(pred, gt), ref.for_dataset("set-a"), _blocks(gt), save_dir) masks.append(json.loads((save_dir / "cp_selected_feature_mask.json").read_text())["keep_mask"]) assert masks[0] == masks[1] == [bool(b) for b in ref.keep_mask] old_a = select_gt_features(np.vstack([gt, pred_a])) @@ -399,23 +395,27 @@ def test_sidecar_binds_the_recorded_reference_and_dataset(two_sets, tmp_path: Pa ref, _, cells = two_sets eval_dir = tmp_path / "eval" eval_dir.mkdir() - _stage_cp_dataset_inputs( - _lists(cells["set-a"][:40], cells["set-a"][:40]), ref.for_dataset("set-a-lite"), 40, eval_dir - ) + lite_gt = cells["set-a"][:_LITE_ROWS] + _stage_cp_dataset_inputs(_lists(lite_gt, lite_gt), ref.for_dataset("set-a-lite"), _blocks(lite_gt), eval_dir) space = sidecar_cp_space(eval_dir) assert (space.dataset, space.scaler_dataset, space.reference_sha256) == ("set-a-lite", "set-a", ref.sha256) np.testing.assert_array_equal(space.mean, ref.for_dataset("set-a").mean) - _reference(tmp_path, [_fit("set-a", _gt(seed=8))], lite={"set-a-lite": {"parent": "set-a"}}) # rebuilt in place + rebuilt = [_fit("set-a", _gt(seed=8)), _fit("set-a-lite", lite_gt, in_mask_fit=False, parent="set-a")] + _reference(tmp_path, rebuilt) # rebuilt in place with pytest.raises(ValueError, match="changed since the eval"): sidecar_cp_space(eval_dir) def test_binding_separates_datasets_and_tracks_the_gt_matrix(tmp_path: Path) -> None: - """The binding differs per dataset (lite vs parent too) and moves with GT matrix or lite stamp, not built_at.""" - fits = [_fit("set-a", _gt(), built_at="t0"), _fit("set-b", _gt(seed=1), built_at="t0")] - lite = {"set-a-lite": {"parent": "set-a", "gt_cache_dir": "/lite", "cp_cache_built_at": "L0"}} - ref, payload = _reference(tmp_path, fits, lite=lite) + """The binding differs per dataset (lite vs parent too) and moves with that dataset's GT matrix, not built_at.""" + a = _gt() + fits = [ + _fit("set-a", a, built_at="t0"), + _fit("set-b", _gt(seed=1), built_at="t0"), + _fit("set-a-lite", a[:_LITE_ROWS], in_mask_fit=False, parent="set-a", positions=_POSITIONS[:1]), + ] + ref, payload = _reference(tmp_path, fits) bindings = {d: ref.for_dataset(d).binding_sha256 for d in ("set-a", "set-b", "set-a-lite")} assert len(set(bindings.values())) == 3 @@ -427,10 +427,9 @@ def rebound(edit) -> dict[str, str]: assert again.sha256 == ref.sha256 # provenance only: numeric hash unchanged return {d: again.for_dataset(d).binding_sha256 for d in bindings} - built_at = rebound(lambda p: p["fit"]["datasets"]["set-a"].update(cp_cache_built_at="t1")) - assert built_at == bindings + assert rebound(lambda p: p["fit"]["datasets"]["set-a"].update(cp_cache_built_at="t1")) == bindings gt = rebound(lambda p: p["fit"]["datasets"]["set-a"].update(gt_matrix_sha256="0" * 64)) - assert gt["set-a"] != bindings["set-a"] and gt["set-a-lite"] != bindings["set-a-lite"] - assert gt["set-b"] == bindings["set-b"] - lite_stamp = rebound(lambda p: p["fit"]["lite"]["set-a-lite"].update(cp_cache_built_at="L1")) - assert lite_stamp["set-a-lite"] != bindings["set-a-lite"] and lite_stamp["set-a"] == bindings["set-a"] + assert gt["set-a"] != bindings["set-a"] + assert gt["set-b"] == bindings["set-b"] and gt["set-a-lite"] == bindings["set-a-lite"] + lite_gt = rebound(lambda p: p["fit"]["lite"]["set-a-lite"].update(gt_matrix_sha256="0" * 64)) + assert lite_gt["set-a-lite"] != bindings["set-a-lite"] and lite_gt["set-a"] == bindings["set-a"] diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline.py b/applications/dynacell/src/dynacell/evaluation/pipeline.py index 3ae5ca1d8..48619f369 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline.py @@ -327,7 +327,10 @@ def _extend_backbone( def _stage_cp_dataset_inputs( - cp: _BackboneLists, cp_space: DatasetCPSpace, n_gt_cells: int, save_dir: Path + cp: _BackboneLists, + cp_space: DatasetCPSpace, + gt_cp_blocks: dict[tuple[str, int], np.ndarray], + save_dir: Path, ) -> tuple[str, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: """Stage the dataset-level CP inputs in the reference feature space. @@ -343,9 +346,10 @@ def _stage_cp_dataset_inputs( Aggregated CP rows of the run. cp_space : DatasetCPSpace Reference bound to the eval's dataset. - n_gt_cells : int - Finite GT CP rows the run scored (before the pairwise pred/GT drop); - must equal the scaler's fit count for a non-lite dataset. + gt_cp_blocks : dict + ``{(position, t): GT-finite CP rows}`` the run scored, before the pairwise + pred/GT non-finite drop; must hash to the reference's recorded GT matrix + (:meth:`DatasetCPSpace.check_gt_cells`) before anything is written. save_dir : pathlib.Path Eval dir receiving the sidecar. @@ -354,7 +358,7 @@ def _stage_cp_dataset_inputs( tuple ``("CP", pred_metric, target_metric, pred_probe, target_probe, pred_fovs, target_fovs)``. """ - cp_space.check_n_cells(n_gt_cells) + cp_space.check_gt_cells(gt_cp_blocks) pred_cp_raw = np.concatenate(cp.pred_feats, axis=0) target_cp_raw = np.concatenate(cp.gt_feats, axis=0) mask_payload = { @@ -404,7 +408,9 @@ class FovResult: dynaclr: _BackboneLists = field(default_factory=_BackboneLists) celldino: _BackboneLists = field(default_factory=_BackboneLists) morphem: _BackboneLists = field(default_factory=_BackboneLists) - n_gt_cp_finite: int = 0 # finite GT CP rows, before the pairwise pred/GT non-finite drop + # GT-finite CP rows per timepoint, before the pairwise pred/GT non-finite drop: the + # cells the dataset-level content gate hashes. + gt_cp_blocks: dict[int, np.ndarray] = field(default_factory=dict) timings: list[tuple[str, int | None, str, float]] = field(default_factory=list) @@ -859,7 +865,7 @@ def _nuclei_seeds(segment_fn, **infer) -> np.ndarray | None: ) microssim_data: list[dict] = [] - n_gt_cp_finite = 0 + gt_cp_blocks: dict[int, np.ndarray] = {} fov_pixel_metrics: list[dict] = [] fov_mask_metrics: list[dict] = [] fov_feature_metrics: list[dict] = [] @@ -951,7 +957,7 @@ def _nuclei_seeds(segment_fn, **infer) -> np.ndarray | None: pred_celldino = pred_per_t["celldino"][t] if pred_per_t["celldino"] is not None else None pred_morphem = pred_per_t["morphem"][t] if pred_per_t["morphem"] is not None else None if gt_cp_per_t[t].size: - n_gt_cp_finite += int(np.isfinite(gt_cp_per_t[t]).all(axis=1).sum()) + gt_cp_blocks[t] = gt_cp_per_t[t][np.isfinite(gt_cp_per_t[t]).all(axis=1)] pred_cp, gt_cp_t = drop_paired_nonfinite_rows(pred_cp, gt_cp_per_t[t]) pred_cp_z, gt_cp_z = _cp_row_features(pred_cp, gt_cp_t, cp_space) # Prefixes come from _COLUMN_PREFIX, not literals: these per-timepoint @@ -1022,7 +1028,7 @@ def _nuclei_seeds(segment_fn, **infer) -> np.ndarray | None: dynaclr=dynaclr, celldino=celldino, morphem=morphem, - n_gt_cp_finite=n_gt_cp_finite, + gt_cp_blocks=gt_cp_blocks, timings=get_timings()[timings_start:], ) @@ -1311,10 +1317,6 @@ def evaluate_predictions( f"cp_space must be given exactly when compute_feature_metrics=true " f"(compute_feature_metrics={config.compute_feature_metrics}, cp_space={cp_space is not None})" ) - # Before any model load or cache write: a GT CP cache re-cached after the reference - # was fit must fail the run up front, not after the per-FOV loop. - if cp_space is not None: - cp_space.check_gt_cache(OmegaConf.select(config, "io.gt_cache_dir", default=None)) use_gpu = bool(getattr(config, "use_gpu", True)) @@ -1353,7 +1355,9 @@ def evaluate_predictions( seg_path = Path(io_config.cell_segmentation_path) if io_config.cell_segmentation_path is not None else None parent_lists: dict[str, _BackboneLists] = {name: _BackboneLists() for name in _BACKBONE_KEYS} - n_gt_cp_finite = [0] # summed across FOVs by _aggregate; checked against the CP scaler's fit + # GT-finite CP rows by (position, t), merged across FOVs by _aggregate; hashed against + # the CP reference fit before any dataset-level metric is computed. + gt_cp_blocks: dict[tuple[str, int], np.ndarray] = {} # Deep-feature extractors by backbone key. Bound at function scope because the # precompute gate and the dataset-metrics block both derive from it and sit in @@ -1625,7 +1629,7 @@ def _keep(name: str) -> bool: extend_worker_timings = runtime.executor == "process" def _aggregate(result: FovResult) -> None: - n_gt_cp_finite[0] += result.n_gt_cp_finite + gt_cp_blocks.update({(result.pos_name, t): rows for t, rows in result.gt_cp_blocks.items()}) _aggregate_fov_result( result, segmentation_results, @@ -1750,9 +1754,7 @@ def _aggregate(result: FovResult) -> None: prefix_inputs: list[tuple[str, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]] = [] if parent_lists["cp"].pred_feats: - prefix_inputs.append( - _stage_cp_dataset_inputs(parent_lists["cp"], cp_space, n_gt_cp_finite[0], save_dir) - ) + prefix_inputs.append(_stage_cp_dataset_inputs(parent_lists["cp"], cp_space, gt_cp_blocks, save_dir)) for key in deep_kinds: # cp is handled above (reference space) bb = parent_lists[key] @@ -1919,9 +1921,6 @@ def _final_metrics_cache_valid(config: DictConfig) -> bool: current_sha256 = None if config.compute_feature_metrics: space = eval_cp_space(config) - # Same GT re-cache guard as the scoring path: cached CP rows scored against a - # GT cache that has since been re-cached must not be reused silently. - space.check_gt_cache(OmegaConf.select(config, "io.gt_cache_dir", default=None)) # The cached rows cover the positions of the run that wrote them; a partial walk # must go through the scoring path so ``check_positions`` sees its GT position set. if config.limit_positions is not None or OmegaConf.select(config, "io.exclude_fov_names", default=None): diff --git a/applications/dynacell/tests/_eval_fixtures.py b/applications/dynacell/tests/_eval_fixtures.py index 116f4f0f5..9955c9c45 100644 --- a/applications/dynacell/tests/_eval_fixtures.py +++ b/applications/dynacell/tests/_eval_fixtures.py @@ -226,17 +226,14 @@ def make_cp_reference( column names :func:`~dynacell.evaluation.cp_reference.cp_space` derives from the same config. It fits one scaler per entry of ``datasets`` over the plate's ``A/1/{i}`` positions; dataset ``i`` is drawn with offset ``10 * i`` and scale - ``1 + i``, so every scaler differs. ``lite`` maps lite dataset -> parent. Every - non-lite dataset records ``config.io.gt_cache_dir`` and a real ``built_at``, and a - matching ``cp_features`` entry (full CP identity) is written into that cache's - manifest, so the eval's re-cache check passes the way it does on real data. Sets - ``feature_metrics.cp`` (GLCM off unless already configured), - ``feature_metrics.cp.reference_path`` and ``benchmark.dataset_ref.dataset`` - (the first dataset), and returns the reference's sha256. + ``1 + i``, so every scaler differs. ``lite`` maps lite dataset -> parent; a lite + set's cells are its parent's first half. Sets ``feature_metrics.cp`` (GLCM off + unless already configured), ``feature_metrics.cp.reference_path`` and + ``benchmark.dataset_ref.dataset`` (the first dataset), and returns the + reference's sha256. """ # Imported here like the other helpers in this file: live_pipeline_module() drops # every dynacell.evaluation.* module, so a top-level binding would go stale. - from dynacell.evaluation.cache import built_at_now, cache_paths, load_manifest, save_manifest from dynacell.evaluation.cp_reference import DatasetFit, cp_space, fit_cp_reference, write_cp_reference if OmegaConf.select(config, "feature_metrics.cp", default=None) is None: @@ -248,38 +245,24 @@ def make_cp_reference( ) OmegaConf.update(config, "benchmark", {"dataset_ref": {"dataset": datasets[0], "target": "sec61b"}}, merge=True) identity, names = cp_space(config) - gt_cache_dir = str(config.io.gt_cache_dir) - paths = cache_paths(gt_cache_dir) - manifest = load_manifest(paths) - built_at = built_at_now() - manifest["artifacts"]["cp_features"] = { - "path": "features/cp.zarr", - "built_at": built_at, - "spacing": list(config.pixel_metrics.spacing), - **identity, - } - save_manifest(paths, manifest) rng = np.random.default_rng(seed) + positions = [f"A/1/{p}" for p in range(N_POSITIONS)] + cells = {name: rng.standard_normal((n_cells, len(names))) * (1 + i) + 10.0 * i for i, name in enumerate(datasets)} fits = [ + DatasetFit(dataset=name, cells=c, record={"positions": positions}, in_mask_fit=True) + for name, c in cells.items() + ] + fits += [ DatasetFit( dataset=name, - cells=rng.standard_normal((n_cells, len(names))) * (1 + i) + 10.0 * i, - record={ - "positions": [f"A/1/{p}" for p in range(N_POSITIONS)], - "gt_cache_dir": gt_cache_dir, - "cp_cache_built_at": built_at, - }, - in_mask_fit=True, + cells=cells[parent][: n_cells // 2], + record={"positions": positions}, + in_mask_fit=False, + parent=parent, ) - for i, name in enumerate(datasets) - ] - lite_entries = { - name: {"parent": parent, "gt_cache_dir": None, "cp_cache_built_at": None} for name, parent in (lite or {}).items() - } - payload = fit_cp_reference( - fits, target_name=config.target_name, feature_names=names, cp_identity=identity, lite=lite_entries - ) + ] + payload = fit_cp_reference(fits, target_name=config.target_name, feature_names=names, cp_identity=identity) write_cp_reference(payload, path, force=True) config.feature_metrics.cp.reference_path = str(path) return payload["sha256"] diff --git a/applications/dynacell/tests/test_cross_condition_probe.py b/applications/dynacell/tests/test_cross_condition_probe.py index 4ba7f3a8f..67984c31a 100644 --- a/applications/dynacell/tests/test_cross_condition_probe.py +++ b/applications/dynacell/tests/test_cross_condition_probe.py @@ -35,7 +35,7 @@ def _write_reference(path: Path, seed: int = 0) -> str: in_mask_fit=True, ) payload = fit_cp_reference( - [fit], target_name="membrane", feature_names=tuple(f"f{i}" for i in range(_DIM)), cp_identity={}, lite={} + [fit], target_name="membrane", feature_names=tuple(f"f{i}" for i in range(_DIM)), cp_identity={} ) write_cp_reference(payload, path, force=True) return payload["sha256"] diff --git a/applications/dynacell/tests/test_evaluation_cp_end_to_end.py b/applications/dynacell/tests/test_evaluation_cp_end_to_end.py new file mode 100644 index 000000000..88aa6fb19 --- /dev/null +++ b/applications/dynacell/tests/test_evaluation_cp_end_to_end.py @@ -0,0 +1,172 @@ +"""End-to-end CP (GLCM+) scoring through the real ``evaluate_predictions``. + +Tiny HCS plates (3 positions x 2 timepoints) carry a GT volume, a cell +segmentation with 16 textured cells plus one single-voxel cell (whose CP row is +non-finite), and a prediction derived from the GT. CP regionprops, the GT/pred CP +caches, the CP reference, pixel and mask metrics all run for real; only the deep +extractors are stubbed (``fov_deep_features`` returns seeded random rows, one per +cell), since they are not what these tests are about. + +The reference is fit by the production builder path on the GT CP cache the first +eval step writes, so the eval's content gate compares two independently collected +views of the same GT cells. +""" + +from __future__ import annotations + +import zlib +from pathlib import Path + +import numpy as np +import pytest +from iohub.ngff import open_ome_zarr + +from dynacell.evaluation.cache import cache_paths, open_features_group, write_features_to_group +from dynacell.evaluation.model_loader import EvalModels + +from ._eval_fixtures import N_POSITIONS, D, H, T, W, build_eval_config, live_pipeline_module, make_mask_cache + +_DATASET = "a549-mantis-sec61b-mock" +_POSITIONS = [f"A/1/{i}" for i in range(N_POSITIONS)] + + +def _segmentation() -> np.ndarray: + """``(T, D, H, W)`` labels: a 4 x 4 grid of 6 x 6 x 6 cells plus one single-voxel cell.""" + seg = np.zeros((T, D, H, W), dtype=np.int32) + label = 1 + for row in range(4): + for col in range(4): + y, x = 1 + 8 * row, 1 + 8 * col + seg[:, 2:8, y : y + 6, x : x + 6] = label + label += 1 + seg[:, 9, 0, 0] = label # one voxel: skewness/kurtosis are NaN -> a non-finite GT row + return seg + + +def _gt_volume(seed: int) -> np.ndarray: + """``(T, D, H, W)`` GT: uniform background, cells with per-cell texture inside the background range.""" + rng = np.random.default_rng(seed) + gt = rng.uniform(0.0, 1.0, size=(T, D, H, W)) + seg = _segmentation() + for label in range(1, 17): + inside = seg == label + gt[inside] = rng.uniform(0.25, 0.75) + 0.2 * rng.standard_normal(inside.sum()) + return gt.astype(np.float32) + + +def _write_plate(path: Path, channel: str, volumes: list[np.ndarray], dtype=np.float32) -> None: + with open_ome_zarr(path, mode="w", layout="hcs", channel_names=[channel], version="0.5") as plate: + for i, vol in enumerate(volumes): + plate.create_position("A", "1", str(i)).create_image("0", vol[:, None].astype(dtype)) + + +def _deep_stub(ctx, pos_name, image, cell_segmentation, extractor, kind, z_slabs=None): + """One seeded random 8-d row per cell and timepoint, identical for GT and pred.""" + rng = np.random.default_rng(zlib.crc32(f"{pos_name}/{kind}".encode())) + return [rng.standard_normal((len(np.unique(cell_segmentation[t])) - 1, 8)) for t in range(image.shape[0])] + + +def _stub_models() -> EvalModels: + return EvalModels( + seg_model=None, + dinov3=object(), + dynaclr=object(), + celldino=None, + morphem=None, + dinov3_model_name=None, + dynaclr_ckpt_path=None, + dynaclr_encoder_cfg=None, + celldino_weights_path=None, + morphem_model_name=None, + ) + + +class Harness: + """GT/seg plates, mask caches, a CP reference fit on the GT CP cache, and a runner.""" + + def __init__(self, root: Path, monkeypatch): + self.root = root + self.pipeline = live_pipeline_module() + from dynacell.evaluation import cp_reference, pipeline_cache + + self.cp_reference = cp_reference + self.gt = [_gt_volume(seed=i) for i in range(N_POSITIONS)] + self.seg = _segmentation() + _write_plate(root / "gt.zarr", "target", self.gt) + _write_plate(root / "seg.zarr", "cell_segmentation", [self.seg] * N_POSITIONS, dtype=np.int32) + make_mask_cache(root / "gt_cache", root / "gt.zarr", "target", side="gt") + monkeypatch.setattr(self.pipeline, "fov_deep_features", _deep_stub) + monkeypatch.setattr(self.pipeline, "precompute_deep_features", lambda *a, **k: None) + + # GT CP cache, written by the pipeline's own cache writer, then the reference from it. + config = self.config(root / "unused_pred.zarr", root / "unused_out") + ctx = pipeline_cache.init_cache_context(config, side="gt") + for name, vol in zip(_POSITIONS, self.gt, strict=True): + pipeline_cache.fov_cp_features(ctx, name, vol, self.seg) + pipeline_cache.flush_manifest(ctx) + identity, names = cp_reference.cp_space(config) + cells, record = cp_reference.read_gt_cp_cells(ctx, root / "gt.zarr", len(names)) + fit = cp_reference.DatasetFit(dataset=_DATASET, cells=cells, record=record, in_mask_fit=True) + payload = cp_reference.fit_cp_reference([fit], target_name="er", feature_names=names, cp_identity=identity) + cp_reference.write_cp_reference(payload, root / "er.json") + + def config(self, pred_path: Path, save_dir: Path): + config = build_eval_config( + pred_path, + self.root / "gt.zarr", + self.root / "gt_cache", + save_dir.parent / f"{save_dir.name}_pred_cache", + save_dir, + executor="serial", + fov_workers=1, + ) + config.io.require_complete_cache = False + config.io.cell_segmentation_path = str(self.root / "seg.zarr") + config.compute_feature_metrics = True + config.feature_metrics.compute_fid = False + config.feature_metrics.compute_prc = False + config.feature_metrics.compute_mind = False + config.feature_metrics.cp = { + "norm": {"p_lo": 1.0, "p_hi": 99.0}, + "glcm": {"enabled": False}, + "reference_path": str(self.root / "er.json"), + } + config.benchmark = {"dataset_ref": {"dataset": _DATASET, "target": "sec61b"}} + return config + + def run(self, name: str, pred: list[np.ndarray], **overrides) -> tuple[dict, object]: + """Evaluate ``pred`` against the GT; return the dataset-level feature row and the config.""" + pred_path = self.root / f"{name}.zarr" + _write_plate(pred_path, "prediction", pred) + config = self.config(pred_path, self.root / name) + make_mask_cache(Path(config.io.pred_cache_dir), pred_path, "prediction", side="pred") + for key, value in overrides.items(): + config.force_recompute[key] = value + _, _, feature_rows = self.pipeline.evaluate_predictions( + config, models=_stub_models(), cp_space=self.pipeline.eval_cp_space(config) + ) + return feature_rows[0], config + + +@pytest.fixture +def harness(tmp_path: Path, monkeypatch) -> Harness: + return Harness(tmp_path, monkeypatch) + + +def test_identical_gt_recache_passes_the_content_gate(harness: Harness) -> None: + """``force_recompute.gt_cp`` rewrites the GT CP cache (new built_at) with identical cells: scoring proceeds.""" + row, _ = harness.run("recache", [g.copy() for g in harness.gt], gt_cp=True) + assert np.isfinite(row["Dataset_CP_KID"]) + + +def test_same_count_gt_value_change_is_refused(harness: Harness) -> None: + """A GT CP cache whose values moved (same cell count) since the fit fails before any metric is written.""" + with open_features_group(cache_paths(harness.root / "gt_cache"), "cp", mode="a") as group: + feats = np.asarray(group["A/1/1/t1"]) + feats[0, 0] += 0.5 + write_features_to_group(group, "A/1/1", 1, feats) + save_dir = harness.root / "moved" + with pytest.raises(Exception, match="differ from the CP reference fit") as err: + harness.run("moved", [g.copy() for g in harness.gt]) + assert type(err.value).__name__ == "StaleCacheError" + assert not list(save_dir.glob("*_metrics.*")) and not (save_dir / "cp_selected_feature_mask.json").exists() diff --git a/applications/dynacell/tests/test_evaluation_pipeline_parallel.py b/applications/dynacell/tests/test_evaluation_pipeline_parallel.py index f414a8656..35621aadf 100644 --- a/applications/dynacell/tests/test_evaluation_pipeline_parallel.py +++ b/applications/dynacell/tests/test_evaluation_pipeline_parallel.py @@ -209,7 +209,7 @@ def test_worker_run_fov_hands_the_cp_space_to_process_one_fov(tmp_path, monkeypa in_mask_fit=True, ) write_cp_reference( - fit_cp_reference([fit], target_name="er", feature_names=names, cp_identity={}, lite={}), tmp_path / "er.json" + fit_cp_reference([fit], target_name="er", feature_names=names, cp_identity={}), tmp_path / "er.json" ) cp_space = pickle.loads(pickle.dumps(load_cp_reference(tmp_path / "er.json", target_name="er").for_dataset("ds"))) diff --git a/applications/dynacell/tests/test_evaluation_timepoint_guard.py b/applications/dynacell/tests/test_evaluation_timepoint_guard.py index 33ac2554a..ee613cb38 100644 --- a/applications/dynacell/tests/test_evaluation_timepoint_guard.py +++ b/applications/dynacell/tests/test_evaluation_timepoint_guard.py @@ -18,7 +18,6 @@ from iohub.ngff import open_ome_zarr from omegaconf import OmegaConf -from dynacell.evaluation.cache import cache_paths, load_manifest, save_manifest from dynacell.evaluation.model_loader import EvalModels from dynacell.evaluation.provenance import PROVENANCE_FILENAME, write_metrics_provenance @@ -205,22 +204,6 @@ def test_cache_reuse_refuses_a_gt_store_extended_since_the_fit(tmp_path: Path) - pipeline._final_metrics_cache_valid(config) -def test_cache_reuse_refuses_a_gt_recache(tmp_path: Path) -> None: - """A reusable final-metrics cache is refused once the GT CP cache was re-cached after the build.""" - pipeline = live_pipeline_module() - config, _ = _feature_cache_config(tmp_path) # records the cache's real built_at - _write_final_caches(pipeline, tmp_path, _dataset_row(("CP", "DINOv3", "DynaCLR"), _ALL_FAMILIES), config) - assert pipeline._final_metrics_cache_valid(config) - - paths = cache_paths(tmp_path / "gt_cache") - manifest = load_manifest(paths) - manifest["artifacts"]["cp_features"]["built_at"] = "2099-01-01T00:00:00+00:00" # re-cached after the build - save_manifest(paths, manifest) - with pytest.raises(Exception, match="was built at 2099-01-01") as err: - pipeline._final_metrics_cache_valid(config) - assert type(err.value).__name__ == "StaleCacheError" - - def _two_set_cache(pipeline, tmp_path: Path): """A final-metrics cache scored for set-a of a two-set reference; returns ``(config, reference path)``.""" config, _ = _feature_cache_config(tmp_path) @@ -251,17 +234,9 @@ def test_cache_is_invalid_once_the_gt_matrix_changed(tmp_path: Path) -> None: def test_cache_stays_valid_after_a_harmless_rebuild(tmp_path: Path) -> None: - """A rebuild that only moves built_at (same GT matrix) keeps the cache reusable. - - check_gt_cache passes because the rebuild records the cache's new built_at, and - the binding is unchanged because the GT-matrix sha256 is. - """ + """A rebuild that only moves the audit-only built_at (same GT matrix) keeps the cache reusable.""" pipeline = live_pipeline_module() config, reference = _two_set_cache(pipeline, tmp_path) - paths = cache_paths(tmp_path / "gt_cache") - manifest = load_manifest(paths) - manifest["artifacts"]["cp_features"]["built_at"] = "2099-01-01T00:00:00+00:00" - save_manifest(paths, manifest) payload = json.loads(reference.read_text()) for record in payload["fit"]["datasets"].values(): record["cp_cache_built_at"] = "2099-01-01T00:00:00+00:00" @@ -375,20 +350,5 @@ def test_partial_position_walk_is_refused_for_cp(tmp_path: Path, monkeypatch) -> config.limit_positions = 2 monkeypatch.setattr(pipeline, "_calibrate_microssim", lambda *a, **k: pytest.fail("reached per-FOV work")) cp_space = pipeline.eval_cp_space(config) - with pytest.raises(ValueError, match="cannot be combined with compute_feature_metrics"): + with pytest.raises(ValueError, match="compute_feature_metrics=false for smoke runs"): pipeline.evaluate_predictions(config, models=_stub_models(), cp_space=cp_space) - - -def test_gt_recache_after_the_reference_is_refused_up_front(tmp_path: Path, monkeypatch) -> None: - """A GT CP cache whose ``built_at`` moved since the build fails before any model load.""" - pipeline = live_pipeline_module() - config, _ = _feature_cache_config(tmp_path) # records the cache's real built_at - paths = cache_paths(tmp_path / "gt_cache") - manifest = load_manifest(paths) - manifest["artifacts"]["cp_features"]["built_at"] = "2099-01-01T00:00:00+00:00" # re-cached after the build - save_manifest(paths, manifest) - monkeypatch.setattr(pipeline, "load_eval_models", lambda *a, **k: pytest.fail("models loaded")) - cp_space = pipeline.eval_cp_space(config) - with pytest.raises(Exception, match="was built at 2099-01-01") as err: - pipeline.evaluate_predictions(config, cp_space=cp_space) - assert type(err.value).__name__ == "StaleCacheError" diff --git a/applications/dynacell/tools/build_cp_reference.py b/applications/dynacell/tools/build_cp_reference.py index 832f39064..be6d3b13b 100644 --- a/applications/dynacell/tools/build_cp_reference.py +++ b/applications/dynacell/tools/build_cp_reference.py @@ -9,24 +9,27 @@ evaluates on (``benchmark.dataset_ref.dataset`` under ``--leaves-root``), HEK included (its own scaler, outside the mask fit); * lite datasets mapped to their parent's scaler, the parent read from the lite - split's ``selection_criteria.lite_of``. No scaler is fit on lite cells. + split's ``selection_criteria.lite_of``. No scaler is fit on lite cells, but their + positions, count and GT-matrix sha256 are recorded from the lite GT caches so the + eval's content gate covers lite evals too. This tool only READS. Each dataset ref is resolved exactly as an eval resolves it (``apply_dataset_ref`` on the packaged ``eval.yaml`` defaults), the GT cache is opened through the pipeline's own ``init_cache_context`` with ``require_complete_cache=true`` (so a cache of another store or CP recipe is refused), and every ``(position, timepoint)`` of the GT store must already be in -the CP cache. A miss raises; features are never computed. Non-finite cells are -dropped the way the pipeline drops them (``drop_paired_nonfinite_rows``). +the CP cache. A miss raises; features are never computed. Non-finite GT cells are +dropped and the rest stacked in the canonical order the eval hashes +(``cp_reference.read_gt_cp_cells`` / ``canonical_gt_matrix``). The write is atomic, and an existing reference with a different content hash is only replaced under ``--force`` (an identical one is left untouched). -``--verify`` re-reads each non-lite dataset's GT CP cells from the current caches -and compares their sha256 and count with the ones recorded in the reference's -``fit`` section; it exits 1 on any mismatch. That is the only check that catches a -value-only GT re-cache (same cell count), which the eval's ``built_at`` and -cell-count checks cannot see. +``--verify`` re-reads every dataset's GT CP cells from the current caches and +compares their sha256 and count with the ones recorded in the reference's ``fit`` +section; it exits 1 on any mismatch. It is the offline form of the eval's content +gate, which refuses to score a dataset whose staged GT cells no longer hash to the +recorded value. Run:: @@ -43,9 +46,7 @@ from pathlib import Path from typing import Any -import numpy as np import yaml -from iohub.ngff import open_ome_zarr from omegaconf import DictConfig, OmegaConf import dynacell.evaluation @@ -53,13 +54,6 @@ from dynacell.data.manifests import load_manifest as load_dataset_manifest from dynacell.data.resolver import resolve_dataset_ref from dynacell.evaluation._ref_hook import apply_dataset_ref -from dynacell.evaluation.cache import ( - StaleCacheError, - cache_paths, - load_manifest, - open_features_group, - read_features_from_group, -) from dynacell.evaluation.cp_reference import ( MASK_FIT_DATASETS, DatasetFit, @@ -67,9 +61,9 @@ fit_cp_reference, gt_matrix_sha256, load_cp_reference, + read_gt_cp_cells, write_cp_reference, ) -from dynacell.evaluation.metrics import drop_paired_nonfinite_rows from dynacell.evaluation.paths import cp_reference_path from dynacell.evaluation.pipeline_cache import init_cache_context @@ -123,84 +117,38 @@ def lite_parent(ref: Ref) -> str | None: return (splits.selection_criteria or {}).get("lite_of") -def _gt_positions(config: DictConfig) -> dict[str, tuple[int, int]]: - """Return ``{position: (T, Z)}`` of the eval's GT store, metadata only.""" - with open_ome_zarr(Path(config.io.gt_path), mode="r") as plate: - return {name: (int(pos.data.shape[0]), int(pos.data.shape[2])) for name, pos in plate.positions()} - +def read_dataset_fit( + target_name: str, ref: Ref, n_features: int, in_mask_fit: bool, parent: str | None = None +) -> DatasetFit: + """Read every finite GT CP cell of one dataset from its cache, read-only, in canonical order. -def read_dataset_fit(target_name: str, ref: Ref, n_features: int, in_mask_fit: bool) -> DatasetFit: - """Read every finite GT CP cell of one dataset from its cache, read-only. + The cells come from :func:`~dynacell.evaluation.cp_reference.read_gt_cp_cells`, + the ordering the eval's content gate hashes in. A lite dataset (``parent`` set) + is read the same way from its own GT cache, so its cells are recorded too. Raises ------ StaleCacheError - If the cache belongs to another store or CP recipe, records no - ``cp_features`` entry, a ``(position, timepoint)`` of the GT store is - missing from it, or an entry has the wrong column count. + If the cache belongs to another store or CP recipe, a ``(position, + timepoint)`` of the GT store is missing from it, or an entry has the wrong + column count. ValueError If a GT position is not a 3-D volume. """ config = eval_config_for(target_name, ref) ctx = init_cache_context(config, side="gt") + cells, read = read_gt_cp_cells(ctx, Path(config.io.gt_path), n_features) entry = ctx.manifest["artifacts"].get("cp_features") - if entry is None: - raise StaleCacheError(f"{ref[0]}: GT cache {ctx.paths.root} records no cp_features artifact") - positions = _gt_positions(config) - blocks: list[np.ndarray] = [] - n_timepoints = n_dropped = 0 - with open_features_group(ctx.paths, "cp", mode="r") as group: - if group is None: - raise StaleCacheError(f"no CP feature cache at {ctx.paths.cp_features()}") - for pos_name, (t_count, z_depth) in positions.items(): - if z_depth < 2: - raise ValueError(f"{config.io.gt_path}/{pos_name} has Z={z_depth}; CP regionprops are 3-D") - for t in range(t_count): - feats = read_features_from_group(group, pos_name, t) - if feats is None: - raise StaleCacheError(f"CP cache miss at {pos_name}/t{t} in {ctx.paths.cp_features()}") - n_timepoints += 1 - if feats.shape[0] == 0: - continue - if feats.shape[1] != n_features: - raise StaleCacheError(f"{pos_name}/t{t} has {feats.shape[1]} CP columns, expected {n_features}") - finite, _ = drop_paired_nonfinite_rows(feats, feats) - n_dropped += feats.shape[0] - finite.shape[0] - blocks.append(np.asarray(finite, dtype=np.float64)) - cells = np.concatenate(blocks, axis=0) if blocks else np.empty((0, n_features)) record = { "target": ref[1], "gt_path": str(config.io.gt_path), "gt_cache_dir": str(ctx.paths.root), "cp_cache_path": str(ctx.paths.cp_features()), - "cp_cache_built_at": entry["built_at"], - "positions": sorted(positions), - "n_timepoints": n_timepoints, - "n_cells_dropped_nonfinite": int(n_dropped), - } - return DatasetFit(dataset=ref[0], cells=cells, record=record, in_mask_fit=in_mask_fit) - - -def lite_entry(target_name: str, ref: Ref, parent: str, parent_positions: list[str]) -> dict[str, Any]: - """Return a lite dataset's reference entry: its parent, and its own GT cache stamp if one exists. - - Raises - ------ - ValueError - If the lite GT store holds a position the parent's fit does not. - """ - config = eval_config_for(target_name, ref) - extra = set(_gt_positions(config)) - set(parent_positions) - if extra: - raise ValueError(f"lite {ref[0]} has positions outside its parent {parent}: {sorted(extra)[:5]}") - gt_cache_dir = str(config.io.gt_cache_dir) - entry = load_manifest(cache_paths(gt_cache_dir))["artifacts"].get("cp_features") - return { - "parent": parent, - "target": ref[1], - "gt_cache_dir": gt_cache_dir, + # Audit only: the eval gates on the cells' sha256, never on this stamp. "cp_cache_built_at": entry["built_at"] if entry is not None else None, + **read, } + return DatasetFit(dataset=ref[0], cells=cells, record=record, in_mask_fit=in_mask_fit, parent=parent) def build(target_name: str, leaves_root: Path) -> dict[str, Any]: @@ -209,7 +157,8 @@ def build(target_name: str, leaves_root: Path) -> dict[str, Any]: Raises ------ ValueError - If the datasets resolve different CP recipes, or a lite parent has no scaler. + If the datasets resolve different CP recipes, a lite parent has no scaler, + or a lite store holds a position its parent's does not. """ mask_refs = set(MASK_FIT_DATASETS[target_name]) leaf_refs = leaf_dataset_refs(target_name, leaves_root) @@ -225,18 +174,21 @@ def build(target_name: str, leaves_root: Path) -> dict[str, Any]: raise ValueError(f"{ref[0]} resolves a different CP recipe than {scaler_refs[0][0]}") fits = [read_dataset_fit(target_name, ref, len(names), ref in mask_refs) for ref in scaler_refs] - positions = {fit.dataset: fit.record["positions"] for fit in fits} - lite: dict[str, dict[str, Any]] = {} + positions = {fit.dataset: set(fit.record["positions"]) for fit in fits} for ref in lite_refs: parent = parents[ref] if parent not in positions: raise ValueError(f"lite {ref[0]} reuses {parent}, which no grouped leaf or mask fit evaluates") - lite[ref[0]] = lite_entry(target_name, ref, parent, positions[parent]) - return fit_cp_reference(fits, target_name=target_name, feature_names=names, cp_identity=identity, lite=lite) + lite = read_dataset_fit(target_name, ref, len(names), False, parent=parent) + extra = set(lite.record["positions"]) - positions[parent] + if extra: + raise ValueError(f"lite {ref[0]} has positions outside its parent {parent}: {sorted(extra)[:5]}") + fits.append(lite) + return fit_cp_reference(fits, target_name=target_name, feature_names=names, cp_identity=identity) def verify(target_name: str, path: Path) -> list[str]: - """Recompute every non-lite dataset's GT-matrix sha256 from the caches and compare. + """Recompute every dataset's (lite included) GT-matrix sha256 from the caches and compare. Parameters ---------- @@ -253,7 +205,7 @@ def verify(target_name: str, path: Path) -> list[str]: """ ref = load_cp_reference(path, target_name=target_name) mismatches = [] - for name, record in sorted(ref.fit["datasets"].items()): + for name, record in sorted({**ref.fit["datasets"], **ref.fit["lite"]}.items()): fit = read_dataset_fit(target_name, (name, record["target"]), len(ref.feature_names), record["in_mask_fit"]) sha256 = gt_matrix_sha256(fit.cells) ok = sha256 == record["gt_matrix_sha256"] and fit.cells.shape[0] == record["n_cells"] @@ -281,8 +233,11 @@ def _summary(payload: dict[str, Any]) -> str: f"dropped), floored {scaler['floored_features'] or 'none'}" ) for name, entry in sorted(payload["lite"].items()): - built_at = fit["lite"][name]["cp_cache_built_at"] - lines.append(f" {name} -> scaler of {entry['parent']} (own cache built_at {built_at})") + d = fit["lite"][name] + lines.append( + f" {name} -> scaler of {entry['parent']}: {len(d['positions'])} positions, {d['n_cells']} cells, " + f"sha256 {d['gt_matrix_sha256'][:12]}" + ) return "\n".join(lines) diff --git a/applications/dynacell/tools/build_cp_reference_test.py b/applications/dynacell/tools/build_cp_reference_test.py index 0d0e42b24..c78ddea9a 100644 --- a/applications/dynacell/tools/build_cp_reference_test.py +++ b/applications/dynacell/tools/build_cp_reference_test.py @@ -48,7 +48,6 @@ def _dataset( skip: tuple[str, int] | None = None, lite_of: str | None = None, positions: tuple[str, ...] = _POSITIONS, - cache: bool = True, ) -> None: """Register one dataset: manifest, split, GT store, and a GT CP cache holding every (pos, t).""" store = root / "stores" / f"{name}.zarr" @@ -86,8 +85,6 @@ def _dataset( (root / "manifests" / name / "splits").mkdir(parents=True) (root / "manifests" / name / "manifest.yaml").write_text(yaml.dump(manifest)) (root / "manifests" / name / "splits" / "sec61b.yaml").write_text(yaml.dump(split)) - if not cache: - return rng = np.random.default_rng(seed) with open_features_group(cache_paths(cache_dir), "cp", mode="a") as group: for pos_name in positions: @@ -141,7 +138,8 @@ def _build_all(root: Path) -> Path: _dataset(root, "ds-a", seed=0) _dataset(root, "ds-b", seed=1) _dataset(root, "ds-h", seed=2) - _dataset(root, "ds-a-lite", seed=0, lite_of="ds-a", positions=_POSITIONS[:1], cache=False) + # Same seed + first position: the lite cache holds exactly ds-a's A/1/0 cells, as a real lite subset does. + _dataset(root, "ds-a-lite", seed=0, lite_of="ds-a", positions=_POSITIONS[:1]) return _leaves(root, ["ds-a", "ds-b", "ds-h", "ds-a-lite"]) @@ -158,13 +156,10 @@ def test_build_writes_a_loadable_reference(registry: Path) -> None: assert sorted(payload["scalers"]) == ["ds-a", "ds-b", "ds-h"] assert fit["datasets"]["ds-h"]["in_mask_fit"] is False assert payload["lite"] == {"ds-a-lite": {"parent": "ds-a"}} - assert fit["lite"] == { - "ds-a-lite": { - "target": "sec61b", - "gt_cache_dir": str(registry / "caches" / "ds-a-lite"), - "cp_cache_built_at": None, - } - } + lite = fit["lite"]["ds-a-lite"] + assert lite["gt_cache_dir"] == str(registry / "caches" / "ds-a-lite") + assert lite["positions"] == ["A/1/0"] and lite["n_cells"] == _T * 5 - 1 # lite GT recorded from its own cache + assert lite["gt_matrix_sha256"] != fit["datasets"]["ds-a"]["gt_matrix_sha256"] a = {**payload["scalers"]["ds-a"], **fit["datasets"]["ds-a"]} assert a["n_cells"] == 2 * _T * 5 - 1 # one non-finite cell dropped assert a["n_cells_dropped_nonfinite"] == 1 @@ -199,7 +194,7 @@ def test_dry_run_writes_nothing(registry: Path, capsys) -> None: main(["--target", "er", "--leaves-root", str(leaves), "--out", str(out), "--dry-run"]) assert not out.exists() printed = capsys.readouterr().out - assert "mask fit: 38 cells" in printed and "ds-a-lite -> scaler of ds-a" in printed + assert "mask fit: 38 cells" in printed and "ds-a-lite -> scaler of ds-a: 1 positions, 9 cells" in printed def test_missing_timepoint_raises(registry: Path) -> None: @@ -228,7 +223,7 @@ def test_lite_outside_its_parent_is_refused(registry: Path) -> None: """A lite store holding a position its parent's scaler was not fit on is refused at build time.""" _dataset(registry, "ds-a", seed=0) _dataset(registry, "ds-b", seed=1) - _dataset(registry, "ds-a-lite", seed=0, lite_of="ds-a", positions=("B/1/0",), cache=False) + _dataset(registry, "ds-a-lite", seed=0, lite_of="ds-a", positions=("B/1/0",)) leaves = _leaves(registry, ["ds-a", "ds-b", "ds-a-lite"]) with pytest.raises(ValueError, match="outside its parent"): main(["--target", "er", "--leaves-root", str(leaves), "--out", str(registry / "x.json")]) diff --git a/applications/dynacell/tools/lite_checks_test.py b/applications/dynacell/tools/lite_checks_test.py index 445696644..f42681f9d 100644 --- a/applications/dynacell/tools/lite_checks_test.py +++ b/applications/dynacell/tools/lite_checks_test.py @@ -107,7 +107,7 @@ def test_check_e_scores_cp_in_the_reference_space(tmp_path: Path) -> None: in_mask_fit=True, ) payload = fit_cp_reference( - [fit], target_name="nucleus", feature_names=tuple(f"f{i}" for i in range(d)), cp_identity={}, lite={} + [fit], target_name="nucleus", feature_names=tuple(f"f{i}" for i in range(d)), cp_identity={} ) reference = tmp_path / "nucleus.json" write_cp_reference(payload, reference) diff --git a/applications/dynacell/tools/lite_subset_sim_test.py b/applications/dynacell/tools/lite_subset_sim_test.py index c0c27c5fc..d1ac91487 100644 --- a/applications/dynacell/tools/lite_subset_sim_test.py +++ b/applications/dynacell/tools/lite_subset_sim_test.py @@ -48,7 +48,7 @@ def _write_reference(path: Path) -> dict: in_mask_fit=True, ) payload = fit_cp_reference( - [fit], target_name="nucleus", feature_names=tuple(f"f{i}" for i in range(_D_CP)), cp_identity={}, lite={} + [fit], target_name="nucleus", feature_names=tuple(f"f{i}" for i in range(_D_CP)), cp_identity={} ) write_cp_reference(payload, path) return payload From d25ee667ec06b85bd2752cbd839c24663b40541a Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 20:30:20 -0700 Subject: [PATCH 20/41] fix(dynacell/eval): check CP cache columns by name against the reference The reference masks and scales CP columns by position, but the CP cache recorded only a column count. A feature reorder without a CP_FEATURE_VERSION bump would silently misalign both mask and scaler (blocker 4 of the #499 review). fov_cp_features now records cp_feature_names in the manifest's cp_features entry. The eval (for GT and pred caches, before the FOV loop) and the builder compare them with the reference's feature_names and raise StaleCacheError on any difference. For caches written before this change, the names are read back exactly from the recipe they already record (cp_feature_version + cp_glcm_enabled) through a frozen CP_FEATURE_NAMES_BY_VERSION table. So existing caches need no re-cache, and a later reorder of the live tuples still fails, both in that table's test and against old caches. The names are not part of the cache identity, so adding them invalidates nothing. A side being recomputed (force_recompute._cp) is not checked. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/metrics.py | 35 ++++++++ .../src/dynacell/evaluation/pipeline.py | 6 ++ .../src/dynacell/evaluation/pipeline_cache.py | 69 +++++++++++++++ .../dynacell/tests/test_cp_feature_names.py | 85 +++++++++++++++++++ .../tests/test_evaluation_cp_end_to_end.py | 20 ++++- .../dynacell/tools/build_cp_reference.py | 19 +++-- .../dynacell/tools/build_cp_reference_test.py | 16 ++++ 7 files changed, 240 insertions(+), 10 deletions(-) create mode 100644 applications/dynacell/tests/test_cp_feature_names.py diff --git a/applications/dynacell/src/dynacell/evaluation/metrics.py b/applications/dynacell/src/dynacell/evaluation/metrics.py index 440b446b4..1446cbf31 100644 --- a/applications/dynacell/src/dynacell/evaluation/metrics.py +++ b/applications/dynacell/src/dynacell/evaluation/metrics.py @@ -452,6 +452,41 @@ def _prop(regionmask, intensity): # :func:`pipeline_cache._auto_invalidate_on_artifact_param_mismatch`. CP_FEATURE_VERSION = "v2_dist_texture" +#: The CP column order each recipe version wrote, frozen as literals (GLCM on; with +#: GLCM off the ``glcm_*`` columns are absent). CP caches written before the feature +#: names were recorded in the manifest carry only ``cp_feature_version`` + +#: ``cp_glcm_enabled``, and this table reads their exact column order back from +#: those -- no re-cache needed. It is deliberately NOT derived from +#: :func:`active_cp_feature_names`: a reorder of the live tuples without a version +#: bump then disagrees with this table (and fails its test), instead of +#: silently re-labelling old caches (see tests/test_cp_feature_names.py). +CP_FEATURE_NAMES_BY_VERSION: dict[str, tuple[str, ...]] = { + "v2_dist_texture": ( + "intensity_mean", + "intensity_std", + "intensity_min", + "intensity_max", + "p10", + "p25", + "p50", + "p75", + "p90", + "iqr", + "skewness", + "kurtosis", + "gradient_mean", + "gradient_std", + "laplacian_var", + "glcm_contrast", + "glcm_dissimilarity", + "glcm_homogeneity", + "glcm_ASM", + "glcm_energy", + "glcm_correlation", + "glcm_entropy", + ), +} + def active_cp_feature_names(glcm_enabled: bool) -> tuple[str, ...]: """Return the ordered CP column names for the active config. diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline.py b/applications/dynacell/src/dynacell/evaluation/pipeline.py index 48619f369..b6ec9d572 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline.py @@ -39,6 +39,7 @@ ) from dynacell.evaluation.model_loader import EvalModels, init_cache_contexts, load_eval_models from dynacell.evaluation.pipeline_cache import ( + check_cp_cache_feature_names, cpdino_infer_kwargs, flush_manifest, fov_cp_features, @@ -1351,6 +1352,11 @@ def evaluate_predictions( morphem_feature_extractor = models.morphem cache_ctx, pred_cache_ctx = init_cache_contexts(config, models) + # The CP reference masks and scales by column position: refuse a GT or pred CP + # cache whose columns are not the reference's, by name and order. + if cp_space is not None: + for ctx in (cache_ctx, pred_cache_ctx): + check_cp_cache_feature_names(ctx, cp_space.feature_names) seg_path = Path(io_config.cell_segmentation_path) if io_config.cell_segmentation_path is not None else None diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline_cache.py b/applications/dynacell/src/dynacell/evaluation/pipeline_cache.py index 0bd53ff73..b372f2f6d 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline_cache.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline_cache.py @@ -44,7 +44,9 @@ ) from dynacell.evaluation.instance_metrics import DEFAULT_IOU_THRESHOLDS from dynacell.evaluation.metrics import ( + CP_FEATURE_NAMES_BY_VERSION, CP_FEATURE_VERSION, + active_cp_feature_names, build_crops, cp_regionprops, deep_features, @@ -1346,6 +1348,69 @@ def _load_or_compute_feature_timepoints( return per_t, manifest_updated # type: ignore[return-value] +def cached_cp_feature_names(entry: dict[str, Any]) -> tuple[str, ...]: + """Return the column names of a cached CP artifact, from its manifest entry. + + Entries written since the names were recorded carry ``cp_feature_names``. + Older ones are read back exactly from the recipe they record + (``cp_feature_version`` + ``cp_glcm_enabled``) through the frozen + :data:`~dynacell.evaluation.metrics.CP_FEATURE_NAMES_BY_VERSION` table, so + existing caches need no re-cache. + + Parameters + ---------- + entry : dict + The manifest's ``artifacts.cp_features`` entry. + + Returns + ------- + tuple of str + Column names in cache order. + + Raises + ------ + StaleCacheError + If the entry records neither names nor a known recipe version. + """ + if "cp_feature_names" in entry: + return tuple(entry["cp_feature_names"]) + version = entry.get("cp_feature_version") + if version not in CP_FEATURE_NAMES_BY_VERSION: + raise StaleCacheError( + f"CP cache entry records no feature names and an unknown recipe {version!r}; " + "rebuild it with force_recompute.gt_cp/pred_cp=true" + ) + names = CP_FEATURE_NAMES_BY_VERSION[version] + return names if entry["cp_glcm_enabled"] else tuple(n for n in names if not n.startswith("glcm_")) + + +def check_cp_cache_feature_names(ctx: _CacheContext, expected: tuple[str, ...]) -> None: + """Refuse a CP cache whose columns are not ``expected``, by name and order. + + No-op when the cache is disabled, holds no CP artifact yet, or this run + recomputes CP for the side (``force_recompute._cp``): then the columns + come from the running code, whose names the CP reference check already + compared. + + Raises + ------ + StaleCacheError + If the cached column names differ from ``expected``. + """ + if not ctx.enabled or ctx.force[f"{ctx.side}_cp"]: + return + entry = ctx.manifest.get("artifacts", {}).get("cp_features") + if entry is None: + return + cached = cached_cp_feature_names(entry) + if cached != tuple(expected): + raise StaleCacheError( + f"{ctx.side} CP cache {ctx.paths.cp_features()} holds columns {list(cached)}, but the CP reference " + f"expects {list(expected)}. Masking by position would misalign them; rebuild the cache with " + f"force_recompute.{ctx.side}_cp=true." + ) + + def fov_cp_features( ctx: _CacheContext, pos_name: str, @@ -1384,6 +1449,10 @@ def fov_cp_features( "path": "features/cp.zarr", "built_at": built_at_now(), **_cp_identity(ctx), + # Column names, so a consumer (the CP reference) can check them by name. + # Not part of the identity: caches written before this key stay valid and + # have their names read back by cached_cp_feature_names. + "cp_feature_names": list(active_cp_feature_names(bool((ctx.cp_glcm or {}).get("enabled", False)))), } _update_manifest_entry(ctx.manifest, ["cp_features"], entry, preserve_identity=ctx.excluded_walk) _add_position(ctx.manifest, ["cp_features"], pos_name) diff --git a/applications/dynacell/tests/test_cp_feature_names.py b/applications/dynacell/tests/test_cp_feature_names.py new file mode 100644 index 000000000..deac345d0 --- /dev/null +++ b/applications/dynacell/tests/test_cp_feature_names.py @@ -0,0 +1,85 @@ +"""CP feature names travel with the CP cache, so the reference masks by name, not by position.""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +from dynacell.evaluation.cache import StaleCacheError, cache_paths, load_manifest, save_manifest +from dynacell.evaluation.metrics import CP_FEATURE_NAMES_BY_VERSION, CP_FEATURE_VERSION, active_cp_feature_names +from dynacell.evaluation.pipeline_cache import ( + cached_cp_feature_names, + check_cp_cache_feature_names, + fov_cp_features, + init_cache_context, +) + +from ._eval_fixtures import build_eval_config + +_NAMES_GLCM = active_cp_feature_names(True) +_NAMES_BASE = active_cp_feature_names(False) + + +def test_frozen_table_matches_the_live_column_order() -> None: + """Reordering the live CP tuples without a CP_FEATURE_VERSION bump fails here, not silently in old caches.""" + assert CP_FEATURE_NAMES_BY_VERSION[CP_FEATURE_VERSION] == _NAMES_GLCM + + +def test_names_are_read_back_from_old_and_new_entries() -> None: + """New entries carry their names; old ones are inferred exactly from the recorded recipe.""" + assert cached_cp_feature_names({"cp_feature_names": ["b", "a"]}) == ("b", "a") + old = {"cp_feature_version": CP_FEATURE_VERSION, "cp_glcm_enabled": True} + assert cached_cp_feature_names(old) == _NAMES_GLCM + assert cached_cp_feature_names({**old, "cp_glcm_enabled": False}) == _NAMES_BASE + with pytest.raises(StaleCacheError, match="unknown recipe 'v0'"): + cached_cp_feature_names({"cp_feature_version": "v0", "cp_glcm_enabled": True}) + + +def _config(tmp_path: Path): + config = build_eval_config( + tmp_path / "pred.zarr", + tmp_path / "gt.zarr", + tmp_path / "gt_cache", + tmp_path / "pred_cache", + tmp_path, + executor="serial", + fov_workers=1, + ) + config.io.require_complete_cache = False + config.feature_metrics.cp = { + "norm": {"p_lo": 1.0, "p_hi": 99.0}, + "glcm": {"enabled": False}, + "reference_path": None, + } + return config + + +def test_the_cache_writer_records_names_and_a_reorder_is_refused(tmp_path: Path) -> None: + """``fov_cp_features`` stamps the column names; a cache whose names are reordered is refused by name.""" + config = _config(tmp_path) + ctx = init_cache_context(config, side="gt") + image = np.random.default_rng(0).uniform(size=(1, 11, 16, 16)).astype(np.float32) + seg = np.zeros((1, 11, 16, 16), dtype=np.int32) + seg[0, 2:8, 2:8, 2:8] = 1 + fov_cp_features(ctx, "A/1/0", image, seg) + assert ctx.manifest["artifacts"]["cp_features"]["cp_feature_names"] == list(_NAMES_BASE) + check_cp_cache_feature_names(ctx, _NAMES_BASE) + + swapped = list(_NAMES_BASE) + swapped[0], swapped[1] = swapped[1], swapped[0] + with pytest.raises(StaleCacheError, match="Masking by position would misalign them"): + check_cp_cache_feature_names(ctx, tuple(swapped)) + + +def test_a_forced_recompute_skips_the_name_check(tmp_path: Path) -> None: + """With force_recompute._cp the cached columns are about to be rewritten, so they are not checked.""" + config = _config(tmp_path) + paths = cache_paths(tmp_path / "gt_cache") + manifest = load_manifest(paths) + manifest["artifacts"]["cp_features"] = {"cp_feature_names": ["x"], "cp_feature_version": "v0"} + save_manifest(paths, manifest) + config.force_recompute.gt_cp = True + ctx = init_cache_context(config, side="gt") + check_cp_cache_feature_names(ctx, _NAMES_BASE) diff --git a/applications/dynacell/tests/test_evaluation_cp_end_to_end.py b/applications/dynacell/tests/test_evaluation_cp_end_to_end.py index 88aa6fb19..81f1db880 100644 --- a/applications/dynacell/tests/test_evaluation_cp_end_to_end.py +++ b/applications/dynacell/tests/test_evaluation_cp_end_to_end.py @@ -21,7 +21,13 @@ import pytest from iohub.ngff import open_ome_zarr -from dynacell.evaluation.cache import cache_paths, open_features_group, write_features_to_group +from dynacell.evaluation.cache import ( + cache_paths, + load_manifest, + open_features_group, + save_manifest, + write_features_to_group, +) from dynacell.evaluation.model_loader import EvalModels from ._eval_fixtures import N_POSITIONS, D, H, T, W, build_eval_config, live_pipeline_module, make_mask_cache @@ -170,3 +176,15 @@ def test_same_count_gt_value_change_is_refused(harness: Harness) -> None: harness.run("moved", [g.copy() for g in harness.gt]) assert type(err.value).__name__ == "StaleCacheError" assert not list(save_dir.glob("*_metrics.*")) and not (save_dir / "cp_selected_feature_mask.json").exists() + + +def test_gt_cache_with_reordered_columns_is_refused(harness: Harness) -> None: + """A GT CP cache whose recorded column names are reordered fails before the FOV loop.""" + paths = cache_paths(harness.root / "gt_cache") + manifest = load_manifest(paths) + names = manifest["artifacts"]["cp_features"]["cp_feature_names"] + names[0], names[1] = names[1], names[0] + save_manifest(paths, manifest) + with pytest.raises(Exception, match="Masking by position would misalign them") as err: + harness.run("reordered", [g.copy() for g in harness.gt]) + assert type(err.value).__name__ == "StaleCacheError" diff --git a/applications/dynacell/tools/build_cp_reference.py b/applications/dynacell/tools/build_cp_reference.py index be6d3b13b..39081d6cf 100644 --- a/applications/dynacell/tools/build_cp_reference.py +++ b/applications/dynacell/tools/build_cp_reference.py @@ -65,7 +65,7 @@ write_cp_reference, ) from dynacell.evaluation.paths import cp_reference_path -from dynacell.evaluation.pipeline_cache import init_cache_context +from dynacell.evaluation.pipeline_cache import check_cp_cache_feature_names, init_cache_context _EVAL_YAML = Path(dynacell.evaluation.__file__).parent / "_configs" / "eval.yaml" _LEAVES_ROOT = Path(__file__).resolve().parents[1] / "configs/benchmarks/virtual_staining/_internal/leaf/grouped" @@ -118,7 +118,7 @@ def lite_parent(ref: Ref) -> str | None: def read_dataset_fit( - target_name: str, ref: Ref, n_features: int, in_mask_fit: bool, parent: str | None = None + target_name: str, ref: Ref, feature_names: tuple[str, ...], in_mask_fit: bool, parent: str | None = None ) -> DatasetFit: """Read every finite GT CP cell of one dataset from its cache, read-only, in canonical order. @@ -129,15 +129,16 @@ def read_dataset_fit( Raises ------ StaleCacheError - If the cache belongs to another store or CP recipe, a ``(position, - timepoint)`` of the GT store is missing from it, or an entry has the wrong - column count. + If the cache belongs to another store or CP recipe, its columns are not + ``feature_names`` by name and order, a ``(position, timepoint)`` of the GT + store is missing from it, or an entry has the wrong column count. ValueError If a GT position is not a 3-D volume. """ config = eval_config_for(target_name, ref) ctx = init_cache_context(config, side="gt") - cells, read = read_gt_cp_cells(ctx, Path(config.io.gt_path), n_features) + check_cp_cache_feature_names(ctx, feature_names) + cells, read = read_gt_cp_cells(ctx, Path(config.io.gt_path), len(feature_names)) entry = ctx.manifest["artifacts"].get("cp_features") record = { "target": ref[1], @@ -173,13 +174,13 @@ def build(target_name: str, leaves_root: Path) -> dict[str, Any]: if cp_space(eval_config_for(target_name, ref)) != (identity, names): raise ValueError(f"{ref[0]} resolves a different CP recipe than {scaler_refs[0][0]}") - fits = [read_dataset_fit(target_name, ref, len(names), ref in mask_refs) for ref in scaler_refs] + fits = [read_dataset_fit(target_name, ref, names, ref in mask_refs) for ref in scaler_refs] positions = {fit.dataset: set(fit.record["positions"]) for fit in fits} for ref in lite_refs: parent = parents[ref] if parent not in positions: raise ValueError(f"lite {ref[0]} reuses {parent}, which no grouped leaf or mask fit evaluates") - lite = read_dataset_fit(target_name, ref, len(names), False, parent=parent) + lite = read_dataset_fit(target_name, ref, names, False, parent=parent) extra = set(lite.record["positions"]) - positions[parent] if extra: raise ValueError(f"lite {ref[0]} has positions outside its parent {parent}: {sorted(extra)[:5]}") @@ -206,7 +207,7 @@ def verify(target_name: str, path: Path) -> list[str]: ref = load_cp_reference(path, target_name=target_name) mismatches = [] for name, record in sorted({**ref.fit["datasets"], **ref.fit["lite"]}.items()): - fit = read_dataset_fit(target_name, (name, record["target"]), len(ref.feature_names), record["in_mask_fit"]) + fit = read_dataset_fit(target_name, (name, record["target"]), ref.feature_names, record["in_mask_fit"]) sha256 = gt_matrix_sha256(fit.cells) ok = sha256 == record["gt_matrix_sha256"] and fit.cells.shape[0] == record["n_cells"] print(f" {name}: {'OK' if ok else 'MISMATCH'} ({fit.cells.shape[0]} cells, sha256 {sha256[:12]})") diff --git a/applications/dynacell/tools/build_cp_reference_test.py b/applications/dynacell/tools/build_cp_reference_test.py index c78ddea9a..86fbe6b99 100644 --- a/applications/dynacell/tools/build_cp_reference_test.py +++ b/applications/dynacell/tools/build_cp_reference_test.py @@ -26,6 +26,7 @@ from dynacell.evaluation.cache import ( StaleCacheError, cache_paths, + load_manifest, open_features_group, save_manifest, write_features_to_group, @@ -254,3 +255,18 @@ def test_verify_passes_on_unchanged_caches_and_fails_on_a_value_only_recache(reg assert main(["--target", "er", "--out", str(out), "--verify"]) == 1 printed = capsys.readouterr().out assert "MISMATCH ds-h" in printed and "verify" in printed and "FAILED" in printed + + +def test_cache_with_reordered_feature_names_is_refused(registry: Path) -> None: + """The builder refuses a GT CP cache whose recorded column names are not the recipe's, by name.""" + _dataset(registry, "ds-a", seed=0) + _dataset(registry, "ds-b", seed=1) + paths = cache_paths(registry / "caches" / "ds-b") + manifest = load_manifest(paths) + names = list(active_cp_feature_names(True)) + names[0], names[1] = names[1], names[0] + manifest["artifacts"]["cp_features"]["cp_feature_names"] = names + save_manifest(paths, manifest) + leaves = _leaves(registry, ["ds-a", "ds-b"]) + with pytest.raises(StaleCacheError, match="Masking by position would misalign them"): + main(["--target", "er", "--leaves-root", str(leaves), "--out", str(registry / "x.json")]) From 5c40ac7ff7d37ee368555b92a442c291ea01792c Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 20:33:47 -0700 Subject: [PATCH 21/41] test(dynacell/eval): score CP offset and contraction through the real pipeline No test ran evaluate_predictions with feature metrics on. Three mutations survived: not aggregating the GT cells, counting non-finite rows, and the serial path passing cp_space=None. The headline test also still passed with the per-side z-score put back, because it called transform + the pairwise metric directly (blocker 5 of the #499 review). The end-to-end harness runs the real pipeline on tiny plates with real CP regionprops, caches, reference, pixel and mask metrics; only the deep extractors are stubbed. It adds cases where pred is the GT with a +0.15 offset, or x0.5, inside the cells. Dataset_CP_KID lands at 14.94 and 22671 against a pred==GT floor of -0.242. The retired per-side z-score on the same run's cells gives 0.19 and 0.18, close to the floor. Each of the four mutations above fails at least two tests. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../tests/test_evaluation_cp_end_to_end.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/applications/dynacell/tests/test_evaluation_cp_end_to_end.py b/applications/dynacell/tests/test_evaluation_cp_end_to_end.py index 81f1db880..47c55772e 100644 --- a/applications/dynacell/tests/test_evaluation_cp_end_to_end.py +++ b/applications/dynacell/tests/test_evaluation_cp_end_to_end.py @@ -188,3 +188,46 @@ def test_gt_cache_with_reordered_columns_is_refused(harness: Harness) -> None: with pytest.raises(Exception, match="Masking by position would misalign them") as err: harness.run("reordered", [g.copy() for g in harness.gt]) assert type(err.value).__name__ == "StaleCacheError" + + +def _inside_cells(harness: Harness, fn) -> list[np.ndarray]: + """A prediction equal to the GT outside the cells and ``fn(GT)`` inside them.""" + inside = (harness.seg > 0) & (harness.seg <= 16) + return [np.where(inside, fn(g), g).astype(np.float32) for g in harness.gt] + + +def _old_per_side_kid(save_dir: Path, cp_space) -> float: + """The retired transform on the same run's cells: reference mask, then each side by its own stats.""" + from dynacell.evaluation.feature_metrics import compute_feature_similarity + + def zscore(x: np.ndarray) -> np.ndarray: + return (x - x.mean(0)) / (x.std(0) + 1e-8) + + emb = { + side: np.load(save_dir / "embeddings" / f"{side}_cp_single_cell_embeddings.npz")["embeddings"] + for side in ("pred", "gt") + } + pred, gt = cp_space.select(emb["pred"]), cp_space.select(emb["gt"]) + return compute_feature_similarity( + zscore(pred), zscore(gt), "CP", compute_fid=False, compute_prc=False, compute_mind=False + )["CP_KID"] + + +@pytest.mark.parametrize( + ("name", "fn"), + [("offset", lambda g: g + 0.15), ("contracted", lambda g: 0.5 * g)], + ids=["pred=GT+offset", "pred=GTx0.5"], +) +def test_cp_kid_registers_offset_and_contraction_end_to_end(harness: Harness, name: str, fn) -> None: + """Through the real pipeline, an offset or contracted prediction scores clearly above the pred==GT floor. + + Contrast: the retired per-side z-score, applied to the same run's CP cells, + scores both near the floor, because it standardizes each side by its own + statistics and so absorbs a per-feature offset or scale. + """ + floor_row, _ = harness.run("identity", [g.copy() for g in harness.gt]) + row, config = harness.run(name, _inside_cells(harness, fn)) + floor, kid = floor_row["Dataset_CP_KID"], row["Dataset_CP_KID"] + old = _old_per_side_kid(Path(config.save.save_dir), harness.pipeline.eval_cp_space(config)) + assert kid > floor + 1.0, (kid, floor) + assert abs(old - floor) < 0.1 * (kid - floor), (old, floor, kid) From 26be0e444fc0d69e94e169d2e88ea7eb1b1a3ffb Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 20:35:08 -0700 Subject: [PATCH 22/41] test(dynacell/eval): pin lite staging to the parent scaler with pred != GT The lite staging tests used pred == GT and never checked the staged arrays, so an eval-time refit on the lite cells passed (item 6 of the #499 review). The new test stages pred = 0.5*GT + 1 on the lite set. It asserts the staged metric arrays equal the parent scaler's transform of pred and GT, and differ from a lite-cell refit. A mutation that refits on the lite GT at staging now fails it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../dynacell/evaluation/cp_reference_test.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py index 6c33fea05..96cc61e40 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -433,3 +433,24 @@ def rebound(edit) -> dict[str, str]: assert gt["set-b"] == bindings["set-b"] and gt["set-a-lite"] == bindings["set-a-lite"] lite_gt = rebound(lambda p: p["fit"]["lite"]["set-a-lite"].update(gt_matrix_sha256="0" * 64)) assert lite_gt["set-a-lite"] != bindings["set-a-lite"] and lite_gt["set-a"] == bindings["set-a"] + + +def test_lite_staging_uses_the_parent_scaler(two_sets, tmp_path: Path) -> None: + """A lite eval's staged CP arrays are the PARENT scaler's transform of pred and GT, never a lite refit. + + pred != GT and the lite cells' own statistics differ from the parent's, so an + eval-time refit on the lite cells (or on either side) gives different arrays. + """ + ref, _, cells = two_sets + lite_gt = cells["set-a"][:_LITE_ROWS] + pred = 0.5 * lite_gt + 1.0 + lite, parent = ref.for_dataset("set-a-lite"), ref.for_dataset("set-a") + staged = _stage_cp_dataset_inputs(_lists(pred, lite_gt), lite, _blocks(lite_gt), tmp_path) + np.testing.assert_array_equal(staged[1], parent.transform(pred)) + np.testing.assert_array_equal(staged[2], parent.transform(lite_gt)) + kept = lite_gt[:, ref.keep_mask] + refit = (kept - kept.mean(axis=0)) / kept.std(axis=0) + assert not np.allclose(staged[2], refit) + assert not np.allclose(staged[1], staged[2]) + sidecar = json.loads((tmp_path / "cp_selected_feature_mask.json").read_text()) + assert (sidecar["dataset"], sidecar["scaler_dataset"]) == ("set-a-lite", "set-a") From 4426fadcbc6c95b77e1afeb47af1e21d8ab8bf03 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 20:38:14 -0700 Subject: [PATCH 23/41] fix(dynacell/eval): let a cross-condition probe failure fail the grouped run The grouped driver wrapped the probe in `except Exception` and only printed. So an unloadable CP reference or sidecar (missing, rebuilt, or a pre-PR sidecar without reference_path) silently dropped the probe for every model and feature (item 7 of the #499 review). The except had no specific reason: the probe runs after every condition's metrics are saved, and the probe already records missing embeddings as a per-row skip. It is removed, and a test shows a pre-PR sidecar now raises out of evaluate_predictions_grouped. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/pipeline.py | 14 ++++----- .../test_evaluation_cp_scaler_selection.py | 31 +++++++++++++++++++ 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline.py b/applications/dynacell/src/dynacell/evaluation/pipeline.py index b6ec9d572..23c6aa027 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline.py @@ -2266,7 +2266,10 @@ def get_models() -> EvalModels: # mock (FOV-stratified logistic probe, per feature space, GT and pred). # Default-on whenever feature metrics were computed (the probe consumes the # embeddings those produce) and a mock + >=1 infected condition are present. - # Gated by ``cross_condition_probe.enabled``; never fails the eval. + # Gated by ``cross_condition_probe.enabled``. It runs after every condition's metrics + # are saved, so a failure loses nothing already written; it is not swallowed, since a + # CP reference/sidecar failure there means the probe would score in the wrong space. + # (Missing embeddings of one side are already a recorded skip inside the probe.) probe_enabled = bool( OmegaConf.select( config, @@ -2277,12 +2280,9 @@ def get_models() -> EvalModels: if probe_enabled: n_splits = int(OmegaConf.select(config, "cross_condition_probe.n_splits", default=5)) rng_seed = int(OmegaConf.select(config, "cross_condition_probe.rng_seed", default=2020)) - try: - written = _cross_condition_run_for_group(condition_save_dirs, n_splits=n_splits, rng_seed=rng_seed) - if written: - print(f"[grouped] cross-condition probe wrote {len(written)} CSV(s): {[str(p) for p in written]}") - except Exception as e: # noqa: BLE001 -- diagnostic add-on must not fail the eval - print(f"[grouped] cross-condition probe skipped: {type(e).__name__}: {e}") + written = _cross_condition_run_for_group(condition_save_dirs, n_splits=n_splits, rng_seed=rng_seed) + if written: + print(f"[grouped] cross-condition probe wrote {len(written)} CSV(s): {[str(p) for p in written]}") return results diff --git a/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py b/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py index 5e3e003cf..47af012aa 100644 --- a/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py +++ b/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py @@ -110,3 +110,34 @@ def test_unknown_dataset_is_refused(tmp_path: Path) -> None: config.benchmark.dataset_ref.dataset = "unknown-set" with pytest.raises(KeyError, match="no scaler for dataset 'unknown-set'"): pipeline.eval_cp_space(config) + + +def test_grouped_probe_failure_on_the_cp_reference_propagates(tmp_path: Path, monkeypatch) -> None: + """A cross-condition probe that cannot load the CP reference fails the grouped run, not a print.""" + pipeline = live_pipeline_module() + base = _config(tmp_path) + conditions = [] + for cond in ("mock", "denv"): + save_dir = tmp_path / "er" / "model" / "ipsc" / f"a549__{cond}" + emb = save_dir / "embeddings" + emb.mkdir(parents=True) + for side in ("pred", "gt"): + for kind in ("cp", "dinov3", "dynaclr", "celldino", "morphem"): + np.savez(emb / f"{side}_{kind}_single_cell_embeddings.npz", embeddings=np.zeros((4, 3)), fov=["f"] * 4) + # A pre-PR sidecar: no reference_path, so the CP mask cannot be resolved. + (save_dir / "cp_selected_feature_mask.json").write_text('{"keep_mask": [true, true, true]}') + conditions.append({"name": cond, "save": {"save_dir": str(save_dir)}}) + config = OmegaConf.merge( + base, + { + "conditions": conditions, + "force_recompute": {"final_metrics": True}, + "cross_condition_probe": {"enabled": True}, + }, + ) + monkeypatch.setattr(pipeline, "apply_dataset_ref", lambda cfg: None) + monkeypatch.setattr(pipeline, "load_eval_models", lambda cfg: object()) + monkeypatch.setattr(pipeline, "evaluate_predictions", lambda cfg, *, models, cp_space: ([], [], [])) + monkeypatch.setattr(pipeline, "save_metrics", lambda *a, **k: None) + with pytest.raises(KeyError, match="reference_path"): + pipeline.evaluate_predictions_grouped(config) From 9abbf13d92d3e698c78ad2356ced1b717462bcbc Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 20:39:15 -0700 Subject: [PATCH 24/41] fix(dynacell/eval): stamp only this dataset's CP space, not the whole reference binding_sha256 hashed the whole-reference sha256. So adding or refitting any dataset of a target (a new HEK leaf, a movie set) invalidated every eval dir of that target, pixel and mask metrics included, since the final-metrics cache is all-or-nothing (item 8 of the #499 review). The stamped cp_space_sha256 now covers only what this dataset's CP numbers depend on: the CP recipe identity, criteria, feature names, keep-mask, this dataset's scaler (mean, std, floored features; the parent's for lite, plus the lite -> parent link) and its GT-matrix sha256. A test adds an unrelated dataset and the existing binding holds; changing that dataset's own std or floors moves it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/cp_reference.py | 35 +++++++++++++++---- .../dynacell/evaluation/cp_reference_test.py | 24 +++++++++++++ 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index 6be424a31..5c1673fe3 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -470,15 +470,23 @@ class DatasetCPSpace: fit_positions: frozenset[str] fit_n_cells: int gt_matrix_sha256: str + cp_identity: dict[str, Any] + criteria: dict[str, Any] + floored_features: tuple[str, ...] @property def binding_sha256(self) -> str: - """Return a sha256 identifying this eval's CP space AND the GT cells it was fit on. - - Stamped as ``cp_space_sha256`` so cache reuse refuses another dataset's - space and GT cells that differ from the fit. Covers the reference hash, the - dataset, its scaler dataset, the lite flag, and this dataset's recorded - GT-matrix sha256 (a lite set's own). + """Return a sha256 over exactly what this eval's CP numbers depend on. + + Stamped as ``cp_space_sha256`` and compared on final-metrics cache reuse. + It covers the CP recipe identity, the selection criteria, the feature + names and keep-mask, THIS dataset's scaler (mean, std, floored features -- + the parent's for a lite set, plus the lite -> parent link), and this + dataset's recorded GT-matrix sha256 (a lite set's own). It deliberately + leaves out the whole-reference hash, so adding or refitting another dataset + of the target does not invalidate this dataset's cached eval dirs (whose + cache is all-or-nothing, pixel and mask metrics included), while another + dataset's space, a changed scaler, or different GT cells all do. Returns ------- @@ -486,10 +494,18 @@ def binding_sha256(self) -> str: Hex sha256 over the canonical JSON of those fields. """ body = { - "reference_sha256": self.reference_sha256, + "cp_identity": self.cp_identity, + "criteria": self.criteria, + "feature_names": list(self.feature_names), + "keep_mask": [bool(b) for b in self.keep_mask], "dataset": self.dataset, "scaler_dataset": self.scaler_dataset, "is_lite": self.is_lite, + "scaler": { + "mean": [float(v) for v in self.mean], + "std": [float(v) for v in self.std], + "floored_features": list(self.floored_features), + }, "gt_matrix_sha256": self.gt_matrix_sha256, } return hashlib.sha256(json.dumps(body, sort_keys=True, separators=(",", ":")).encode()).hexdigest() @@ -581,6 +597,7 @@ class CPReference: feature_names: tuple[str, ...] keep_mask: np.ndarray cp_identity: dict[str, Any] + criteria: dict[str, Any] scalers: dict[str, dict[str, Any]] lite: dict[str, dict[str, Any]] fit: dict[str, Any] @@ -617,6 +634,9 @@ def for_dataset(self, dataset: str) -> DatasetCPSpace: fit_positions=frozenset(own["positions"]), fit_n_cells=int(own["n_cells"]), gt_matrix_sha256=own["gt_matrix_sha256"], + cp_identity=self.cp_identity, + criteria=self.criteria, + floored_features=tuple(scaler["floored_features"]), ) @@ -640,6 +660,7 @@ def _read_cp_reference(path: Path, *, build_hint: str) -> CPReference: feature_names=tuple(payload["feature_names"]), keep_mask=np.asarray(payload["keep_mask"], dtype=bool), cp_identity=payload["cp_identity"], + criteria=payload["criteria"], scalers=payload["scalers"], lite=payload["lite"], fit=payload["fit"], diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py index 96cc61e40..f37b881c2 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -454,3 +454,27 @@ def test_lite_staging_uses_the_parent_scaler(two_sets, tmp_path: Path) -> None: assert not np.allclose(staged[1], staged[2]) sidecar = json.loads((tmp_path / "cp_selected_feature_mask.json").read_text()) assert (sidecar["dataset"], sidecar["scaler_dataset"]) == ("set-a-lite", "set-a") + + +def test_binding_covers_only_this_datasets_space(tmp_path: Path) -> None: + """Adding an unrelated dataset keeps an existing dataset's binding; changing its own scaler moves it.""" + a, b = _gt(), _gt(seed=1) + base = [_fit("set-a", a), _fit("set-b", b)] + ref, payload = _reference(tmp_path, base) + before = ref.for_dataset("set-a").binding_sha256 + + (tmp_path / "added").mkdir() + added, _ = _reference(tmp_path / "added", [*base, _fit("hek", _gt(seed=2, offset=-5.0), in_mask_fit=False)]) + assert added.sha256 != ref.sha256 # the whole reference did change + assert added.for_dataset("set-a").binding_sha256 == before + + moved = json.loads(json.dumps(payload)) + moved["scalers"]["set-a"]["std"][0] *= 1.5 + moved["sha256"] = payload_sha256(moved) + (tmp_path / "er.json").write_text(json.dumps(moved)) + assert load_cp_reference(tmp_path / "er.json", target_name="er").for_dataset("set-a").binding_sha256 != before + floored = json.loads(json.dumps(payload)) + floored["scalers"]["set-a"]["floored_features"] = ["f0"] + floored["sha256"] = payload_sha256(floored) + (tmp_path / "er.json").write_text(json.dumps(floored)) + assert load_cp_reference(tmp_path / "er.json", target_name="er").for_dataset("set-a").binding_sha256 != before From 5fc5ef89d6c227e3de51a1794c0a1f0ed8fe1b9a Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 20:39:58 -0700 Subject: [PATCH 25/41] fix(dynacell/eval): name benchmark.dataset_ref when a CP eval has none eval_cp_space read config.benchmark.dataset_ref.dataset directly, so a feature-metrics eval without a dataset_ref died with an AttributeError on None (item 9 of the #499 review). It now raises a ValueError naming benchmark.dataset_ref.dataset, why the scaler needs it, and the compute_feature_metrics=false escape hatch. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../dynacell/src/dynacell/evaluation/cp_reference.py | 11 ++++++++++- .../tests/test_evaluation_cp_scaler_selection.py | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index 5c1673fe3..552aa8efa 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -773,6 +773,8 @@ def eval_cp_space(config: DictConfig) -> DatasetCPSpace: If the reference's CP recipe identity or feature names differ from the eval's. KeyError If the reference has no scaler for the eval's dataset. + ValueError + If the config has no ``benchmark.dataset_ref.dataset``. """ identity, names = cp_space(config) ref = load_cp_reference(resolve_cp_reference_path(config), target_name=config.target_name) @@ -783,4 +785,11 @@ def eval_cp_space(config: DictConfig) -> DatasetCPSpace: f" reference features: {len(ref.feature_names)}; eval features: {len(names)}\n" f"Rebuild it from caches of the current recipe:\n {_build_command(ref.target_name)}" ) - return ref.for_dataset(config.benchmark.dataset_ref.dataset) + dataset = OmegaConf.select(config, "benchmark.dataset_ref.dataset", default=None) + if dataset is None: + raise ValueError( + "CP feature metrics need benchmark.dataset_ref.dataset: the CP reference holds one GT scaler per " + "test set, keyed by that name. Set benchmark.dataset_ref (dataset + target) in the eval config, or " + "pass compute_feature_metrics=false." + ) + return ref.for_dataset(dataset) diff --git a/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py b/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py index 47af012aa..7b25fb8d6 100644 --- a/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py +++ b/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py @@ -141,3 +141,12 @@ def test_grouped_probe_failure_on_the_cp_reference_propagates(tmp_path: Path, mo monkeypatch.setattr(pipeline, "save_metrics", lambda *a, **k: None) with pytest.raises(KeyError, match="reference_path"): pipeline.evaluate_predictions_grouped(config) + + +def test_missing_dataset_ref_names_the_config_key(tmp_path: Path) -> None: + """Without benchmark.dataset_ref the error names the key and the feature-less escape hatch.""" + pipeline = live_pipeline_module() + config = _config(tmp_path) + config.benchmark = None + with pytest.raises(ValueError, match=r"benchmark\.dataset_ref\.dataset.*compute_feature_metrics=false"): + pipeline.eval_cp_space(config) From f69f32538aa40f3ae4b1c0112d0c1b3b0b02ea01 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 20:39:59 -0700 Subject: [PATCH 26/41] docs(dynacell/tools): show the feature-less limit_positions smoke run With the CP reference, limit_positions plus feature metrics raises (the scaler and GT-cell hash cover every GT position). The raise is kept, and its message already points at compute_feature_metrics=false. The run_eval_direct.slurm header example now includes it (item 10 of the #499 review). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- applications/dynacell/tools/run_eval_direct.slurm | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/applications/dynacell/tools/run_eval_direct.slurm b/applications/dynacell/tools/run_eval_direct.slurm index f13c8199e..cc92946c9 100755 --- a/applications/dynacell/tools/run_eval_direct.slurm +++ b/applications/dynacell/tools/run_eval_direct.slurm @@ -16,7 +16,11 @@ # all others invoke ``dynacell evaluate``. # # Extra positional args after the leaf path are passed verbatim as Hydra -# overrides (e.g. ``limit_positions=2 force_recompute.final_metrics=true``). +# overrides, e.g. a smoke run on two FOVs: +# ``limit_positions=2 compute_feature_metrics=false force_recompute.final_metrics=true`` +# (``compute_feature_metrics=false`` is required with ``limit_positions``: the CP +# feature metrics are scored against a reference fit on every GT position, so a +# partial walk with them on raises.) #SBATCH --job-name=eval_direct #SBATCH --time=1-00:00:00 From d6dc5d50e04891660aea255d28b03d7cb1e5611c Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 20:41:01 -0700 Subject: [PATCH 27/41] test(dynacell/eval): check CP scaler binding with the benchmark's dataset names The scaler-selection tests used synthetic names, which would not catch a swap between conditions sharing a prefix (item 11 of the #499 review). This test uses aics-hipsc, a549-mantis-sec61b-{mock,denv,zikv} and the denv lite set. It checks each grouped condition's scaler against the reference's scaler for exactly its name, and that the three A549 scalers are pairwise distinct. A denv->mock mapping mutation fails it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../test_evaluation_cp_scaler_selection.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py b/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py index 7b25fb8d6..c0a26eec0 100644 --- a/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py +++ b/applications/dynacell/tests/test_evaluation_cp_scaler_selection.py @@ -9,6 +9,7 @@ from __future__ import annotations +import json from pathlib import Path import numpy as np @@ -150,3 +151,59 @@ def test_missing_dataset_ref_names_the_config_key(tmp_path: Path) -> None: config.benchmark = None with pytest.raises(ValueError, match=r"benchmark\.dataset_ref\.dataset.*compute_feature_metrics=false"): pipeline.eval_cp_space(config) + + +_REAL_SETS = ("aics-hipsc", "a549-mantis-sec61b-mock", "a549-mantis-sec61b-denv", "a549-mantis-sec61b-zikv") +_REAL_LITE = {"a549-mantis-sec61b-denv-lite": "a549-mantis-sec61b-denv"} + + +def test_real_dataset_names_are_not_swapped(tmp_path: Path, monkeypatch) -> None: + """With the benchmark's own dataset names, mock/denv/zikv conditions each get their own scaler. + + A swap between conditions that share a prefix (mock for denv) would still pass the + synthetic-name tests; here every condition is checked against the reference's + scaler for exactly its name, and the three A549 scalers are pairwise distinct. + """ + pipeline = live_pipeline_module() + config = build_eval_config( + tmp_path / "pred.zarr", + tmp_path / "gt.zarr", + tmp_path / "g", + tmp_path / "p", + tmp_path / "out", + executor="serial", + fov_workers=1, + ) + config.compute_feature_metrics = True + make_cp_reference(config, tmp_path / "er.json", datasets=_REAL_SETS, lite=_REAL_LITE) + reference = json.loads((tmp_path / "er.json").read_text()) + names = (*_REAL_SETS, *_REAL_LITE) + conditions = [ + {"name": d, "benchmark": {"dataset_ref": {"dataset": d}}, "save": {"save_dir": str(tmp_path / d)}} + for d in names + ] + grouped = OmegaConf.merge( + config, + { + "conditions": conditions, + "force_recompute": {"final_metrics": True}, + "cross_condition_probe": {"enabled": False}, + }, + ) + seen = {} + monkeypatch.setattr(pipeline, "apply_dataset_ref", lambda cfg: None) + monkeypatch.setattr(pipeline, "load_eval_models", lambda cfg: object()) + monkeypatch.setattr( + pipeline, + "evaluate_predictions", + lambda cfg, *, models, cp_space: seen.__setitem__(cfg.benchmark.dataset_ref.dataset, cp_space) or ([], [], []), + ) + monkeypatch.setattr(pipeline, "save_metrics", lambda *a, **k: None) + pipeline.evaluate_predictions_grouped(grouped) + + for dataset in names: + scaler = _REAL_LITE.get(dataset, dataset) + assert seen[dataset].scaler_dataset == scaler + np.testing.assert_array_equal(seen[dataset].mean, reference["scalers"][scaler]["mean"]) + a549 = [seen[d].mean for d in _REAL_SETS[1:]] + assert all(not np.allclose(x, y) for i, x in enumerate(a549) for y in a549[i + 1 :]) From 5f6b26f8853374ebed9eb49738003070f5efb255 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 20:41:43 -0700 Subject: [PATCH 28/41] docs(dynacell/eval): document that GLCM+ cosine cannot see contraction to the GT mean The paper owner asked for this property to be stated next to the scaler description (#499 review). After the shared scaler, a prediction mean + 0.5*(GT - mean) gives every pred/GT cell pair the same direction, so median cosine is exactly 1.0000 while KID flags it (reviewer-measured on real cells: 0.194 vs a -0.041 floor). A test reproduces it on synthetic cells. The README also gets two lines on the pooled GT-only mask plus the per-test-set GT scaler. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/README.md | 2 ++ .../dynacell/evaluation/cp_reference_test.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/applications/dynacell/src/dynacell/evaluation/README.md b/applications/dynacell/src/dynacell/evaluation/README.md index 89c46feeb..8e2643a0a 100644 --- a/applications/dynacell/src/dynacell/evaluation/README.md +++ b/applications/dynacell/src/dynacell/evaluation/README.md @@ -25,6 +25,8 @@ End-to-end evaluation pipeline for virtual staining predictions against fluoresc - `io.cell_segmentation_path` — *optional* precomputed cell segmentation HCS OME-Zarr. Required when `compute_feature_metrics=true` or when building CP/DINOv3/DynaCLR/CELL-DINO cache entries. Position layout must match GT/pred 1:1. - `io.gt_cache_dir`, `io.pred_cache_dir` — *optional* artifact cache directories; must be distinct. See [Caches](#caches). - `feature_metrics.cp.reference_path` — the target's CP (GLCM+) reference, required when `compute_feature_metrics=true`. `null` (default) resolves to `DATA_ROOT/cp_reference/.json` for full and lite evals alike. The reference holds one feature mask chosen on GT cells only, pooled over iPSC + A549 mock/denv/zikv, and one GT mean/std per test set (keyed by `benchmark.dataset_ref.dataset`; HEK gets its own, lite reuses its parent's). Every eval applies the mask and its dataset's scaler to pred AND GT. The eval refuses a partial position walk, and refuses to score unless its staged GT CP cells hash to the sha256 the reference recorded for that dataset (a content gate: an identical re-cache passes, any value change fails). Build it from existing GT caches with `tools/build_cp_reference.py --target `. Its hash is stamped in `metrics_provenance.json`, and a rebuilt reference invalidates the final-metrics cache. + - GLCM+ (CP) space in two lines: one feature mask selected on GT cells only, pooled over the target's iPSC + A549 mock/denv/zikv test sets; then each test set's own GT mean/std standardizes pred and GT alike, so offsets and scale errors relative to that test set's GT stay visible. + - GLCM+ median cosine is invariant to contraction toward the GT mean: for `pred = mean + 0.5·(GT − mean)`, every pred/GT cell pair is parallel after the shared scaler and the median cosine is exactly 1.0000, while KID catches it (reviewer-measured on real cells: KID 0.194 vs a pred==GT floor of −0.041; reproduced on synthetic cells in `cp_reference_test.py`). Cosine measures direction only; over-smoothing toward the mean shows up in KID, not cosine. ## Quick start diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py index f37b881c2..c745ae1ae 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -478,3 +478,22 @@ def test_binding_covers_only_this_datasets_space(tmp_path: Path) -> None: floored["sha256"] = payload_sha256(floored) (tmp_path / "er.json").write_text(json.dumps(floored)) assert load_cp_reference(tmp_path / "er.json", target_name="er").for_dataset("set-a").binding_sha256 != before + + +def test_cosine_is_blind_to_contraction_toward_the_gt_mean_but_kid_is_not(two_sets) -> None: + """``pred = mean + 0.5 * (GT - mean)`` has median cosine exactly 1 after the scaler; KID still flags it. + + Centering on the dataset's GT mean makes each pred/GT cell pair parallel, and + cosine ignores magnitude, so GLCM+ cosine measures direction only and + over-smoothing toward the mean shows up in KID (documented in the evaluation + README; the reviewer measured KID 0.194 vs a floor of -0.041 on real cells). + """ + ref, _, cells = two_sets + gt = cells["set-a"] + mean = gt.mean(axis=0) + pred = mean + 0.5 * (gt - mean) + space = ref.for_dataset("set-a") + got = compute_feature_similarity_pairwise(space.transform(pred), space.transform(gt), "CP", compute_fid=False) + floor = compute_feature_similarity_pairwise(space.transform(gt), space.transform(gt), "CP", compute_fid=False) + assert got["CP_Median_Cosine_Similarity"] == pytest.approx(1.0, abs=1e-6) + assert got["CP_KID"] > floor["CP_KID"] + 0.1 From cdd04109d54f33b11712d31582c2784d6cc1741e Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 20:57:51 -0700 Subject: [PATCH 29/41] fix(dynacell/eval): resolve a relative CP reference override to an absolute path The override was stamped into each eval dir's sidecar as given, so a relative path could not be reloaded by the cross-condition probe or the lite tools when they ran from another working directory. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../dynacell/src/dynacell/evaluation/cp_reference.py | 4 +++- .../src/dynacell/evaluation/cp_reference_test.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index 552aa8efa..6722869db 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -745,7 +745,9 @@ def resolve_cp_reference_path(config: DictConfig) -> Path: """ override = config.feature_metrics.cp.reference_path if override is not None: - return Path(override) + # Absolute, so the path stamped into an eval dir's sidecar still resolves when + # a post-hoc tool (cross-condition probe, lite tools) runs from another cwd. + return Path(override).resolve() if config.target_name not in MASK_FIT_DATASETS: raise ValueError(f"no CP reference for target {config.target_name!r}; expected {sorted(MASK_FIT_DATASETS)}") return cp_reference_path(config.target_name) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py index c745ae1ae..8d7a12cfd 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -255,6 +255,15 @@ def test_registry_path_is_shared_by_lite_and_full() -> None: resolve_cp_reference_path(config) +def test_relative_reference_override_resolves_to_an_absolute_path(tmp_path: Path, monkeypatch) -> None: + """A relative override is made absolute, so the sidecar path survives a later cwd change.""" + monkeypatch.chdir(tmp_path) + config = OmegaConf.create({"target_name": "er", "feature_metrics": {"cp": {"reference_path": "refs/er.json"}}}) + path = resolve_cp_reference_path(config) + assert path.is_absolute() + assert path == tmp_path.resolve() / "refs" / "er.json" + + def test_positions_must_match_the_fit(two_sets) -> None: """Every dataset (lite included) must score exactly the positions its cells were recorded over.""" ref, _, _ = two_sets From d87a0293858729d93edd6b0af55d443ff6869b6a Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 21:10:20 -0700 Subject: [PATCH 30/41] fix(dynacell/eval): integrity-protect the CP reference's content-gate fields The reference hash skipped the whole fit section, so the positions, cell counts and GT-matrix hashes the content gate enforces could be edited by hand without failing the load-time hash check. Hash every fit field except the audit-only paths and cp_cache_built_at, which move on a harmless re-cache or relocation. Also correct the gt_matrix_sha256 docstring: the eval now recomputes it over the staged cells. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/cp_reference.py | 43 +++++++++++++------ .../dynacell/evaluation/cp_reference_test.py | 20 +++++++-- .../tests/test_evaluation_timepoint_guard.py | 7 +-- 3 files changed, 50 insertions(+), 20 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index 6722869db..6bac76184 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -108,13 +108,28 @@ ), } -#: Keys excluded from the content hash. The hash covers only what changes CP numbers -#: (recipe identity, criteria, feature names, keep-mask, every scaler's mean/std and -#: floored features, the lite -> parent map). ``fit`` holds the fit provenance the eval -#: checks read (positions, cell counts, GT-matrix hashes; ``built_at`` for audit), so a -#: rebuild after a harmless GT re-cache that leaves every scaler unchanged keeps the -#: hash and invalidates no final-metrics cache stamped with it. -_UNHASHED_KEYS = frozenset({"sha256", "created_at", "fit"}) +#: Top-level keys excluded from the content hash. +_UNHASHED_KEYS = frozenset({"sha256", "created_at"}) + +#: Audit-only fields of each ``fit`` provenance record, excluded from the content hash. +#: Everything else in ``fit`` -- positions, cell counts and GT-matrix hashes, which the +#: content gate enforces -- is hashed, so a hand-edited gate field fails the load-time +#: hash check. Paths and ``built_at`` are excluded because they move on a harmless GT +#: re-cache (identical cells) or a relocation, which must not change the hash. +_AUDIT_FIT_KEYS = frozenset({"gt_path", "gt_cache_dir", "cp_cache_path", "cp_cache_built_at"}) + + +def _hashed_fit(fit: dict[str, Any]) -> dict[str, Any]: + """Return ``fit`` without its audit-only per-dataset fields.""" + return { + "mask_fit": fit["mask_fit"], + **{ + group: { + name: {k: v for k, v in rec.items() if k not in _AUDIT_FIT_KEYS} for name, rec in fit[group].items() + } + for group in ("datasets", "lite") + }, + } def _build_command(target_name: str) -> str: @@ -151,7 +166,8 @@ def payload_sha256(payload: dict[str, Any]) -> str: Parameters ---------- payload : dict - Reference JSON payload. ``sha256`` and ``created_at`` are ignored. + Reference JSON payload. ``sha256``, ``created_at`` and the audit-only ``fit`` + fields (paths, ``cp_cache_built_at``) are ignored. Returns ------- @@ -159,6 +175,7 @@ def payload_sha256(payload: dict[str, Any]) -> str: Hex sha256 over the canonical (sorted-key, compact) JSON encoding. """ body = {k: v for k, v in payload.items() if k not in _UNHASHED_KEYS} + body["fit"] = _hashed_fit(payload["fit"]) return hashlib.sha256(json.dumps(body, sort_keys=True, separators=(",", ":")).encode()).hexdigest() @@ -167,9 +184,9 @@ def gt_matrix_sha256(gt: np.ndarray) -> str: The digest covers the shape and the float64 C-order bytes, so it changes with any cell value or the cell count. It is recorded per dataset in the reference's - ``fit`` section for audit; the eval does not recompute it (that would mean - re-reading every GT cell). ``build_cp_reference.py --verify`` recomputes it from - the current caches and fails on a mismatch. + ``fit`` section; the eval recomputes it over the GT cells it stages and refuses to + score on a mismatch (:meth:`DatasetCPSpace.check_gt_cells`), and + ``build_cp_reference.py --verify`` recomputes it from the current caches. Parameters ---------- @@ -423,8 +440,8 @@ def write_cp_reference(payload: dict[str, Any], path: Path, *, force: bool = Fal bool ``True`` if the file was written, ``False`` if an identical reference (same hash AND same fit provenance) was already there (no-op). Same hash with new - provenance (e.g. a re-cache that moved ``built_at``, or a lite dataset's - record) is written without ``force``: no scaler or mask changes. + audit-only provenance (a re-cache that moved ``built_at``, a relocated cache + path) is written without ``force``: no scaler, mask or gate field changes. Raises ------ diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py index 8d7a12cfd..e787c9fe2 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -213,9 +213,21 @@ def test_changed_scaler_changes_the_hash() -> None: relinked = json.loads(json.dumps(payload)) relinked["lite"]["set-a-lite"]["parent"] = "set-b" assert payload_sha256(relinked) != payload["sha256"] - provenance_only = json.loads(json.dumps(payload)) - provenance_only["fit"]["datasets"]["set-b"]["n_cells"] = 1 - assert payload_sha256(provenance_only) == payload["sha256"] + gate_field = json.loads(json.dumps(payload)) + gate_field["fit"]["datasets"]["set-b"]["n_cells"] = 1 # a content-gate field is integrity-protected + assert payload_sha256(gate_field) != payload["sha256"] + audit_only = json.loads(json.dumps(payload)) + audit_only["fit"]["datasets"]["set-b"]["cp_cache_built_at"] = "2099-01-01T00:00:00+00:00" + assert payload_sha256(audit_only) == payload["sha256"] + + +def test_hand_edited_gate_field_fails_the_load(tmp_path: Path) -> None: + """Editing a recorded GT-matrix hash without re-hashing is refused at load, not trusted by the gate.""" + payload = fit_cp_reference([_fit("set-a", _gt())], target_name="er", feature_names=_NAMES, cp_identity={}) + payload["fit"]["datasets"]["set-a"]["gt_matrix_sha256"] = "0" * 64 + (tmp_path / "er.json").write_text(json.dumps(payload)) + with pytest.raises(ValueError, match="does not match its recorded sha256"): + load_cp_reference(tmp_path / "er.json", target_name="er") def test_write_is_atomic_and_refuses_a_different_reference(tmp_path: Path) -> None: @@ -431,9 +443,9 @@ def test_binding_separates_datasets_and_tracks_the_gt_matrix(tmp_path: Path) -> def rebound(edit) -> dict[str, str]: moved = json.loads(json.dumps(payload)) edit(moved) + moved["sha256"] = payload_sha256(moved) # a legitimate rebuild re-hashes (tmp_path / "er.json").write_text(json.dumps(moved)) again = load_cp_reference(tmp_path / "er.json", target_name="er") - assert again.sha256 == ref.sha256 # provenance only: numeric hash unchanged return {d: again.for_dataset(d).binding_sha256 for d in bindings} assert rebound(lambda p: p["fit"]["datasets"]["set-a"].update(cp_cache_built_at="t1")) == bindings diff --git a/applications/dynacell/tests/test_evaluation_timepoint_guard.py b/applications/dynacell/tests/test_evaluation_timepoint_guard.py index ee613cb38..5c6194f14 100644 --- a/applications/dynacell/tests/test_evaluation_timepoint_guard.py +++ b/applications/dynacell/tests/test_evaluation_timepoint_guard.py @@ -18,6 +18,7 @@ from iohub.ngff import open_ome_zarr from omegaconf import OmegaConf +from dynacell.evaluation.cp_reference import payload_sha256 from dynacell.evaluation.model_loader import EvalModels from dynacell.evaluation.provenance import PROVENANCE_FILENAME, write_metrics_provenance @@ -223,13 +224,13 @@ def test_cache_scored_for_another_dataset_is_invalid(tmp_path: Path) -> None: def test_cache_is_invalid_once_the_gt_matrix_changed(tmp_path: Path) -> None: - """Same numeric reference hash, but the dataset's recorded GT matrix changed: CP rows are stale.""" + """Same scalers and mask, but the dataset's recorded GT matrix changed: CP rows are stale.""" pipeline = live_pipeline_module() config, reference = _two_set_cache(pipeline, tmp_path) payload = json.loads(reference.read_text()) - payload["fit"]["datasets"]["set-a"]["gt_matrix_sha256"] = "0" * 64 # unhashed fit provenance + payload["fit"]["datasets"]["set-a"]["gt_matrix_sha256"] = "0" * 64 + payload["sha256"] = payload_sha256(payload) # a legitimate rebuild re-hashes reference.write_text(json.dumps(payload)) - assert pipeline.eval_cp_space(config).reference_sha256 == payload["sha256"] # numeric hash unchanged assert not pipeline._final_metrics_cache_valid(config) From 338746747bcc0d8431b78fdb09bea469aa1e7b8b Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 21:21:54 -0700 Subject: [PATCH 31/41] docs(dynacell/eval): pair the README smoke-run examples with feature metrics off A partial position walk now raises when CP feature metrics are on, so the two `limit_positions` smoke examples would fail as written. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- applications/dynacell/src/dynacell/evaluation/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/README.md b/applications/dynacell/src/dynacell/evaluation/README.md index 8e2643a0a..9ee3eeae9 100644 --- a/applications/dynacell/src/dynacell/evaluation/README.md +++ b/applications/dynacell/src/dynacell/evaluation/README.md @@ -38,7 +38,7 @@ uv run dynacell evaluate \ save.save_dir=/hpc/.../eval_fnet3d_sec61b ``` -Add `compute_feature_metrics=true` to enable feature metrics. Smoke test on a subset of FOVs with `limit_positions=N`. +Add `compute_feature_metrics=true` to enable feature metrics. Smoke test on a subset of FOVs with `limit_positions=N compute_feature_metrics=false`: CP feature metrics are scored in a reference fit on the full GT cell set, so a partial position walk with feature metrics on raises. ## Submission tooling @@ -112,7 +112,7 @@ Canonical leaves at `configs/benchmarks/virtual_staining/// Date: Fri, 25 Sep 2026 21:33:14 -0700 Subject: [PATCH 32/41] fix(dynacell/eval): gate CP scoring on GT moments within tolerance, not an exact sha GPU cp_regionprops is not bit-reproducible. The reviewer measured 5 distinct hashes in 5 A40 runs (|diff| ~1e-15); here two runs of one block differ by 8.9e-16. So the exact-sha content gate failed every GPU force_recompute.all / gt_cp / --overwrite, after the FOV loop had already rewritten the shared GT cache, poisoning the dataset until a --force rebuild (F1, blocker, #499 re-review). The gate now requires an exact position set and GT-finite cell count, plus per-feature tolerance on the staged GT's raw mean/std over ALL CP columns against the fit's recorded moments: |dmean| <= 1e-6*std_fit and |std/std_fit - 1| <= 1e-6. A constant feature (std_fit == 0) must stay within 1e-12*(1+|mean_fit|). The tolerances are named constants. - Each fit record (lite from its own caches) stores gt_moments. They are hashed gate fields; gt_matrix_sha256 is audit-only (unhashed). - --verify applies the same gate, via the shared gt_cells_problems, and reports an exact-sha difference as information. - cp_space_sha256 hashes the recorded count + moments instead of the GT sha, so GPU jitter never moves a binding. - Tests: a 1e-12 relative jitter of every GT cell passes end to end. The same-count value change still raises. CUDA-only tests recompute one block twice on the GPU, and run an e2e force_recompute.gt_cp with use_gpu=True against a CPU-built reference. Both pass on the A40. The GPU fixtures leave out the single-voxel cell: cuCIM's GPU regionprops raises a TypeError on a one-voxel region. Schema 5. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/cp_reference.py | 210 +++++++++++++----- .../dynacell/evaluation/cp_reference_test.py | 20 +- .../tests/test_evaluation_cp_end_to_end.py | 68 +++++- .../tests/test_evaluation_timepoint_guard.py | 2 +- .../dynacell/tools/build_cp_reference.py | 22 +- 5 files changed, 244 insertions(+), 78 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index 6bac76184..1ba61eb1b 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -25,12 +25,17 @@ Each scaler is fit on exactly the GT cells an eval of that dataset scores: every position and timepoint of the GT store, read from the GT CP cache, with the non-finite rows dropped. The reference records every dataset's (lite included) -position list and the sha256 of those cells in canonical order -(:func:`canonical_gt_matrix`). An eval refuses a different position set up front +position list, cell count and raw per-feature GT moments (mean/std over ALL CP +columns). An eval refuses a different position set up front (:meth:`DatasetCPSpace.check_positions`) and, once its GT CP cells are staged, -refuses to score unless they hash to the recorded value -(:meth:`DatasetCPSpace.check_gt_cells`). That content gate is independent of the -cache manifest's ``built_at``, which is kept in the fit record for audit only. +refuses to score unless the count matches exactly and every feature's mean/std +matches the recorded moments within :data:`GT_MOMENT_RTOL` +(:meth:`DatasetCPSpace.check_gt_cells`). The gate is a tolerance, not an exact +hash, because GPU ``cp_regionprops`` is not bit-reproducible (run-to-run jitter +~1e-15 on an A40), so a GPU recompute of identical GT must pass; any real value +change is many orders of magnitude larger. The exact sha256 of the canonical GT +matrix (:func:`canonical_gt_matrix`) is kept for audit and ``--verify`` only, and +the cache manifest's ``built_at`` for audit only. A per-dataset std can collapse on a feature the pooled mask keeps (a feature that is near-constant within one test set). Such a std is floored at @@ -70,11 +75,20 @@ from dynacell.evaluation.pipeline_cache import cp_recipe_identity #: Schema of the reference JSON. Bump on any change to its keys or their meaning. -CP_REFERENCE_SCHEMA = 4 +CP_REFERENCE_SCHEMA = 5 #: Sidecar each eval writes beside its metrics, naming the CP reference it scored in. CP_SIDECAR_FILENAME = "cp_selected_feature_mask.json" +#: Content-gate tolerance on each CP feature's GT moments (see +#: :meth:`DatasetCPSpace.check_gt_cells`): |mean - mean_fit| <= GT_MOMENT_RTOL * std_fit +#: and |std / std_fit - 1| <= GT_MOMENT_RTOL. GPU regionprops jitter is ~1e-15 +#: relative, a real GT change is >= ~1e-3, so 1e-6 separates them by >= 3 decades each way. +GT_MOMENT_RTOL = 1e-6 +#: For a feature constant in the fit (std_fit == 0): |mean - mean_fit| and the staged std +#: must both be <= GT_MOMENT_ZERO_STD_ATOL * (1 + |mean_fit|). +GT_MOMENT_ZERO_STD_ATOL = 1e-12 + #: A per-dataset std below this fraction of the feature's pooled GT std is floored to it. STD_FLOOR_FRACTION = 1e-3 @@ -115,8 +129,9 @@ #: Everything else in ``fit`` -- positions, cell counts and GT-matrix hashes, which the #: content gate enforces -- is hashed, so a hand-edited gate field fails the load-time #: hash check. Paths and ``built_at`` are excluded because they move on a harmless GT -#: re-cache (identical cells) or a relocation, which must not change the hash. -_AUDIT_FIT_KEYS = frozenset({"gt_path", "gt_cache_dir", "cp_cache_path", "cp_cache_built_at"}) +#: re-cache (identical cells) or a relocation, which must not change the hash; the exact +#: ``gt_matrix_sha256`` because GPU jitter moves it too (audit / ``--verify`` info only). +_AUDIT_FIT_KEYS = frozenset({"gt_path", "gt_cache_dir", "cp_cache_path", "cp_cache_built_at", "gt_matrix_sha256"}) def _hashed_fit(fit: dict[str, Any]) -> dict[str, Any]: @@ -183,10 +198,11 @@ def gt_matrix_sha256(gt: np.ndarray) -> str: """Return a sha256 over a stacked GT cell matrix. The digest covers the shape and the float64 C-order bytes, so it changes with - any cell value or the cell count. It is recorded per dataset in the reference's - ``fit`` section; the eval recomputes it over the GT cells it stages and refuses to - score on a mismatch (:meth:`DatasetCPSpace.check_gt_cells`), and - ``build_cp_reference.py --verify`` recomputes it from the current caches. + any cell value (GPU jitter of ~1e-15 included) or the cell count. It is recorded + per dataset in the reference's ``fit`` section for audit, and + ``build_cp_reference.py --verify`` reports an exact mismatch as information. It + gates nothing: the eval's content gate compares tolerance-checked moments + (:func:`gt_moments_mismatch`). Parameters ---------- @@ -204,6 +220,61 @@ def gt_matrix_sha256(gt: np.ndarray) -> str: return digest.hexdigest() +def gt_moments(cells: np.ndarray) -> dict[str, list[float]]: + """Return the raw per-feature GT moments the content gate compares. + + Parameters + ---------- + cells : np.ndarray + ``(n_cells, n_features)`` GT-finite CP cells, all columns. + + Returns + ------- + dict + ``{"mean": [...], "std": [...]}`` over axis 0, float64, one entry per column. + """ + arr = np.asarray(cells, dtype=np.float64) + return {"mean": [float(v) for v in arr.mean(axis=0)], "std": [float(v) for v in arr.std(axis=0)]} + + +def gt_moments_mismatch( + cells: np.ndarray, mean_fit: np.ndarray, std_fit: np.ndarray, feature_names: tuple[str, ...] +) -> list[str]: + """Return the features whose staged GT moments are outside the gate tolerance. + + Criterion per feature: ``|mean - mean_fit| <= GT_MOMENT_RTOL * std_fit`` and + ``|std / std_fit - 1| <= GT_MOMENT_RTOL``; for ``std_fit == 0`` both + ``|mean - mean_fit|`` and ``std`` must be ``<= GT_MOMENT_ZERO_STD_ATOL * (1 + |mean_fit|)``. + + Parameters + ---------- + cells : np.ndarray + Staged ``(n_cells, n_features)`` GT-finite CP cells. + mean_fit, std_fit : np.ndarray + The recorded raw moments of the fit cells. + feature_names : tuple of str + Column names, for the report. + + Returns + ------- + list of str + ``"name: mean a vs b, std c vs d"`` for every failing feature (empty = pass). + """ + now = gt_moments(cells) + mean, std = np.asarray(now["mean"]), np.asarray(now["std"]) + constant = std_fit == 0 + safe = np.where(constant, 1.0, std_fit) + ok_var = (np.abs(mean - mean_fit) <= GT_MOMENT_RTOL * std_fit) & (np.abs(std / safe - 1) <= GT_MOMENT_RTOL) + zero_tol = GT_MOMENT_ZERO_STD_ATOL * (1 + np.abs(mean_fit)) + ok_const = (np.abs(mean - mean_fit) <= zero_tol) & (std <= zero_tol) + ok = np.where(constant, ok_const, ok_var) + return [ + f"{name}: mean {mean[i]:.9g} vs {mean_fit[i]:.9g}, std {std[i]:.9g} vs {std_fit[i]:.9g}" + for i, name in enumerate(feature_names) + if not ok[i] + ] + + @dataclass class DatasetFit: """One test set's GT cells and the record of where they came from. @@ -256,6 +327,43 @@ def canonical_gt_matrix(blocks: dict[tuple[str, int], np.ndarray], n_features: i return np.concatenate(rows, axis=0) if rows else np.empty((0, n_features)) +def _read_gt_cp_blocks( + ctx, gt_path: Path, n_features: int +) -> tuple[dict[tuple[str, int], np.ndarray], dict[str, Any], list[str]]: + """Read the GT-finite CP rows of every cached ``(position, t)`` of a GT store. + + Returns ``(blocks, record, missing)``: ``missing`` lists the ``pos/t`` slots the + cache does not hold. Raises on a malformed store or slot (non-3-D position, + wrong column count, no CP cache at all). + """ + with open_ome_zarr(gt_path, mode="r") as plate: + shapes = {name: (int(pos.data.shape[0]), int(pos.data.shape[2])) for name, pos in plate.positions()} + blocks: dict[tuple[str, int], np.ndarray] = {} + missing: list[str] = [] + n_timepoints = n_dropped = 0 + with open_features_group(ctx.paths, "cp", mode="r") as group: + if group is None: + raise StaleCacheError(f"no CP feature cache at {ctx.paths.cp_features()}") + for pos_name, (t_count, z_depth) in sorted(shapes.items()): + if z_depth < 2: + raise ValueError(f"{gt_path}/{pos_name} has Z={z_depth}; CP regionprops are 3-D") + for t in range(t_count): + feats = read_features_from_group(group, pos_name, t) + if feats is None: + missing.append(f"{pos_name}/t{t}") + continue + n_timepoints += 1 + if feats.shape[0] == 0: + continue + if feats.shape[1] != n_features: + raise StaleCacheError(f"{pos_name}/t{t} has {feats.shape[1]} CP columns, expected {n_features}") + finite = np.isfinite(feats).all(axis=1) + n_dropped += int((~finite).sum()) + blocks[(pos_name, t)] = feats[finite] + record = {"positions": sorted(shapes), "n_timepoints": n_timepoints, "n_cells_dropped_nonfinite": n_dropped} + return blocks, record, missing + + def read_gt_cp_cells(ctx, gt_path: Path, n_features: int) -> tuple[np.ndarray, dict[str, Any]]: """Read every finite GT CP cell of one GT store from its cache, read-only, in canonical order. @@ -282,29 +390,9 @@ def read_gt_cp_cells(ctx, gt_path: Path, n_features: int) -> tuple[np.ndarray, d ValueError If a GT position is not a 3-D volume. """ - with open_ome_zarr(gt_path, mode="r") as plate: - shapes = {name: (int(pos.data.shape[0]), int(pos.data.shape[2])) for name, pos in plate.positions()} - blocks: dict[tuple[str, int], np.ndarray] = {} - n_timepoints = n_dropped = 0 - with open_features_group(ctx.paths, "cp", mode="r") as group: - if group is None: - raise StaleCacheError(f"no CP feature cache at {ctx.paths.cp_features()}") - for pos_name, (t_count, z_depth) in sorted(shapes.items()): - if z_depth < 2: - raise ValueError(f"{gt_path}/{pos_name} has Z={z_depth}; CP regionprops are 3-D") - for t in range(t_count): - feats = read_features_from_group(group, pos_name, t) - if feats is None: - raise StaleCacheError(f"CP cache miss at {pos_name}/t{t} in {ctx.paths.cp_features()}") - n_timepoints += 1 - if feats.shape[0] == 0: - continue - if feats.shape[1] != n_features: - raise StaleCacheError(f"{pos_name}/t{t} has {feats.shape[1]} CP columns, expected {n_features}") - finite = np.isfinite(feats).all(axis=1) - n_dropped += int((~finite).sum()) - blocks[(pos_name, t)] = feats[finite] - record = {"positions": sorted(shapes), "n_timepoints": n_timepoints, "n_cells_dropped_nonfinite": n_dropped} + blocks, record, missing = _read_gt_cp_blocks(ctx, gt_path, n_features) + if missing: + raise StaleCacheError(f"CP cache miss at {missing[0]} in {ctx.paths.cp_features()} ({len(missing)} missing)") return canonical_gt_matrix(blocks, n_features), record @@ -375,6 +463,7 @@ def _provenance(fit: DatasetFit) -> dict[str, Any]: **fit.record, "in_mask_fit": fit.in_mask_fit, "n_cells": int(fit.cells.shape[0]), + "gt_moments": gt_moments(fit.cells), "gt_matrix_sha256": gt_matrix_sha256(fit.cells), } @@ -486,7 +575,8 @@ class DatasetCPSpace: std: np.ndarray fit_positions: frozenset[str] fit_n_cells: int - gt_matrix_sha256: str + gt_mean: np.ndarray + gt_std: np.ndarray cp_identity: dict[str, Any] criteria: dict[str, Any] floored_features: tuple[str, ...] @@ -499,7 +589,8 @@ def binding_sha256(self) -> str: It covers the CP recipe identity, the selection criteria, the feature names and keep-mask, THIS dataset's scaler (mean, std, floored features -- the parent's for a lite set, plus the lite -> parent link), and this - dataset's recorded GT-matrix sha256 (a lite set's own). It deliberately + dataset's recorded GT cell count and raw moments (a lite set's own) -- + not the exact GT-matrix sha256, so GPU jitter never moves it. It deliberately leaves out the whole-reference hash, so adding or refitting another dataset of the target does not invalidate this dataset's cached eval dirs (whose cache is all-or-nothing, pixel and mask metrics included), while another @@ -523,7 +614,8 @@ def binding_sha256(self) -> str: "std": [float(v) for v in self.std], "floored_features": list(self.floored_features), }, - "gt_matrix_sha256": self.gt_matrix_sha256, + "gt_n_cells": self.fit_n_cells, + "gt_moments": {"mean": [float(v) for v in self.gt_mean], "std": [float(v) for v in self.gt_std]}, } return hashlib.sha256(json.dumps(body, sort_keys=True, separators=(",", ":")).encode()).hexdigest() @@ -572,15 +664,26 @@ def check_positions(self, positions: list[str]) -> None: "compute_feature_metrics=false for smoke runs." ) + def gt_cells_problems(self, cells: np.ndarray) -> list[str]: + """Return why a canonical GT matrix fails the content gate (empty = it passes). + + Shared by the eval (:meth:`check_gt_cells`) and ``build_cp_reference.py + --verify``: exact cell count, then per-feature moments within tolerance. + """ + if cells.shape[0] != self.fit_n_cells: + return [f"{cells.shape[0]} GT cells vs {self.fit_n_cells} in the fit"] + return gt_moments_mismatch(cells, self.gt_mean, self.gt_std, self.feature_names) + def check_gt_cells(self, blocks: dict[tuple[str, int], np.ndarray]) -> None: - """Refuse to score unless the eval's GT CP cells are exactly the reference's fit cells. + """Refuse to score unless the eval's GT CP cells are the reference's fit cells. - The content gate. It hashes the GT cells this run actually staged -- after - any GT CP recompute -- in the builder's canonical form - (:func:`canonical_gt_matrix`) and compares with the recorded GT-matrix - sha256. So a GT re-cache that leaves the cells identical (``--overwrite``, - ``force_recompute.gt_cp``/``all``) passes, and any value or count change - fails, however the cache manifest was merged. + The content gate, on the GT cells this run actually staged (after any GT CP + recompute): the GT-finite cell count must equal the fit's exactly, and every + CP feature's raw mean/std must match the recorded moments within + :data:`GT_MOMENT_RTOL` (:func:`gt_moments_mismatch`). So a GT re-cache that + reproduces the cells -- on CPU bit-exactly, on GPU up to ~1e-15 jitter + (``--overwrite``, ``force_recompute.gt_cp``/``all``) -- passes, while a + value or count change fails, however the cache manifest was merged. Parameters ---------- @@ -591,16 +694,14 @@ def check_gt_cells(self, blocks: dict[tuple[str, int], np.ndarray]) -> None: Raises ------ StaleCacheError - If the hash differs from the recorded one. + If the count differs or any feature's moments are out of tolerance. """ - cells = canonical_gt_matrix(blocks, len(self.feature_names)) - sha256 = gt_matrix_sha256(cells) - if sha256 != self.gt_matrix_sha256: + problems = self.gt_cells_problems(canonical_gt_matrix(blocks, len(self.feature_names))) + if problems: raise StaleCacheError( - f"{self.dataset}: the GT CP cells this eval scores ({cells.shape[0]} cells, sha256 {sha256[:12]}) " - f"differ from the CP reference fit ({self.fit_n_cells} cells, sha256 {self.gt_matrix_sha256[:12]}) " - f"in {self.reference_path}. The GT CP cache changed since the reference was built; rebuild it:\n" - f" {_build_command(self.target_name)} --force" + f"{self.dataset}: the GT CP cells this eval scores differ from the CP reference fit in " + f"{self.reference_path}: {'; '.join(problems[:5])}. The GT CP cache changed since the reference " + f"was built; rebuild it:\n {_build_command(self.target_name)} --force" ) @@ -650,7 +751,8 @@ def for_dataset(self, dataset: str) -> DatasetCPSpace: std=np.asarray(scaler["std"], dtype=np.float64), fit_positions=frozenset(own["positions"]), fit_n_cells=int(own["n_cells"]), - gt_matrix_sha256=own["gt_matrix_sha256"], + gt_mean=np.asarray(own["gt_moments"]["mean"], dtype=np.float64), + gt_std=np.asarray(own["gt_moments"]["std"], dtype=np.float64), cp_identity=self.cp_identity, criteria=self.criteria, floored_features=tuple(scaler["floored_features"]), diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py index e787c9fe2..d25dc92ea 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -222,9 +222,13 @@ def test_changed_scaler_changes_the_hash() -> None: def test_hand_edited_gate_field_fails_the_load(tmp_path: Path) -> None: - """Editing a recorded GT-matrix hash without re-hashing is refused at load, not trusted by the gate.""" + """Editing a recorded gate moment without re-hashing is refused at load; the exact GT sha is audit only.""" payload = fit_cp_reference([_fit("set-a", _gt())], target_name="er", feature_names=_NAMES, cp_identity={}) - payload["fit"]["datasets"]["set-a"]["gt_matrix_sha256"] = "0" * 64 + audit = json.loads(json.dumps(payload)) + audit["fit"]["datasets"]["set-a"]["gt_matrix_sha256"] = "0" * 64 + (tmp_path / "er.json").write_text(json.dumps(audit)) + load_cp_reference(tmp_path / "er.json", target_name="er") # audit-only field: not hashed + payload["fit"]["datasets"]["set-a"]["gt_moments"]["mean"][0] += 1.0 (tmp_path / "er.json").write_text(json.dumps(payload)) with pytest.raises(ValueError, match="does not match its recorded sha256"): load_cp_reference(tmp_path / "er.json", target_name="er") @@ -290,17 +294,18 @@ def test_positions_must_match_the_fit(two_sets) -> None: def test_content_gate_passes_identical_cells_and_refuses_a_value_change(two_sets) -> None: - """The gate hashes the staged GT cells: an identical re-cache passes, a same-count value change raises.""" + """Identical cells, or cells jittered at GPU level, pass; a same-count value change or a count change raises.""" ref, _, cells = two_sets space = ref.for_dataset("set-a") gt = cells["set-a"] space.check_gt_cells(_blocks(gt.copy())) # e.g. force_recompute.gt_cp reproducing the same cells space.check_gt_cells(dict(reversed(list(_blocks(gt).items())))) # staging order does not matter + space.check_gt_cells(_blocks(gt * (1 + 1e-12))) # far above GPU jitter (~1e-15), still passes changed = gt.copy() - changed[5, 2] += 1e-6 # same count, one value moved + changed[5, 2] += 1.0 # same count, one cell moved by ~1/3 sd with pytest.raises(StaleCacheError, match="(?s)differ from the CP reference fit .*--target er --force"): space.check_gt_cells(_blocks(changed)) - with pytest.raises(StaleCacheError, match="299 cells"): + with pytest.raises(StaleCacheError, match="299 GT cells vs 300"): space.check_gt_cells(_blocks(gt[:-1])) lite = ref.for_dataset("set-a-lite") lite.check_gt_cells(_blocks(gt[:_LITE_ROWS])) @@ -449,10 +454,11 @@ def rebound(edit) -> dict[str, str]: return {d: again.for_dataset(d).binding_sha256 for d in bindings} assert rebound(lambda p: p["fit"]["datasets"]["set-a"].update(cp_cache_built_at="t1")) == bindings - gt = rebound(lambda p: p["fit"]["datasets"]["set-a"].update(gt_matrix_sha256="0" * 64)) + assert rebound(lambda p: p["fit"]["datasets"]["set-a"].update(gt_matrix_sha256="0" * 64)) == bindings + gt = rebound(lambda p: p["fit"]["datasets"]["set-a"]["gt_moments"]["mean"].__setitem__(0, 1.0)) assert gt["set-a"] != bindings["set-a"] assert gt["set-b"] == bindings["set-b"] and gt["set-a-lite"] == bindings["set-a-lite"] - lite_gt = rebound(lambda p: p["fit"]["lite"]["set-a-lite"].update(gt_matrix_sha256="0" * 64)) + lite_gt = rebound(lambda p: p["fit"]["lite"]["set-a-lite"]["gt_moments"]["std"].__setitem__(0, 1.0)) assert lite_gt["set-a-lite"] != bindings["set-a-lite"] and lite_gt["set-a"] == bindings["set-a"] diff --git a/applications/dynacell/tests/test_evaluation_cp_end_to_end.py b/applications/dynacell/tests/test_evaluation_cp_end_to_end.py index 47c55772e..6e73b2f32 100644 --- a/applications/dynacell/tests/test_evaluation_cp_end_to_end.py +++ b/applications/dynacell/tests/test_evaluation_cp_end_to_end.py @@ -14,11 +14,13 @@ from __future__ import annotations +import tempfile import zlib from pathlib import Path import numpy as np import pytest +import torch from iohub.ngff import open_ome_zarr from dynacell.evaluation.cache import ( @@ -36,8 +38,12 @@ _POSITIONS = [f"A/1/{i}" for i in range(N_POSITIONS)] -def _segmentation() -> np.ndarray: - """``(T, D, H, W)`` labels: a 4 x 4 grid of 6 x 6 x 6 cells plus one single-voxel cell.""" +def _segmentation(single_voxel: bool = True) -> np.ndarray: + """``(T, D, H, W)`` labels: a 4 x 4 grid of 6 x 6 x 6 cells, plus (optionally) one single-voxel cell. + + The GPU variants leave the single voxel out: cuCIM's GPU ``regionprops_table`` + raises a TypeError on a one-voxel region (a cuCIM limitation, not this code's). + """ seg = np.zeros((T, D, H, W), dtype=np.int32) label = 1 for row in range(4): @@ -45,7 +51,8 @@ def _segmentation() -> np.ndarray: y, x = 1 + 8 * row, 1 + 8 * col seg[:, 2:8, y : y + 6, x : x + 6] = label label += 1 - seg[:, 9, 0, 0] = label # one voxel: skewness/kurtosis are NaN -> a non-finite GT row + if single_voxel: + seg[:, 9, 0, 0] = label # one voxel: skewness/kurtosis are NaN -> a non-finite GT row return seg @@ -90,14 +97,14 @@ def _stub_models() -> EvalModels: class Harness: """GT/seg plates, mask caches, a CP reference fit on the GT CP cache, and a runner.""" - def __init__(self, root: Path, monkeypatch): + def __init__(self, root: Path, monkeypatch, single_voxel: bool = True): self.root = root self.pipeline = live_pipeline_module() from dynacell.evaluation import cp_reference, pipeline_cache self.cp_reference = cp_reference self.gt = [_gt_volume(seed=i) for i in range(N_POSITIONS)] - self.seg = _segmentation() + self.seg = _segmentation(single_voxel) _write_plate(root / "gt.zarr", "target", self.gt) _write_plate(root / "seg.zarr", "cell_segmentation", [self.seg] * N_POSITIONS, dtype=np.int32) make_mask_cache(root / "gt_cache", root / "gt.zarr", "target", side="gt") @@ -140,11 +147,12 @@ def config(self, pred_path: Path, save_dir: Path): config.benchmark = {"dataset_ref": {"dataset": _DATASET, "target": "sec61b"}} return config - def run(self, name: str, pred: list[np.ndarray], **overrides) -> tuple[dict, object]: + def run(self, name: str, pred: list[np.ndarray], use_gpu: bool = False, **overrides) -> tuple[dict, object]: """Evaluate ``pred`` against the GT; return the dataset-level feature row and the config.""" pred_path = self.root / f"{name}.zarr" _write_plate(pred_path, "prediction", pred) config = self.config(pred_path, self.root / name) + config.use_gpu = use_gpu make_mask_cache(Path(config.io.pred_cache_dir), pred_path, "prediction", side="pred") for key, value in overrides.items(): config.force_recompute[key] = value @@ -165,6 +173,54 @@ def test_identical_gt_recache_passes_the_content_gate(harness: Harness) -> None: assert np.isfinite(row["Dataset_CP_KID"]) +def test_gpu_level_jitter_in_every_gt_cell_passes(harness: Harness) -> None: + """Every cached GT CP cell scaled by (1 + 1e-12) -- far above GPU regionprops jitter -- still scores.""" + with open_features_group(cache_paths(harness.root / "gt_cache"), "cp", mode="a") as group: + for pos in _POSITIONS: + for t in range(T): + write_features_to_group(group, pos, t, np.asarray(group[f"{pos}/t{t}"]) * (1 + 1e-12)) + row, _ = harness.run("jitter", [g.copy() for g in harness.gt]) + assert np.isfinite(row["Dataset_CP_KID"]) + + +_CUDA = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA GPU") + + +@_CUDA +def test_gpu_recompute_of_the_gt_passes_the_content_gate(tmp_path: Path, monkeypatch) -> None: + """``force_recompute.gt_cp`` with ``use_gpu=True`` against the CPU-built reference scores (no exact-bit gate).""" + harness = Harness(tmp_path, monkeypatch, single_voxel=False) + row, _ = harness.run("gpu_recache", [g.copy() for g in harness.gt], use_gpu=True, gt_cp=True) + assert np.isfinite(row["Dataset_CP_KID"]) + + +@_CUDA +def test_two_gpu_recomputes_of_one_block_pass_the_gate() -> None: + """One real-shaped block recomputed twice on the GPU: whatever the bit-level jitter, the gate passes.""" + from dynacell.evaluation.cp_reference import DatasetFit, fit_cp_reference, write_cp_reference + from dynacell.evaluation.cp_reference import load_cp_reference as load + from dynacell.evaluation.metrics import active_cp_feature_names, cp_regionprops + + image, seg = _gt_volume(seed=3)[0], _segmentation(single_voxel=False)[0] + runs = [ + cp_regionprops( + image, seg, [1.0, 1.0, 1.0], norm={"p_lo": 1.0, "p_hi": 99.0}, glcm_cfg={"enabled": True}, use_gpu=True + ) + for _ in range(2) + ] + finite = [r[np.isfinite(r).all(axis=1)] for r in runs] + names = active_cp_feature_names(True) + fit = DatasetFit(dataset="gpu", cells=finite[0], record={"positions": ["A/1/0"]}, in_mask_fit=True) + payload = fit_cp_reference([fit], target_name="er", feature_names=names, cp_identity={}) + path = Path(tempfile.mkdtemp()) / "er.json" + write_cp_reference(payload, path) + load(path, target_name="er").for_dataset("gpu").check_gt_cells({("A/1/0", 0): finite[1]}) + print( + f"GPU runs bit-identical: {np.array_equal(runs[0], runs[1], equal_nan=True)}, " + f"max |diff| {np.nanmax(np.abs(runs[0] - runs[1])):.3g}" + ) + + def test_same_count_gt_value_change_is_refused(harness: Harness) -> None: """A GT CP cache whose values moved (same cell count) since the fit fails before any metric is written.""" with open_features_group(cache_paths(harness.root / "gt_cache"), "cp", mode="a") as group: diff --git a/applications/dynacell/tests/test_evaluation_timepoint_guard.py b/applications/dynacell/tests/test_evaluation_timepoint_guard.py index 5c6194f14..42310b816 100644 --- a/applications/dynacell/tests/test_evaluation_timepoint_guard.py +++ b/applications/dynacell/tests/test_evaluation_timepoint_guard.py @@ -228,7 +228,7 @@ def test_cache_is_invalid_once_the_gt_matrix_changed(tmp_path: Path) -> None: pipeline = live_pipeline_module() config, reference = _two_set_cache(pipeline, tmp_path) payload = json.loads(reference.read_text()) - payload["fit"]["datasets"]["set-a"]["gt_matrix_sha256"] = "0" * 64 + payload["fit"]["datasets"]["set-a"]["gt_moments"]["mean"][0] += 1.0 payload["sha256"] = payload_sha256(payload) # a legitimate rebuild re-hashes reference.write_text(json.dumps(payload)) assert not pipeline._final_metrics_cache_valid(config) diff --git a/applications/dynacell/tools/build_cp_reference.py b/applications/dynacell/tools/build_cp_reference.py index 39081d6cf..966528c5e 100644 --- a/applications/dynacell/tools/build_cp_reference.py +++ b/applications/dynacell/tools/build_cp_reference.py @@ -189,7 +189,12 @@ def build(target_name: str, leaves_root: Path) -> dict[str, Any]: def verify(target_name: str, path: Path) -> list[str]: - """Recompute every dataset's (lite included) GT-matrix sha256 from the caches and compare. + """Re-read every dataset's (lite included) GT CP cells from the caches and apply the content gate. + + The same tolerance gate the eval applies (exact cell count, per-feature GT + moments within tolerance; ``DatasetCPSpace.gt_cells_problems``). An exact + GT-matrix sha256 difference alone is reported as information: GPU + recomputes move it by ~1e-15 without changing any number that matters. Parameters ---------- @@ -201,21 +206,18 @@ def verify(target_name: str, path: Path) -> list[str]: Returns ------- list of str - One message per dataset whose recomputed sha256 or cell count differs from - the recorded one; empty when all match. + One message per dataset that fails the gate; empty when all pass. """ ref = load_cp_reference(path, target_name=target_name) mismatches = [] for name, record in sorted({**ref.fit["datasets"], **ref.fit["lite"]}.items()): fit = read_dataset_fit(target_name, (name, record["target"]), ref.feature_names, record["in_mask_fit"]) + problems = ref.for_dataset(name).gt_cells_problems(fit.cells) sha256 = gt_matrix_sha256(fit.cells) - ok = sha256 == record["gt_matrix_sha256"] and fit.cells.shape[0] == record["n_cells"] - print(f" {name}: {'OK' if ok else 'MISMATCH'} ({fit.cells.shape[0]} cells, sha256 {sha256[:12]})") - if not ok: - mismatches.append( - f"{name}: recorded {record['n_cells']} cells sha256 {record['gt_matrix_sha256'][:12]}, " - f"caches now give {fit.cells.shape[0]} cells sha256 {sha256[:12]}" - ) + exact = "exact sha256 match" if sha256 == record["gt_matrix_sha256"] else "info: exact sha256 differs" + print(f" {name}: {'OK' if not problems else 'MISMATCH'} ({fit.cells.shape[0]} cells; {exact})") + if problems: + mismatches.append(f"{name}: {'; '.join(problems[:5])}") return mismatches From bb7b4aba7d54332708520f322f95e9202092e2e7 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 21:34:23 -0700 Subject: [PATCH 33/41] fix(dynacell/eval): run the CP content gate before the FOV loop when the GT cache is complete The gate ran only after the FOV loop, so a doomed run (GT changed since the fit) spent the whole loop first (ordering nit, #499 re-review). When the GT CP cache already holds every slot and this run does not recompute GT CP, the cells the loop will score are known up front. The gate is applied to them right after the position check. The post-staging gate stays authoritative (for recomputes and partially cached stores). A test shows a stale complete cache failing before any FOV is processed. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/cp_reference.py | 17 +++++++++++++++++ .../src/dynacell/evaluation/pipeline.py | 15 ++++++++++++++- .../tests/test_evaluation_cp_end_to_end.py | 12 ++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index 1ba61eb1b..b22ccc13f 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -396,6 +396,23 @@ def read_gt_cp_cells(ctx, gt_path: Path, n_features: int) -> tuple[np.ndarray, d return canonical_gt_matrix(blocks, n_features), record +def complete_cached_gt_cp_blocks(ctx, gt_path: Path, n_features: int) -> dict[tuple[str, int], np.ndarray] | None: + """Return the GT-finite CP blocks of a GT store when its CP cache holds every slot, else ``None``. + + Lets the eval run the content gate before its FOV loop when the GT CP cache is + already complete (the common re-eval case), so a doomed run fails in seconds + rather than after the loop. ``None`` for a disabled cache or any missing slot: + the post-staging gate is then the only (and always the authoritative) check. + """ + if not ctx.enabled: + return None + with open_features_group(ctx.paths, "cp", mode="r") as group: + if group is None: + return None + blocks, _, missing = _read_gt_cp_blocks(ctx, gt_path, n_features) + return None if missing else blocks + + def fit_cp_reference( fits: list[DatasetFit], *, diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline.py b/applications/dynacell/src/dynacell/evaluation/pipeline.py index 23c6aa027..95ad8e2c3 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline.py @@ -18,7 +18,12 @@ from dynacell.evaluation._ref_hook import apply_dataset_ref from dynacell.evaluation.cache import FeatureKind -from dynacell.evaluation.cp_reference import CP_SIDECAR_FILENAME, DatasetCPSpace, eval_cp_space +from dynacell.evaluation.cp_reference import ( + CP_SIDECAR_FILENAME, + DatasetCPSpace, + complete_cached_gt_cp_blocks, + eval_cp_space, +) from dynacell.evaluation.cross_condition_probe import run_for_group as _cross_condition_run_for_group from dynacell.evaluation.feature_metrics import ( compute_feature_similarity, @@ -1528,6 +1533,14 @@ def _keep(name: str) -> bool: # before any per-FOV work (limit_positions / exclude_fov_names on a non-lite dataset). if cp_space is not None: cp_space.check_positions([name for name, _ in gt_positions]) + # Early content gate: when the GT CP cache already holds every slot and this run + # will not recompute it, the cells the loop will score are known now, so a GT + # that no longer matches the fit fails in seconds instead of after the loop. The + # post-staging gate in _stage_cp_dataset_inputs stays the authoritative check. + if not cache_ctx.force["gt_cp"]: + cached = complete_cached_gt_cp_blocks(cache_ctx, gt_path, len(cp_space.feature_names)) + if cached is not None: + cp_space.check_gt_cells(cached) # Leaf-level MicroMS3IM calibration: fit α once on a random # subsample of (FOV, t) volumes and reuse the fitted sim for diff --git a/applications/dynacell/tests/test_evaluation_cp_end_to_end.py b/applications/dynacell/tests/test_evaluation_cp_end_to_end.py index 6e73b2f32..dea79ead7 100644 --- a/applications/dynacell/tests/test_evaluation_cp_end_to_end.py +++ b/applications/dynacell/tests/test_evaluation_cp_end_to_end.py @@ -287,3 +287,15 @@ def test_cp_kid_registers_offset_and_contraction_end_to_end(harness: Harness, na old = _old_per_side_kid(Path(config.save.save_dir), harness.pipeline.eval_cp_space(config)) assert kid > floor + 1.0, (kid, floor) assert abs(old - floor) < 0.1 * (kid - floor), (old, floor, kid) + + +def test_a_complete_stale_gt_cache_fails_before_the_fov_loop(harness: Harness, monkeypatch) -> None: + """With every GT CP slot cached and no GT recompute, the content gate runs before any FOV is scored.""" + with open_features_group(cache_paths(harness.root / "gt_cache"), "cp", mode="a") as group: + feats = np.asarray(group["A/1/0/t0"]) + feats[0, 0] += 0.5 + write_features_to_group(group, "A/1/0", 0, feats) + monkeypatch.setattr(harness.pipeline, "_process_one_fov", lambda *a, **k: pytest.fail("reached the FOV loop")) + with pytest.raises(Exception, match="differ from the CP reference fit") as err: + harness.run("early", [g.copy() for g in harness.gt]) + assert type(err.value).__name__ == "StaleCacheError" From 4076b2848a37d8a5567b448358a9b97415a931cd Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 21:36:28 -0700 Subject: [PATCH 34/41] fix(dynacell/eval): have eval-dir sidecars pin their dataset's CP space, not the reference The cp_selected_feature_mask.json sidecar pinned the whole-reference sha256, while the final-metrics stamp is per dataset. After any rebuild (a dataset added, a --force for F1), cache-valid dirs kept their old sidecar and sidecar_cp_space raised. Since the probe no longer swallows errors, that failed every grouped re-run with a reused condition, and the lite tools too (F2, #499 re-review). The sidecar now records reference_path, dataset and that dataset's cp_space_sha256 (binding), built by one cp_sidecar_payload shared by the pipeline and the tests. The whole-reference sha is kept as audit. sidecar_cp_space loads the current reference, binds the recorded dataset and compares bindings. The cross-condition probe requires one reference path and one mask across its dirs. Test: an old sidecar loads after a rebuild that adds an unrelated dataset, and raises once its own scaler changes. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/cp_reference.py | 49 +++++++++++++++---- .../dynacell/evaluation/cp_reference_test.py | 24 +++++++++ .../evaluation/cross_condition_probe.py | 8 +-- .../src/dynacell/evaluation/pipeline.py | 12 +---- .../tests/test_cross_condition_probe.py | 26 +++++++--- .../dynacell/tools/lite_checks_test.py | 10 +++- .../dynacell/tools/lite_subset_sim_test.py | 14 ++++-- 7 files changed, 105 insertions(+), 38 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index b22ccc13f..951226ea5 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -832,14 +832,40 @@ def load_cp_reference(path: Path, *, target_name: str) -> CPReference: return ref +def cp_sidecar_payload(space: DatasetCPSpace) -> dict[str, Any]: + """Return the :data:`CP_SIDECAR_FILENAME` payload an eval writes for the space it scored in. + + ``reference_path`` + ``dataset`` + ``cp_space_sha256`` are what + :func:`sidecar_cp_space` reads back; the rest (whole-reference sha256, scaler + dataset, mask) is audit. + """ + return { + "reference_path": space.reference_path, + "dataset": space.dataset, + # What post-hoc readers compare: this dataset's CP space, which survives rebuilds + # that leave it unchanged. The whole-reference hash is audit only. + "cp_space_sha256": space.binding_sha256, + "reference_sha256": space.reference_sha256, + "scaler_dataset": space.scaler_dataset, + "feature_names": list(space.feature_names), + "keep_mask": [bool(b) for b in space.keep_mask], + "n_kept": int(space.keep_mask.sum()), + "n_total": int(space.keep_mask.size), + } + + def sidecar_cp_space(eval_dir: Path) -> DatasetCPSpace: """Return the CP space a finished eval dir was scored in, from its :data:`CP_SIDECAR_FILENAME`. - The sidecar records the reference path, its sha256 and the eval's dataset. The - reference is loaded from that path and must still carry that sha256; it is then - bound to the recorded dataset, so a lite eval gets its parent's scaler exactly as - the pipeline did. Used by post-hoc consumers of eval dirs (the cross-condition - probe, the lite subset tools) so they score CP the way the eval did. + The sidecar records the reference path, the eval's dataset and that dataset's + ``cp_space_sha256`` (:attr:`DatasetCPSpace.binding_sha256`). The current + reference at that path is loaded and bound to the recorded dataset (a lite eval + gets its parent's scaler exactly as the pipeline did), and its binding must equal + the recorded one. A rebuild that leaves this dataset's CP space unchanged (another + dataset added or refit, an audit field moved) therefore keeps the sidecar + readable; one that changes it is refused. Used by post-hoc consumers of eval dirs + (the cross-condition probe, the lite subset tools) so they score CP the way the + eval did. Parameters ---------- @@ -856,17 +882,20 @@ def sidecar_cp_space(eval_dir: Path) -> DatasetCPSpace: FileNotFoundError If the sidecar or the reference it names is missing. ValueError - If the reference on disk no longer has the sha256 the eval recorded. + If the dataset's CP space in the current reference differs from the one the + eval recorded. """ sidecar = json.loads((eval_dir / CP_SIDECAR_FILENAME).read_text()) path = Path(sidecar["reference_path"]) ref = _read_cp_reference(path, build_hint="the reference this eval was scored in") - if ref.sha256 != sidecar["reference_sha256"]: + space = ref.for_dataset(sidecar["dataset"]) + if space.binding_sha256 != sidecar["cp_space_sha256"]: raise ValueError( - f"{eval_dir}: CP reference {path} changed since the eval " - f"(sha256 {ref.sha256[:12]} now, {sidecar['reference_sha256'][:12]} at eval time)" + f"{eval_dir}: the CP space of {sidecar['dataset']} in {path} changed since the eval " + f"(cp_space_sha256 {space.binding_sha256[:12]} now, {sidecar['cp_space_sha256'][:12]} at eval time); " + "re-evaluate this dir" ) - return ref.for_dataset(sidecar["dataset"]) + return space def resolve_cp_reference_path(config: DictConfig) -> Path: diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py index d25dc92ea..bfe4ee7bd 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -524,3 +524,27 @@ def test_cosine_is_blind_to_contraction_toward_the_gt_mean_but_kid_is_not(two_se floor = compute_feature_similarity_pairwise(space.transform(gt), space.transform(gt), "CP", compute_fid=False) assert got["CP_Median_Cosine_Similarity"] == pytest.approx(1.0, abs=1e-6) assert got["CP_KID"] > floor["CP_KID"] + 0.1 + + +def test_old_sidecar_survives_a_rebuild_that_leaves_its_dataset_unchanged(tmp_path: Path) -> None: + """A sidecar pins its dataset's CP space, not the whole reference. + + Adding an unrelated dataset (a new HEK scaler, outside the mask fit) rebuilds the + reference with a new sha256 but leaves set-a's binding, so the old sidecar still + loads; refitting set-a's own scaler makes it raise. + """ + a, b = _gt(), _gt(seed=1) + base = [_fit("set-a", a), _fit("set-b", b)] + ref, _ = _reference(tmp_path, base) + eval_dir = tmp_path / "eval" + eval_dir.mkdir() + _stage_cp_dataset_inputs(_lists(a, a), ref.for_dataset("set-a"), _blocks(a), eval_dir) + old_sha = ref.sha256 + + added, _ = _reference(tmp_path, [*base, _fit("hek", _gt(seed=2, offset=-5.0), in_mask_fit=False)]) + assert added.sha256 != old_sha + assert sidecar_cp_space(eval_dir).reference_sha256 == added.sha256 # bound to the current reference + + _reference(tmp_path, [_fit("set-a", a * 1.1), _fit("set-b", b)]) # set-a's scaler moves + with pytest.raises(ValueError, match="CP space of set-a .* changed since the eval"): + sidecar_cp_space(eval_dir) diff --git a/applications/dynacell/src/dynacell/evaluation/cross_condition_probe.py b/applications/dynacell/src/dynacell/evaluation/cross_condition_probe.py index 462e05b32..781d948b5 100644 --- a/applications/dynacell/src/dynacell/evaluation/cross_condition_probe.py +++ b/applications/dynacell/src/dynacell/evaluation/cross_condition_probe.py @@ -134,9 +134,11 @@ def _cp_reference_mask(eval_dirs: list[Path]) -> np.ndarray: no longer has the recorded hash. """ spaces = [sidecar_cp_space(d) for d in eval_dirs] - refs = {(s.reference_path, s.reference_sha256) for s in spaces} - if len(refs) != 1: - raise ValueError(f"eval dirs {[str(d) for d in eval_dirs]} were scored in different CP references: {refs}") + paths = {s.reference_path for s in spaces} + if len(paths) != 1 or any(not np.array_equal(s.keep_mask, spaces[0].keep_mask) for s in spaces): + raise ValueError( + f"eval dirs {[str(d) for d in eval_dirs]} were scored in different CP references or masks: {paths}" + ) return spaces[0].keep_mask diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline.py b/applications/dynacell/src/dynacell/evaluation/pipeline.py index 95ad8e2c3..6d0ecd6dc 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline.py @@ -22,6 +22,7 @@ CP_SIDECAR_FILENAME, DatasetCPSpace, complete_cached_gt_cp_blocks, + cp_sidecar_payload, eval_cp_space, ) from dynacell.evaluation.cross_condition_probe import run_for_group as _cross_condition_run_for_group @@ -367,16 +368,7 @@ def _stage_cp_dataset_inputs( cp_space.check_gt_cells(gt_cp_blocks) pred_cp_raw = np.concatenate(cp.pred_feats, axis=0) target_cp_raw = np.concatenate(cp.gt_feats, axis=0) - mask_payload = { - "reference_path": cp_space.reference_path, - "reference_sha256": cp_space.reference_sha256, - "dataset": cp_space.dataset, - "scaler_dataset": cp_space.scaler_dataset, - "feature_names": list(cp_space.feature_names), - "keep_mask": [bool(b) for b in cp_space.keep_mask], - "n_kept": int(cp_space.keep_mask.sum()), - "n_total": int(cp_space.keep_mask.size), - } + mask_payload = cp_sidecar_payload(cp_space) (save_dir / CP_SIDECAR_FILENAME).write_text(json.dumps(mask_payload, indent=2)) return ( "CP", diff --git a/applications/dynacell/tests/test_cross_condition_probe.py b/applications/dynacell/tests/test_cross_condition_probe.py index 67984c31a..1b141d20d 100644 --- a/applications/dynacell/tests/test_cross_condition_probe.py +++ b/applications/dynacell/tests/test_cross_condition_probe.py @@ -12,7 +12,13 @@ import pytest import dynacell.evaluation.cross_condition_probe as probe -from dynacell.evaluation.cp_reference import DatasetFit, fit_cp_reference, write_cp_reference +from dynacell.evaluation.cp_reference import ( + DatasetFit, + cp_sidecar_payload, + fit_cp_reference, + load_cp_reference, + write_cp_reference, +) from dynacell.evaluation.cross_condition_probe import ( _FEATURE_TYPES, GROUP_PROBE_FILENAME, @@ -64,10 +70,15 @@ def _write_group_embeddings( np.savez(emb / f"{source}_{feat}_single_cell_embeddings.npz", embeddings=x, fov=fov, timepoint=tp) # The CP sidecar every eval writes, naming the reference it scored in. reference = eval_dir.parent / "cp_reference.json" - sha256 = _write_reference(reference) if not reference.exists() else json.loads(reference.read_text())["sha256"] - (eval_dir / "cp_selected_feature_mask.json").write_text( - json.dumps({"reference_path": str(reference), "reference_sha256": sha256, "dataset": "ds"}) - ) + if not reference.exists(): + _write_reference(reference) + _write_sidecar(eval_dir, reference) + + +def _write_sidecar(eval_dir: Path, reference: Path) -> None: + """Write the sidecar the pipeline writes for dataset ``ds`` of ``reference``.""" + space = load_cp_reference(reference, target_name="membrane").for_dataset("ds") + (eval_dir / "cp_selected_feature_mask.json").write_text(json.dumps(cp_sidecar_payload(space))) def _read_rows(csv_path: Path) -> list[dict]: @@ -191,8 +202,7 @@ def test_cp_probe_refuses_evals_scored_in_different_references(tmp_path): _write_group_embeddings(mock, seed=1) _write_group_embeddings(denv, seed=2) other = tmp_path / "other.json" - sidecar = json.loads((denv / "cp_selected_feature_mask.json").read_text()) - sidecar.update(reference_path=str(other), reference_sha256=_write_reference(other, seed=9)) - (denv / "cp_selected_feature_mask.json").write_text(json.dumps(sidecar)) + _write_reference(other, seed=9) + _write_sidecar(denv, other) with pytest.raises(ValueError, match="different CP references"): run([mock, denv], tmp_path / "probe.csv") diff --git a/applications/dynacell/tools/lite_checks_test.py b/applications/dynacell/tools/lite_checks_test.py index f42681f9d..34b97f6dc 100644 --- a/applications/dynacell/tools/lite_checks_test.py +++ b/applications/dynacell/tools/lite_checks_test.py @@ -25,7 +25,13 @@ from lite_checks import check_b2, check_e from lite_subset_sim import mmd2_from_features, poly3_features -from dynacell.evaluation.cp_reference import DatasetFit, fit_cp_reference, write_cp_reference +from dynacell.evaluation.cp_reference import ( + DatasetFit, + cp_sidecar_payload, + fit_cp_reference, + load_cp_reference, + write_cp_reference, +) _BUCKET = "a549__mock" _FOVS = [f"0/0/fov{i:04d}" for i in range(12)] @@ -119,7 +125,7 @@ def test_check_e_scores_cp_in_the_reference_space(tmp_path: Path) -> None: for side, arr in (("gt", gt), ("pred", pred)): np.savez(path / "embeddings" / f"{side}_cp_single_cell_embeddings.npz", embeddings=arr, fov=fov) (path / "cp_selected_feature_mask.json").write_text( - json.dumps({"reference_path": str(reference), "reference_sha256": payload["sha256"], "dataset": "ds"}) + json.dumps(cp_sidecar_payload(load_cp_reference(reference, target_name="nucleus").for_dataset("ds"))) ) result = check_e("nucleus", "ipsc", ["fnet3d_paper/ipsc"], tmp_path, sizes=(50,), draws=2, data_root=tmp_path) diff --git a/applications/dynacell/tools/lite_subset_sim_test.py b/applications/dynacell/tools/lite_subset_sim_test.py index d1ac91487..14ab2ac05 100644 --- a/applications/dynacell/tools/lite_subset_sim_test.py +++ b/applications/dynacell/tools/lite_subset_sim_test.py @@ -22,7 +22,13 @@ import pytest from lite_subset_sim import System, load_systems, mmd2_from_features, poly3_features, poly_kernel -from dynacell.evaluation.cp_reference import DatasetFit, fit_cp_reference, write_cp_reference +from dynacell.evaluation.cp_reference import ( + DatasetFit, + cp_sidecar_payload, + fit_cp_reference, + load_cp_reference, + write_cp_reference, +) from dynacell.evaluation.feature_metrics import _kid, _median_cosine_similarity _BLOCKS = [("A/1/0", 0), ("A/1/0", 1), ("A/1/1", 0), ("A/1/1", 1), ("A/1/2", 0), ("A/1/2", 1)] @@ -90,10 +96,8 @@ def _write_eval_dir(path: Path, seed: int, reference: Path) -> dict[str, dict[st path / "embeddings" / f"{side}_{tok}_single_cell_embeddings.npz", embeddings=arr, fov=fov, timepoint=tp ) cells[tok] = {"pred": pred, "gt": gt, "fov": fov, "timepoint": tp} - sha256 = json.loads(reference.read_text())["sha256"] - (path / "cp_selected_feature_mask.json").write_text( - json.dumps({"reference_path": str(reference), "reference_sha256": sha256, "dataset": _DATASET}) - ) + space = load_cp_reference(reference, target_name="nucleus").for_dataset(_DATASET) + (path / "cp_selected_feature_mask.json").write_text(json.dumps(cp_sidecar_payload(space))) return cells From 0c2fb983dc476292756ee238d987386ea119063b Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 21:36:46 -0700 Subject: [PATCH 35/41] docs(dynacell/eval): describe the per-dataset CP binding and tolerance gate accurately The reference_path bullet said a rebuilt reference invalidates the final-metrics cache (the binding is per dataset now), and that an identical re-cache passes an exact-hash gate (false on GPU) (F3, #499 re-review). It now describes the count + moments tolerance gate and its early run on a complete cache, --verify, and the per-dataset cp_space_sha256 in both the provenance stamp and the sidecar. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- applications/dynacell/src/dynacell/evaluation/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/dynacell/src/dynacell/evaluation/README.md b/applications/dynacell/src/dynacell/evaluation/README.md index 9ee3eeae9..afab900c7 100644 --- a/applications/dynacell/src/dynacell/evaluation/README.md +++ b/applications/dynacell/src/dynacell/evaluation/README.md @@ -24,7 +24,7 @@ End-to-end evaluation pipeline for virtual staining predictions against fluoresc - `io.gt_path` — fluorescence ground truth (channel: `io.gt_channel_name`) - `io.cell_segmentation_path` — *optional* precomputed cell segmentation HCS OME-Zarr. Required when `compute_feature_metrics=true` or when building CP/DINOv3/DynaCLR/CELL-DINO cache entries. Position layout must match GT/pred 1:1. - `io.gt_cache_dir`, `io.pred_cache_dir` — *optional* artifact cache directories; must be distinct. See [Caches](#caches). -- `feature_metrics.cp.reference_path` — the target's CP (GLCM+) reference, required when `compute_feature_metrics=true`. `null` (default) resolves to `DATA_ROOT/cp_reference/.json` for full and lite evals alike. The reference holds one feature mask chosen on GT cells only, pooled over iPSC + A549 mock/denv/zikv, and one GT mean/std per test set (keyed by `benchmark.dataset_ref.dataset`; HEK gets its own, lite reuses its parent's). Every eval applies the mask and its dataset's scaler to pred AND GT. The eval refuses a partial position walk, and refuses to score unless its staged GT CP cells hash to the sha256 the reference recorded for that dataset (a content gate: an identical re-cache passes, any value change fails). Build it from existing GT caches with `tools/build_cp_reference.py --target `. Its hash is stamped in `metrics_provenance.json`, and a rebuilt reference invalidates the final-metrics cache. +- `feature_metrics.cp.reference_path` — the target's CP (GLCM+) reference, required when `compute_feature_metrics=true`. `null` (default) resolves to `DATA_ROOT/cp_reference/.json` for full and lite evals alike. The reference holds one feature mask chosen on GT cells only, pooled over iPSC + A549 mock/denv/zikv, and one GT mean/std per test set (keyed by `benchmark.dataset_ref.dataset`; HEK gets its own, lite reuses its parent's). Every eval applies the mask and its dataset's scaler to pred AND GT. The eval refuses a partial position walk, and refuses to score unless its staged GT CP cells match the reference's fit for that dataset: the same GT-finite cell count exactly, and every CP feature's raw GT mean/std within a relative tolerance of 1e-6 (`GT_MOMENT_RTOL`). A re-cache that reproduces the cells passes, including a GPU recompute, which is only reproducible to ~1e-15; a real value change fails. When the GT CP cache is already complete, the check also runs before the FOV loop. Build the reference from existing GT caches with `tools/build_cp_reference.py --target `; `--verify` re-applies the same check to the current caches. `metrics_provenance.json` stamps the per-dataset `cp_space_sha256`: the mask, feature names, recipe, and that dataset's scaler and recorded GT moments. A rebuild therefore invalidates only the eval dirs whose own dataset's CP space changed; adding or refitting another dataset leaves them reusable. The `cp_selected_feature_mask.json` sidecar records the same per-dataset hash. - GLCM+ (CP) space in two lines: one feature mask selected on GT cells only, pooled over the target's iPSC + A549 mock/denv/zikv test sets; then each test set's own GT mean/std standardizes pred and GT alike, so offsets and scale errors relative to that test set's GT stay visible. - GLCM+ median cosine is invariant to contraction toward the GT mean: for `pred = mean + 0.5·(GT − mean)`, every pred/GT cell pair is parallel after the shared scaler and the median cosine is exactly 1.0000, while KID catches it (reviewer-measured on real cells: KID 0.194 vs a pred==GT floor of −0.041; reproduced on synthetic cells in `cp_reference_test.py`). Cosine measures direction only; over-smoothing toward the mean shows up in KID, not cosine. From 72b2464e3bf5763675153923a8d6ae6f7115a642 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 21:38:07 -0700 Subject: [PATCH 36/41] fix(dynacell/eval): check CP column names in every CP cache writer The name check ran per manifest at eval start, not per write. A writer that skipped it (precompute-gt) could restamp cp_feature_names over slots of the old column order, and the mixed cache then passed (F4, #499 re-review). fov_cp_features, the one writer the eval and precompute-gt share, now runs check_cp_cache_feature_names against the running code's column names before any write. A test simulates a reordered live column list and shows the precompute-style write refused with the recorded names untouched. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/pipeline_cache.py | 16 +++++++++++-- .../dynacell/tests/test_cp_feature_names.py | 23 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline_cache.py b/applications/dynacell/src/dynacell/evaluation/pipeline_cache.py index b372f2f6d..402826fff 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline_cache.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline_cache.py @@ -1405,8 +1405,8 @@ def check_cp_cache_feature_names(ctx: _CacheContext, expected: tuple[str, ...]) cached = cached_cp_feature_names(entry) if cached != tuple(expected): raise StaleCacheError( - f"{ctx.side} CP cache {ctx.paths.cp_features()} holds columns {list(cached)}, but the CP reference " - f"expects {list(expected)}. Masking by position would misalign them; rebuild the cache with " + f"{ctx.side} CP cache {ctx.paths.cp_features()} holds columns {list(cached)}, but " + f"{list(expected)} are expected. Masking by position would misalign them; rebuild the cache with " f"force_recompute.{ctx.side}_cp=true." ) @@ -1421,7 +1421,19 @@ def fov_cp_features( The cache side is taken from ``ctx.side``. Result is a list of ``T`` arrays, each shape ``(n_cells_t, n_features)``. + + Every writer goes through here (the eval and ``precompute-gt``), and it first + refuses a cache whose recorded columns are not the running code's + (:func:`check_cp_cache_feature_names`). Otherwise a write under a reordered + column list would restamp ``cp_feature_names`` over slots of the old order, + and the mixed cache would then pass the name check. + + Raises + ------ + StaleCacheError + If the cache's recorded CP column names differ from the running code's. """ + check_cp_cache_feature_names(ctx, active_cp_feature_names(bool((ctx.cp_glcm or {}).get("enabled", False)))) per_t, manifest_updated = _load_or_compute_feature_timepoints( ctx, kind="cp", diff --git a/applications/dynacell/tests/test_cp_feature_names.py b/applications/dynacell/tests/test_cp_feature_names.py index deac345d0..a75965c8f 100644 --- a/applications/dynacell/tests/test_cp_feature_names.py +++ b/applications/dynacell/tests/test_cp_feature_names.py @@ -83,3 +83,26 @@ def test_a_forced_recompute_skips_the_name_check(tmp_path: Path) -> None: config.force_recompute.gt_cp = True ctx = init_cache_context(config, side="gt") check_cp_cache_feature_names(ctx, _NAMES_BASE) + + +def test_a_precompute_style_write_under_reordered_names_is_refused(tmp_path: Path, monkeypatch) -> None: + """The writer precompute-gt uses (``fov_cp_features``) refuses to write into a cache of another column order. + + Simulates code whose live column order changed without a version bump: the + cache records the old order, the writer would compute (and restamp) the new one. + """ + import dynacell.evaluation.pipeline_cache as pipeline_cache + + config = _config(tmp_path) + ctx = init_cache_context(config, side="gt") + image = np.random.default_rng(0).uniform(size=(1, 11, 16, 16)).astype(np.float32) + seg = np.zeros((1, 11, 16, 16), dtype=np.int32) + seg[0, 2:8, 2:8, 2:8] = 1 + fov_cp_features(ctx, "A/1/0", image, seg) # written under the current order + + reordered = list(_NAMES_BASE) + reordered[0], reordered[1] = reordered[1], reordered[0] + monkeypatch.setattr(pipeline_cache, "active_cp_feature_names", lambda glcm: tuple(reordered)) + with pytest.raises(StaleCacheError, match="Masking by position would misalign them"): + fov_cp_features(ctx, "A/1/1", image, seg) + assert ctx.manifest["artifacts"]["cp_features"]["cp_feature_names"] == list(_NAMES_BASE) From 6c2ee7bfae668d43f1ecd89892947a9e183f2f20 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 22:12:49 -0700 Subject: [PATCH 37/41] fix(dynacell/eval): bind a cached CP space to its fit position set cp_space_sha256 covered the GT cell count and moments but not the position set, so a reference rebuilt on other positions with the same count, moments and scaler kept the binding and let cached metrics from the old positions be reused. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- applications/dynacell/src/dynacell/evaluation/cp_reference.py | 3 ++- .../dynacell/src/dynacell/evaluation/cp_reference_test.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index 951226ea5..cd965a719 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -606,7 +606,7 @@ def binding_sha256(self) -> str: It covers the CP recipe identity, the selection criteria, the feature names and keep-mask, THIS dataset's scaler (mean, std, floored features -- the parent's for a lite set, plus the lite -> parent link), and this - dataset's recorded GT cell count and raw moments (a lite set's own) -- + dataset's recorded GT position set, cell count and raw moments (a lite set's own) -- not the exact GT-matrix sha256, so GPU jitter never moves it. It deliberately leaves out the whole-reference hash, so adding or refitting another dataset of the target does not invalidate this dataset's cached eval dirs (whose @@ -631,6 +631,7 @@ def binding_sha256(self) -> str: "std": [float(v) for v in self.std], "floored_features": list(self.floored_features), }, + "gt_positions": sorted(self.fit_positions), "gt_n_cells": self.fit_n_cells, "gt_moments": {"mean": [float(v) for v in self.gt_mean], "std": [float(v) for v in self.gt_std]}, } diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py index bfe4ee7bd..e4449df3a 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference_test.py @@ -460,6 +460,9 @@ def rebound(edit) -> dict[str, str]: assert gt["set-b"] == bindings["set-b"] and gt["set-a-lite"] == bindings["set-a-lite"] lite_gt = rebound(lambda p: p["fit"]["lite"]["set-a-lite"]["gt_moments"]["std"].__setitem__(0, 1.0)) assert lite_gt["set-a-lite"] != bindings["set-a-lite"] and lite_gt["set-a"] == bindings["set-a"] + # Same count, moments and scaler, different fit position set: the binding must move. + moved_pos = rebound(lambda p: p["fit"]["datasets"]["set-a"].update(positions=["Z/9/9"])) + assert moved_pos["set-a"] != bindings["set-a"] and moved_pos["set-b"] == bindings["set-b"] def test_lite_staging_uses_the_parent_scaler(two_sets, tmp_path: Path) -> None: From 23109df014e99aa2f09389b00a474376e42b9344 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 22:12:50 -0700 Subject: [PATCH 38/41] docs(dynacell/tools): describe --verify's tolerant gate in the builder docstring --verify fails on the count/moment gate and reports an exact GT-matrix sha difference as information only (GPU extraction is not bit-reproducible); the module docstring still promised exit 1 on any sha mismatch. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- applications/dynacell/tools/build_cp_reference.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/applications/dynacell/tools/build_cp_reference.py b/applications/dynacell/tools/build_cp_reference.py index 966528c5e..6f18d7e9d 100644 --- a/applications/dynacell/tools/build_cp_reference.py +++ b/applications/dynacell/tools/build_cp_reference.py @@ -25,11 +25,12 @@ The write is atomic, and an existing reference with a different content hash is only replaced under ``--force`` (an identical one is left untouched). -``--verify`` re-reads every dataset's GT CP cells from the current caches and -compares their sha256 and count with the ones recorded in the reference's ``fit`` -section; it exits 1 on any mismatch. It is the offline form of the eval's content -gate, which refuses to score a dataset whose staged GT cells no longer hash to the -recorded value. +``--verify`` re-reads every dataset's GT CP cells from the current caches and runs +the eval's content gate on them: exact position set and GT-finite cell count, and +each feature's raw mean and std within the recorded moments' tolerance +(``GT_MOMENT_RTOL``). It exits 1 when that gate fails. An exact GT-matrix sha256 +difference is reported as information only, since GPU CP extraction is not +bit-reproducible (differences around 1e-15). Run:: From f85d47213b3c6c2afca60d469d5e5307bd0f52c2 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 22:54:14 -0700 Subject: [PATCH 39/41] fix(dynacell/eval): gate the GT and write the CP sidecar when no pred CP rows remain When every prediction CP row was non-finite, the dataset-level block skipped CP staging, which also skipped the GT content gate and the cp_selected_feature_mask.json sidecar. Run both whenever feature metrics are on; CP metrics stay NaN in that case. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- .../src/dynacell/evaluation/pipeline.py | 28 +++++++++++++++++-- .../tests/test_evaluation_cp_end_to_end.py | 23 +++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/pipeline.py b/applications/dynacell/src/dynacell/evaluation/pipeline.py index 6d0ecd6dc..a50dca4de 100644 --- a/applications/dynacell/src/dynacell/evaluation/pipeline.py +++ b/applications/dynacell/src/dynacell/evaluation/pipeline.py @@ -333,6 +333,28 @@ def _extend_backbone( bb.gt_ts.append(t_arr) +def _gate_and_record_cp_space( + cp_space: DatasetCPSpace, gt_cp_blocks: dict[tuple[str, int], np.ndarray], save_dir: Path +) -> None: + """Run the GT content gate and write the CP sidecar for this eval dir. + + Runs whenever feature metrics are on, whether or not the prediction + contributed any finite CP rows, so the gate is never bypassed and every + feature-metrics eval dir carries its sidecar. + + Parameters + ---------- + cp_space : DatasetCPSpace + Reference bound to the eval's dataset. + gt_cp_blocks : dict + ``{(position, t): GT-finite CP rows}`` the run scored. + save_dir : pathlib.Path + Eval dir receiving the sidecar. + """ + cp_space.check_gt_cells(gt_cp_blocks) + (save_dir / CP_SIDECAR_FILENAME).write_text(json.dumps(cp_sidecar_payload(cp_space), indent=2)) + + def _stage_cp_dataset_inputs( cp: _BackboneLists, cp_space: DatasetCPSpace, @@ -365,11 +387,9 @@ def _stage_cp_dataset_inputs( tuple ``("CP", pred_metric, target_metric, pred_probe, target_probe, pred_fovs, target_fovs)``. """ - cp_space.check_gt_cells(gt_cp_blocks) + _gate_and_record_cp_space(cp_space, gt_cp_blocks, save_dir) pred_cp_raw = np.concatenate(cp.pred_feats, axis=0) target_cp_raw = np.concatenate(cp.gt_feats, axis=0) - mask_payload = cp_sidecar_payload(cp_space) - (save_dir / CP_SIDECAR_FILENAME).write_text(json.dumps(mask_payload, indent=2)) return ( "CP", cp_space.transform(pred_cp_raw), @@ -1766,6 +1786,8 @@ def _aggregate(result: FovResult) -> None: if parent_lists["cp"].pred_feats: prefix_inputs.append(_stage_cp_dataset_inputs(parent_lists["cp"], cp_space, gt_cp_blocks, save_dir)) + else: # no finite pred CP rows: CP metrics are NaN, but the GT gate and sidecar still apply + _gate_and_record_cp_space(cp_space, gt_cp_blocks, save_dir) for key in deep_kinds: # cp is handled above (reference space) bb = parent_lists[key] diff --git a/applications/dynacell/tests/test_evaluation_cp_end_to_end.py b/applications/dynacell/tests/test_evaluation_cp_end_to_end.py index dea79ead7..ba18ba140 100644 --- a/applications/dynacell/tests/test_evaluation_cp_end_to_end.py +++ b/applications/dynacell/tests/test_evaluation_cp_end_to_end.py @@ -299,3 +299,26 @@ def test_a_complete_stale_gt_cache_fails_before_the_fov_loop(harness: Harness, m with pytest.raises(Exception, match="differ from the CP reference fit") as err: harness.run("early", [g.copy() for g in harness.gt]) assert type(err.value).__name__ == "StaleCacheError" + + +def test_no_finite_pred_cp_rows_still_gates_the_gt_and_writes_the_sidecar(harness: Harness, monkeypatch) -> None: + """A prediction whose CP rows are all non-finite leaves CP NaN, yet the GT gate and sidecar still run.""" + real = harness.pipeline.fov_cp_features + + def nan_pred_side(ctx, pos_name, image, cell_segmentation): + per_t = real(ctx, pos_name, image, cell_segmentation) + return [np.full_like(f, np.nan) for f in per_t] if ctx.side == "pred" else per_t + + monkeypatch.setattr(harness.pipeline, "fov_cp_features", nan_pred_side) + row, _ = harness.run("nanpred", [g.copy() for g in harness.gt]) + assert np.isnan(row["Dataset_CP_KID"]) + assert (harness.root / "nanpred" / "cp_selected_feature_mask.json").exists() + + # A GT recompute from a changed GT store bypasses the up-front cache check, so only the + # post-staging gate can refuse it -- and it must, even with no finite pred CP rows. + with open_ome_zarr(harness.root / "gt.zarr", mode="r+") as plate: + image = plate["A/1/1"]["0"] + image[1] = np.asarray(image[1]) ** 2 # non-affine: survives the per-image percentile norm + with pytest.raises(Exception, match="differ from the CP reference fit") as err: + harness.run("nanpred_moved", [g.copy() for g in harness.gt], gt_cp=True) + assert type(err.value).__name__ == "StaleCacheError" From 7cd6403af593cda5d2ae6a59e6e812a019c1799b Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 22:54:15 -0700 Subject: [PATCH 40/41] fix(dynacell/tools): make --verify compare the cache's position set with the fit --verify checked only cell count and moments, so a cache rebuilt from a different position set with the same count and distribution reported OK despite the eval's exact position check. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- applications/dynacell/tools/build_cp_reference.py | 7 +++++-- .../dynacell/tools/build_cp_reference_test.py | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/applications/dynacell/tools/build_cp_reference.py b/applications/dynacell/tools/build_cp_reference.py index 6f18d7e9d..0beae92e3 100644 --- a/applications/dynacell/tools/build_cp_reference.py +++ b/applications/dynacell/tools/build_cp_reference.py @@ -192,7 +192,7 @@ def build(target_name: str, leaves_root: Path) -> dict[str, Any]: def verify(target_name: str, path: Path) -> list[str]: """Re-read every dataset's (lite included) GT CP cells from the caches and apply the content gate. - The same tolerance gate the eval applies (exact cell count, per-feature GT + The same gate the eval applies (exact position set and cell count, per-feature GT moments within tolerance; ``DatasetCPSpace.gt_cells_problems``). An exact GT-matrix sha256 difference alone is reported as information: GPU recomputes move it by ~1e-15 without changing any number that matters. @@ -213,7 +213,10 @@ def verify(target_name: str, path: Path) -> list[str]: mismatches = [] for name, record in sorted({**ref.fit["datasets"], **ref.fit["lite"]}.items()): fit = read_dataset_fit(target_name, (name, record["target"]), ref.feature_names, record["in_mask_fit"]) - problems = ref.for_dataset(name).gt_cells_problems(fit.cells) + space = ref.for_dataset(name) + problems = space.gt_cells_problems(fit.cells) + if set(fit.record["positions"]) != space.fit_positions: + problems.insert(0, "the cache's position set differs from the recorded fit positions") sha256 = gt_matrix_sha256(fit.cells) exact = "exact sha256 match" if sha256 == record["gt_matrix_sha256"] else "info: exact sha256 differs" print(f" {name}: {'OK' if not problems else 'MISMATCH'} ({fit.cells.shape[0]} cells; {exact})") diff --git a/applications/dynacell/tools/build_cp_reference_test.py b/applications/dynacell/tools/build_cp_reference_test.py index 86fbe6b99..fbee1b19f 100644 --- a/applications/dynacell/tools/build_cp_reference_test.py +++ b/applications/dynacell/tools/build_cp_reference_test.py @@ -31,7 +31,7 @@ save_manifest, write_features_to_group, ) -from dynacell.evaluation.cp_reference import MASK_FIT_DATASETS, cp_space, load_cp_reference +from dynacell.evaluation.cp_reference import MASK_FIT_DATASETS, cp_space, load_cp_reference, payload_sha256 from dynacell.evaluation.metrics import active_cp_feature_names _N_FEATURES = len(active_cp_feature_names(True)) # eval.yaml ships GLCM on @@ -270,3 +270,16 @@ def test_cache_with_reordered_feature_names_is_refused(registry: Path) -> None: leaves = _leaves(registry, ["ds-a", "ds-b"]) with pytest.raises(StaleCacheError, match="Masking by position would misalign them"): main(["--target", "er", "--leaves-root", str(leaves), "--out", str(registry / "x.json")]) + + +def test_verify_fails_when_the_cache_positions_differ_from_the_fit(registry: Path, capsys) -> None: + """``--verify`` checks the position set exactly, not only cell count and moments.""" + leaves = _build_all(registry) + out = registry / "er.json" + main(["--target", "er", "--leaves-root", str(leaves), "--out", str(out)]) + payload = json.loads(out.read_text()) + payload["fit"]["datasets"]["ds-h"]["positions"] = ["Z/9/9"] + payload["sha256"] = payload_sha256(payload) # a rebuilt record on another position set + out.write_text(json.dumps(payload)) + assert main(["--target", "er", "--out", str(out), "--verify"]) == 1 + assert "position set differs" in capsys.readouterr().out From 374a3e60732f14e407277cbed68223dd77d24273 Mon Sep 17 00:00:00 2001 From: Alexandr Kalinin Date: Fri, 25 Sep 2026 22:57:04 -0700 Subject: [PATCH 41/41] docs(dynacell/eval): state the CP moments gate's measured margins and blind spots Review nits F6-F8: the README overclaimed that any real value change fails and omitted the position set and cell count from the binding; the GT_MOMENT_RTOL comment claimed three decades of margin where a CPU-vs-GPU recompute leaves ~23x; the builder docstring still said the eval hashes the stacked cells. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XvkM1gKifWmoDPq8Drd4JF --- applications/dynacell/src/dynacell/evaluation/README.md | 2 +- .../dynacell/src/dynacell/evaluation/cp_reference.py | 5 +++-- applications/dynacell/tools/build_cp_reference.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/applications/dynacell/src/dynacell/evaluation/README.md b/applications/dynacell/src/dynacell/evaluation/README.md index afab900c7..a8a2ba757 100644 --- a/applications/dynacell/src/dynacell/evaluation/README.md +++ b/applications/dynacell/src/dynacell/evaluation/README.md @@ -24,7 +24,7 @@ End-to-end evaluation pipeline for virtual staining predictions against fluoresc - `io.gt_path` — fluorescence ground truth (channel: `io.gt_channel_name`) - `io.cell_segmentation_path` — *optional* precomputed cell segmentation HCS OME-Zarr. Required when `compute_feature_metrics=true` or when building CP/DINOv3/DynaCLR/CELL-DINO cache entries. Position layout must match GT/pred 1:1. - `io.gt_cache_dir`, `io.pred_cache_dir` — *optional* artifact cache directories; must be distinct. See [Caches](#caches). -- `feature_metrics.cp.reference_path` — the target's CP (GLCM+) reference, required when `compute_feature_metrics=true`. `null` (default) resolves to `DATA_ROOT/cp_reference/.json` for full and lite evals alike. The reference holds one feature mask chosen on GT cells only, pooled over iPSC + A549 mock/denv/zikv, and one GT mean/std per test set (keyed by `benchmark.dataset_ref.dataset`; HEK gets its own, lite reuses its parent's). Every eval applies the mask and its dataset's scaler to pred AND GT. The eval refuses a partial position walk, and refuses to score unless its staged GT CP cells match the reference's fit for that dataset: the same GT-finite cell count exactly, and every CP feature's raw GT mean/std within a relative tolerance of 1e-6 (`GT_MOMENT_RTOL`). A re-cache that reproduces the cells passes, including a GPU recompute, which is only reproducible to ~1e-15; a real value change fails. When the GT CP cache is already complete, the check also runs before the FOV loop. Build the reference from existing GT caches with `tools/build_cp_reference.py --target `; `--verify` re-applies the same check to the current caches. `metrics_provenance.json` stamps the per-dataset `cp_space_sha256`: the mask, feature names, recipe, and that dataset's scaler and recorded GT moments. A rebuild therefore invalidates only the eval dirs whose own dataset's CP space changed; adding or refitting another dataset leaves them reusable. The `cp_selected_feature_mask.json` sidecar records the same per-dataset hash. +- `feature_metrics.cp.reference_path` — the target's CP (GLCM+) reference, required when `compute_feature_metrics=true`. `null` (default) resolves to `DATA_ROOT/cp_reference/.json` for full and lite evals alike. The reference holds one feature mask chosen on GT cells only, pooled over iPSC + A549 mock/denv/zikv, and one GT mean/std per test set (keyed by `benchmark.dataset_ref.dataset`; HEK gets its own, lite reuses its parent's). Every eval applies the mask and its dataset's scaler to pred AND GT. The eval refuses a partial position walk, and refuses to score unless its staged GT CP cells match the reference's fit for that dataset: the same GT-finite cell count exactly, and every CP feature's raw GT mean/std within a relative tolerance of 1e-6 (`GT_MOMENT_RTOL`). A re-cache that reproduces the cells passes, including a GPU recompute (reproducible only to ~1e-15) and a CPU-vs-GPU recompute (up to ~4e-8, measured). It is a moments gate, not a content hash: a change that moves any feature's GT mean/std by more than the tolerance fails, but permuting values within a column, moving cells between blocks, or a single-cell change below about n × 1e-6 std passes. When the GT CP cache is already complete, the check also runs before the FOV loop. Build the reference from existing GT caches with `tools/build_cp_reference.py --target `; `--verify` re-applies the same check to the current caches. `metrics_provenance.json` stamps the per-dataset `cp_space_sha256`: the mask, feature names, recipe, and that dataset's scaler, recorded GT position set, GT cell count and GT moments. A rebuild therefore invalidates only the eval dirs whose own dataset's CP space changed; adding or refitting another dataset leaves them reusable. The `cp_selected_feature_mask.json` sidecar records the same per-dataset hash. - GLCM+ (CP) space in two lines: one feature mask selected on GT cells only, pooled over the target's iPSC + A549 mock/denv/zikv test sets; then each test set's own GT mean/std standardizes pred and GT alike, so offsets and scale errors relative to that test set's GT stay visible. - GLCM+ median cosine is invariant to contraction toward the GT mean: for `pred = mean + 0.5·(GT − mean)`, every pred/GT cell pair is parallel after the shared scaler and the median cosine is exactly 1.0000, while KID catches it (reviewer-measured on real cells: KID 0.194 vs a pred==GT floor of −0.041; reproduced on synthetic cells in `cp_reference_test.py`). Cosine measures direction only; over-smoothing toward the mean shows up in KID, not cosine. diff --git a/applications/dynacell/src/dynacell/evaluation/cp_reference.py b/applications/dynacell/src/dynacell/evaluation/cp_reference.py index cd965a719..2b973e417 100644 --- a/applications/dynacell/src/dynacell/evaluation/cp_reference.py +++ b/applications/dynacell/src/dynacell/evaluation/cp_reference.py @@ -82,8 +82,9 @@ #: Content-gate tolerance on each CP feature's GT moments (see #: :meth:`DatasetCPSpace.check_gt_cells`): |mean - mean_fit| <= GT_MOMENT_RTOL * std_fit -#: and |std / std_fit - 1| <= GT_MOMENT_RTOL. GPU regionprops jitter is ~1e-15 -#: relative, a real GT change is >= ~1e-3, so 1e-6 separates them by >= 3 decades each way. +#: and |std / std_fit - 1| <= GT_MOMENT_RTOL. Measured on real data: GPU regionprops +#: jitter is ~1e-15 relative, but a CPU-vs-GPU recompute differs by up to 4.3e-8 (std of +#: intensity_min), only ~23x under this tolerance; a real GT change is >= ~1e-3. GT_MOMENT_RTOL = 1e-6 #: For a feature constant in the fit (std_fit == 0): |mean - mean_fit| and the staged std #: must both be <= GT_MOMENT_ZERO_STD_ATOL * (1 + |mean_fit|). diff --git a/applications/dynacell/tools/build_cp_reference.py b/applications/dynacell/tools/build_cp_reference.py index 0beae92e3..1bb55e65a 100644 --- a/applications/dynacell/tools/build_cp_reference.py +++ b/applications/dynacell/tools/build_cp_reference.py @@ -19,7 +19,7 @@ ``require_complete_cache=true`` (so a cache of another store or CP recipe is refused), and every ``(position, timepoint)`` of the GT store must already be in the CP cache. A miss raises; features are never computed. Non-finite GT cells are -dropped and the rest stacked in the canonical order the eval hashes +dropped and the rest stacked in the canonical order the eval stages (``cp_reference.read_gt_cp_cells`` / ``canonical_gt_matrix``). The write is atomic, and an existing reference with a different content hash is