build(dynacell): bump cubic to 0.9.0a2, keep 0.9.0a1 metric caches valid

applications/dynacell/pyproject.toml

@@ -62,9 +62,12 @@ optional-dependencies.eval = [
   # with `normalize`/`normalization` rather than returning a quietly wrong
   # number, and the FSC/spectral, MicroSSIM, MS-SSIM, feature (KID/FID) and
   # average_precision modules were rewritten. Every pixel/feature metric in the
-  # benchmark can move across this pin, so it is paired with a full re-eval --
-  # do not bump it casually. Tag, not a bare commit.
-  "cubic @ git+https://github.com/alxndrkalinin/cubic.git@v0.9.0a1",
+  # benchmark can move across this pin, so a bump needs either a full re-eval or
+  # a measured entry in dynacell.evaluation.provenance.CUBIC_VERSIONS_EQUIVALENT_TO
+  # -- do not bump it casually. Tag, not a bare commit. v0.9.0a2 rewrites only the
+  # MicroSSIM RI-factor fit: chunked float64 reduction (peak 40.5 -> 12.9 GB on
+  # 576x640x960 float32) and a ValueError on empty input; measured equivalent.
+  "cubic @ git+https://github.com/alxndrkalinin/cubic.git@v0.9.0a2",
   "dinov3 @ git+https://github.com/facebookresearch/dinov3",
   "dynaclr",
   "hydra-core>=1.2",
@@ -101,7 +104,7 @@ optional-dependencies.eval_gpu = [
 # selection->waveorder, segmentation->cellpose (cellpose imported lazily).
 optional-dependencies.preprocess = [
   "cellpose>=4.2",
-  "cubic @ git+https://github.com/alxndrkalinin/cubic.git@v0.9.0a1",
+  "cubic @ git+https://github.com/alxndrkalinin/cubic.git@v0.9.0a2",
   "iohub>=0.3.6",
   "tqdm",
   "waveorder",

applications/dynacell/src/dynacell/evaluation/CLAUDE.md

@@ -5,7 +5,7 @@ computing) for any GPU-accelerated numerical work — image preprocessing
 before / after model inference, metric calculations, cropping/resizing,
 percentile clips, Gaussian filters, etc. Cubic is a hard runtime
 dependency of the eval extras (`applications/dynacell/pyproject.toml`
-pins `cubic @ git+…@v0.9.0a1`). Do not gate cubic imports behind `try/except`
+pins `cubic @ git+…@v0.9.0a2`). Do not gate cubic imports behind `try/except`
 or fall back to scipy/skimage paths.
 
 **The cubic version is part of the numeric contract, not just a dep.** Metric
@@ -23,6 +23,10 @@ values move across pins — measured 0.8.0a2 → 0.9.0a1: `Z_FSC_Resolution`
   Unstamped means "written before the stamp existed", i.e. an unknown pin —
   it must recompute, unlike the per-extractor `preprocess_version` bootstrap
   which treats an untagged entry as unconstrained.
+- A bump measured to move no value beyond a stated tolerance lists the old
+  version under the new one in `CUBIC_VERSIONS_EQUIVALENT_TO` so its caches are still reused
+  (0.9.0a1 under 0.9.0a2). The map is keyed by the declared version, so the
+  next bump reuses nothing until someone measures an equivalence against it.
 
 The GPU-resident Cellpose-SAM entry point is
 `cubic.segmentation.segment_cpsam` (single host→device upload, masks

applications/dynacell/src/dynacell/evaluation/provenance.py

@@ -29,6 +29,11 @@
   and :func:`metrics_provenance_matches` lets the final-metrics cache gate
   refuse a cache built by a different ``cubic``.
 
+A version bump measured to move no value beyond a stated tolerance lists the old
+version under the new one in :data:`CUBIC_VERSIONS_EQUIVALENT_TO`, so its caches
+stay reusable instead of forcing every leaf to recompute. 0.9.0a1 -> 0.9.0a2 is such a
+bump: it rewrites only the MicroSSIM RI-factor fit.
+
 The same sidecar stamps the content hash of the CP reference the CP feature
 metrics were scored in (:mod:`dynacell.evaluation.cp_reference`). CP KID/FID/
 cosine move whenever the reference is rebuilt, so a cache stamped with another
@@ -43,7 +48,25 @@
 #: The ``cubic`` version this repo is built against. Must equal the pin in
 #: ``applications/dynacell/pyproject.toml``; ``provenance_test.py`` asserts
 #: they cannot drift apart.
-REQUIRED_CUBIC_VERSION = "0.9.0a1"
+REQUIRED_CUBIC_VERSION = "0.9.0a2"
+
+#: For each declared ``cubic`` version, the earlier versions whose metric values it
+#: reproduces within a stated tolerance, so their caches stay reusable. Keyed by the
+#: declared version so a bump starts with no equivalents until one is measured
+#: against it.
+#:
+#: 0.9.0a1 -> 0.9.0a2 changes only the MicroSSIM RI-factor fit: it reduces the
+#: objective in bounded chunks, accumulating in float64 where 0.9.0a1 took a
+#: float32 ``.mean()``, and raises ``ValueError`` on an empty pool. Tolerance: MicroMS3IM relative delta <= 1e-6, far
+#: below the tables' 2-decimal rounding. Measured 2026-09-28 on the A549-trained
+#: ``fnet3d_paper`` ER ``a549__denv`` leaf at the production calibration settings
+#: (``max_pairs=12``, seed 42, a 576x640x960 float32 pool): the fitted alpha is
+#: 17.883591651916504 under both versions, and MicroMS3IM over all 108 (FOV, t)
+#: differs by 0 abs / 0 rel. The fit objective itself moves ~7e-8 relative between
+#: the versions; the root finder lands on the same alpha. The other
+#: ``pixel_metrics`` / ``mask_metrics`` columns do not use the RI-factor code and
+#: were bit-identical on 1 FOV x 7 timepoints.
+CUBIC_VERSIONS_EQUIVALENT_TO = {"0.9.0a2": frozenset({"0.9.0a1"})}
 
 #: Sidecar written next to ``pixel_metrics.csv`` by :func:`write_metrics_provenance`.
 PROVENANCE_FILENAME = "metrics_provenance.json"
@@ -142,15 +165,20 @@ def metrics_provenance_matches(save_dir: Path, *, cp_space_sha256: str | None) -
     Returns
     -------
     bool
-        True when the recorded ``cubic`` version equals the installed one and,
-        if ``cp_space_sha256`` is given, the recorded binding equals it.
+        True when the recorded ``cubic`` version equals the installed one (or is
+        in :data:`CUBIC_VERSIONS_EQUIVALENT_TO` for the declared pin while the installed one is
+        the declared pin) and, if ``cp_space_sha256`` is given, the recorded binding
+        equals it.
     """
     path = save_dir / PROVENANCE_FILENAME
     if not path.is_file():
         return False
     payload = json.loads(path.read_text())
     recorded = payload.get("versions", {}).get("cubic")
-    if recorded != version("cubic"):
+    installed = version("cubic")
+    equivalents = CUBIC_VERSIONS_EQUIVALENT_TO.get(REQUIRED_CUBIC_VERSION, frozenset())
+    reproduced = installed == REQUIRED_CUBIC_VERSION and recorded in equivalents
+    if recorded != installed and not reproduced:
         return False
     if cp_space_sha256 is None:
         return True

applications/dynacell/src/dynacell/evaluation/provenance_test.py

@@ -8,6 +8,7 @@
 import pytest
 
 from dynacell.evaluation.provenance import (
+    CUBIC_VERSIONS_EQUIVALENT_TO,
     PROVENANCE_FILENAME,
     REQUIRED_CUBIC_VERSION,
     check_cubic_pin,
@@ -82,12 +83,66 @@ def test_missing_sidecar_is_not_a_match(tmp_path):
     assert not metrics_provenance_matches(tmp_path, cp_space_sha256=None)
 
 
-def test_foreign_cubic_version_is_not_a_match(tmp_path):
-    write_metrics_provenance(tmp_path, cp_reference_sha256=None, cp_space_sha256=None)
-    path = tmp_path / PROVENANCE_FILENAME
+def _stamp_cubic(save_dir, cubic, *, cp_space_sha256):
+    """Write a sidecar as if a run under ``cubic`` had produced it."""
+    cp_reference_sha256 = None if cp_space_sha256 is None else f"ref-{cp_space_sha256}"
+    write_metrics_provenance(save_dir, cp_reference_sha256=cp_reference_sha256, cp_space_sha256=cp_space_sha256)
+    path = save_dir / PROVENANCE_FILENAME
     payload = json.loads(path.read_text())
-    payload["versions"]["cubic"] = "0.8.0a2"
+    payload["versions"]["cubic"] = cubic
     path.write_text(json.dumps(payload))
+
+
+def test_foreign_cubic_version_is_not_a_match(tmp_path):
+    _stamp_cubic(tmp_path, "0.8.0a2", cp_space_sha256=None)
+    assert not metrics_provenance_matches(tmp_path, cp_space_sha256=None)
+
+
+@pytest.fixture
+def running_the_declared_pin(monkeypatch):
+    """Report the declared cubic as installed, whatever the test venv holds."""
+    monkeypatch.setattr("dynacell.evaluation.provenance.version", lambda _name: REQUIRED_CUBIC_VERSION)
+
+
+def test_equivalent_versions_exclude_the_declared_one():
+    equivalents = CUBIC_VERSIONS_EQUIVALENT_TO[REQUIRED_CUBIC_VERSION]
+    assert "0.9.0a1" in equivalents
+    assert REQUIRED_CUBIC_VERSION not in equivalents
+
+
+def test_equivalent_cubic_version_is_a_match(tmp_path, running_the_declared_pin):
+    """A 0.9.0a1 cache holds the values the declared pin reproduces, so it is reused."""
+    _stamp_cubic(tmp_path, "0.9.0a1", cp_space_sha256=None)
+    assert metrics_provenance_matches(tmp_path, cp_space_sha256=None)
+
+
+def test_older_cubic_version_is_still_not_a_match(tmp_path, running_the_declared_pin):
+    """0.8.0a2 moved FSC/FRC/Spectral_PCC, so its caches stay refused."""
+    _stamp_cubic(tmp_path, "0.8.0a2", cp_space_sha256=None)
+    assert not metrics_provenance_matches(tmp_path, cp_space_sha256=None)
+
+
+def test_equivalent_cubic_version_keeps_the_cp_rules(tmp_path, running_the_declared_pin):
+    """Accepting the cubic stamp does not relax the CP-space binding check."""
+    _stamp_cubic(tmp_path, "0.9.0a1", cp_space_sha256="abc")
+    assert metrics_provenance_matches(tmp_path, cp_space_sha256="abc")
+    assert not metrics_provenance_matches(tmp_path, cp_space_sha256="other")
+    _stamp_cubic(tmp_path, "0.9.0a1", cp_space_sha256=None)
+    assert not metrics_provenance_matches(tmp_path, cp_space_sha256="abc")
+
+
+def test_equivalence_only_applies_under_the_declared_pin(tmp_path, monkeypatch):
+    """An environment off the pin gets no equivalence: only its own version matches."""
+    monkeypatch.setattr("dynacell.evaluation.provenance.version", lambda _name: "0.8.0a2")
+    _stamp_cubic(tmp_path, "0.9.0a1", cp_space_sha256=None)
+    assert not metrics_provenance_matches(tmp_path, cp_space_sha256=None)
+
+
+def test_unmeasured_pin_has_no_equivalents(tmp_path, monkeypatch):
+    """A newly declared pin with no measured entry reuses no older cache."""
+    monkeypatch.setattr("dynacell.evaluation.provenance.REQUIRED_CUBIC_VERSION", "0.9.0a3")
+    monkeypatch.setattr("dynacell.evaluation.provenance.version", lambda _name: "0.9.0a3")
+    _stamp_cubic(tmp_path, "0.9.0a1", cp_space_sha256=None)
     assert not metrics_provenance_matches(tmp_path, cp_space_sha256=None)
 
 

applications/dynacell/tools/README.md

@@ -40,6 +40,8 @@ builders. Not part of the installable package — run with `uv run python applic
 
 - `smoke_joint_leaf.py` — smoke a joint (multi-dataset) training leaf.
 - `nccl_smoke_test.py` — probe multi-GPU NCCL connectivity before a DDP fit.
+- `scan_nan_microssim_caches.py` — read-only: list eval caches stamped with a cubic version (default 0.9.0a1)
+  whose MicroMS3IM is entirely NaN, i.e. caches a cubic-equivalence reuse would carry a NaN forward in.
 - `*_test.py` — pytest for the generators (`generate_grouped_eval_configs_test.py`,
   `generate_instance_ap_eval_configs_test.py`); further tool tests live in [`../tests/`](../tests/README.md).
 - `slurm_logs/` — captured job logs (runtime output, not source).

applications/dynacell/tools/run_eval_direct.slurm

@@ -74,8 +74,10 @@ ml uv
 # the 0.9.0a1 venv existed, so jobs that did not export DYNACELL_EVAL_VENV kept
 # silently producing 0.8.0a2 FSC/FRC/Spectral_PCC values — 62 of the 121
 # surviving eval logs. ``check_cubic_pin`` now fails such a run loudly, but the
-# default still has to be right.
-export UV_PROJECT_ENVIRONMENT="${DYNACELL_EVAL_VENV:-/hpc/mydata/alex.kalinin/VisCy-venvs/cpdino-eval-cubic090a1}"
+# default still has to be right. ``cpdino-eval-cubic090a2`` is the
+# ``cpdino-eval-cubic090a1`` freeze with only cubic changed; 0.9.0a1 caches stay
+# reusable under it (``provenance.CUBIC_VERSIONS_EQUIVALENT_TO``).
+export UV_PROJECT_ENVIRONMENT="${DYNACELL_EVAL_VENV:-/hpc/mydata/alex.kalinin/VisCy-venvs/cpdino-eval-cubic090a2}"
 export HF_HUB_CACHE="${HF_HUB_CACHE:-/hpc/projects/comp.micro/virtual_staining/models/dynacell/evaluation/hf_cache}"
 export PYTHONUNBUFFERED=1
 

applications/dynacell/tools/scan_nan_microssim_caches.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+r"""List eval caches stamped with a given cubic whose MicroMS3IM column is entirely NaN.
+
+``provenance.CUBIC_VERSIONS_EQUIVALENT_TO`` lets 0.9.0a2 reuse 0.9.0a1 caches. That
+holds only where the 0.9.0a1 cache holds a value. When the leaf-level MicroSSIM
+calibration ran out of memory under 0.9.0a1, ``_calibrate_microssim``'s caller
+wrote ``MicroMS3IM = NaN`` for every FOV. 0.9.0a2's bounded-memory fit would score
+those leaves, but the reused cache keeps the NaN. This scan finds them so they can
+be re-run with ``force_recompute.final_metrics=true``.
+
+Mito A549 mock caches are all-NaN for a reason that predates this bump (the
+MicroSSIM ``data_range`` case), so they are reported as known rather than
+unexplained. Read-only: the tool never writes.
+
+Exit status: 0 when no unexplained cache is found, 1 otherwise.
+
+Usage::
+
+    uv run --no-sync python applications/dynacell/tools/scan_nan_microssim_caches.py \
+        --root /hpc/projects/virtual_staining/training/dynacell
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+import pandas as pd
+
+from dynacell.evaluation.provenance import PROVENANCE_FILENAME
+
+#: Save dirs whose all-NaN MicroMS3IM is the mito A549 mock ``data_range`` case:
+#: canonical ``mito/<model>/<train>/a549__mock`` and legacy
+#: ``a549/evaluations_with_embeddings/eval_<model>_mitochondria_mock``.
+_KNOWN_ALL_NAN = re.compile(r"(/mito/[^/]+/[^/]+/a549__mock|/a549/[^/]+/eval_[^/]*mitochondria_mock)$")
+
+
+def find_sidecars(root: Path, max_depth: int) -> list[Path]:
+    """Return every provenance sidecar under ``root``, not descending into zarr stores."""
+    found = []
+    base = len(root.parts)
+    for dirpath, dirnames, filenames in os.walk(root):
+        depth = len(Path(dirpath).parts) - base
+        dirnames[:] = [d for d in dirnames if not d.endswith((".zarr", ".ozx")) and depth < max_depth]
+        if PROVENANCE_FILENAME in filenames:
+            found.append(Path(dirpath) / PROVENANCE_FILENAME)
+    return sorted(found)
+
+
+def all_nan_microssim(sidecars: list[Path], cubic: str) -> list[Path]:
+    """Return the save dirs among ``sidecars`` stamped ``cubic`` whose MicroMS3IM is entirely NaN."""
+    hits = []
+    for sidecar in sidecars:
+        if json.loads(sidecar.read_text())["versions"]["cubic"] != cubic:
+            continue
+        column = pd.read_csv(sidecar.parent / "pixel_metrics.csv")["MicroMS3IM"]
+        if column.isna().all():
+            hits.append(sidecar.parent)
+    return hits
+
+
+def is_known_all_nan(save_dir: Path) -> bool:
+    """Whether ``save_dir`` is the mito A549 mock case, all-NaN independent of cubic."""
+    return _KNOWN_ALL_NAN.search(save_dir.as_posix()) is not None
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Run the scan and print one line per all-NaN cache."""
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--root", type=Path, required=True)
+    parser.add_argument("--cubic", default="0.9.0a1", help="cubic version stamp to scan (default: 0.9.0a1)")
+    parser.add_argument("--max-depth", type=int, default=6, help="directory depth below --root to search")
+    args = parser.parse_args(argv)
+
+    sidecars = find_sidecars(args.root, args.max_depth)
+    hits = all_nan_microssim(sidecars, args.cubic)
+    unexplained = [d for d in hits if not is_known_all_nan(d)]
+    for save_dir in hits:
+        print(f"{'known' if is_known_all_nan(save_dir) else 'UNEXPLAINED'}\t{save_dir}")
+    print(
+        f"{len(sidecars)} sidecars, {len(hits)} all-NaN MicroMS3IM stamped cubic {args.cubic}, "
+        f"{len(unexplained)} unexplained"
+    )
+    return 1 if unexplained else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

applications/dynacell/tools/scan_nan_microssim_caches_test.py

@@ -0,0 +1,64 @@
+"""Integration tests for ``scan_nan_microssim_caches.py``.
+
+Each test lays out real eval save dirs in ``tmp_path`` (sidecar written by
+``write_metrics_provenance``, then restamped) and runs the tool's ``main``.
+"""
+
+from __future__ import annotations
+
+import json
+
+import numpy as np
+import pandas as pd
+from scan_nan_microssim_caches import main  # noqa: E402
+
+from dynacell.evaluation.provenance import PROVENANCE_FILENAME, write_metrics_provenance
+
+
+def _save_dir(root, rel, microssim, cubic="0.9.0a1"):
+    save_dir = root / rel
+    save_dir.mkdir(parents=True)
+    write_metrics_provenance(save_dir, cp_reference_sha256=None, cp_space_sha256=None)
+    sidecar = save_dir / PROVENANCE_FILENAME
+    payload = json.loads(sidecar.read_text())
+    payload["versions"]["cubic"] = cubic
+    sidecar.write_text(json.dumps(payload))
+    pd.DataFrame({"FOV": ["a", "b"], "MicroMS3IM": microssim}).to_csv(save_dir / "pixel_metrics.csv", index=False)
+    return save_dir
+
+
+def test_unexplained_all_nan_cache_fails(tmp_path, capsys):
+    """An all-NaN cache outside the known case is reported and fails the scan; a partial NaN is not."""
+    hit = _save_dir(tmp_path, "er/fnet3d_paper/a549/a549__denv", [np.nan, np.nan])
+    _save_dir(tmp_path, "er/fnet3d_paper/a549/a549__zikv", [0.8, np.nan])
+    assert main(["--root", str(tmp_path)]) == 1
+    out = capsys.readouterr().out
+    assert f"UNEXPLAINED\t{hit}" in out
+    assert "a549__zikv" not in out
+    assert "2 sidecars, 1 all-NaN MicroMS3IM stamped cubic 0.9.0a1, 1 unexplained" in out
+
+
+def test_mito_a549_mock_is_known(tmp_path, capsys):
+    """Canonical and legacy mito A549 mock dirs are reported as known and pass."""
+    canonical = _save_dir(tmp_path, "mito/fnet2d/joint/a549__mock", [np.nan, np.nan])
+    legacy = _save_dir(tmp_path, "a549/evaluations_with_embeddings/eval_phase_mitochondria_mock", [np.nan, np.nan])
+    assert main(["--root", str(tmp_path)]) == 0
+    out = capsys.readouterr().out
+    assert f"known\t{canonical}" in out
+    assert f"known\t{legacy}" in out
+    assert "0 unexplained" in out
+
+
+def test_mito_other_conditions_are_not_known(tmp_path, capsys):
+    """The known case is mito A549 mock only, not every mito condition."""
+    hit = _save_dir(tmp_path, "mito/fnet2d/joint/a549__denv", [np.nan, np.nan])
+    assert main(["--root", str(tmp_path)]) == 1
+    assert f"UNEXPLAINED\t{hit}" in capsys.readouterr().out
+
+
+def test_other_cubic_stamps_and_zarr_contents_are_skipped(tmp_path, capsys):
+    """Only the requested cubic stamp is scanned, and zarr stores are not descended into."""
+    _save_dir(tmp_path, "er/fnet3d_paper/a549/a549__denv", [np.nan, np.nan], cubic="0.9.0a2")
+    _save_dir(tmp_path, "er/fnet3d_paper/a549/prediction.zarr/inner", [np.nan, np.nan])
+    assert main(["--root", str(tmp_path)]) == 0
+    assert "1 sidecars, 0 all-NaN" in capsys.readouterr().out