diff --git a/applications/dynacell/pyproject.toml b/applications/dynacell/pyproject.toml index 85a49aba9..54cbdda0b 100644 --- a/applications/dynacell/pyproject.toml +++ b/applications/dynacell/pyproject.toml @@ -62,9 +62,12 @@ optional-dependencies.eval = [ # with `normalize`/`normalization` rather than returning a quietly wrong # number, and the FSC/spectral, MicroSSIM, MS-SSIM, feature (KID/FID) and # average_precision modules were rewritten. Every pixel/feature metric in the - # benchmark can move across this pin, so it is paired with a full re-eval -- - # do not bump it casually. Tag, not a bare commit. - "cubic @ git+https://github.com/alxndrkalinin/cubic.git@v0.9.0a1", + # benchmark can move across this pin, so a bump needs either a full re-eval or + # a measured entry in dynacell.evaluation.provenance.CUBIC_VERSIONS_EQUIVALENT_TO + # -- do not bump it casually. Tag, not a bare commit. v0.9.0a2 rewrites only the + # MicroSSIM RI-factor fit: chunked float64 reduction (peak 40.5 -> 12.9 GB on + # 576x640x960 float32) and a ValueError on empty input; measured equivalent. + "cubic @ git+https://github.com/alxndrkalinin/cubic.git@v0.9.0a2", "dinov3 @ git+https://github.com/facebookresearch/dinov3", "dynaclr", "hydra-core>=1.2", @@ -101,7 +104,7 @@ optional-dependencies.eval_gpu = [ # selection->waveorder, segmentation->cellpose (cellpose imported lazily). optional-dependencies.preprocess = [ "cellpose>=4.2", - "cubic @ git+https://github.com/alxndrkalinin/cubic.git@v0.9.0a1", + "cubic @ git+https://github.com/alxndrkalinin/cubic.git@v0.9.0a2", "iohub>=0.3.6", "tqdm", "waveorder", diff --git a/applications/dynacell/src/dynacell/evaluation/CLAUDE.md b/applications/dynacell/src/dynacell/evaluation/CLAUDE.md index c248f4a11..46fcfecbd 100644 --- a/applications/dynacell/src/dynacell/evaluation/CLAUDE.md +++ b/applications/dynacell/src/dynacell/evaluation/CLAUDE.md @@ -5,7 +5,7 @@ computing) for any GPU-accelerated numerical work — image preprocessing before / after model inference, metric calculations, cropping/resizing, percentile clips, Gaussian filters, etc. Cubic is a hard runtime dependency of the eval extras (`applications/dynacell/pyproject.toml` -pins `cubic @ git+…@v0.9.0a1`). Do not gate cubic imports behind `try/except` +pins `cubic @ git+…@v0.9.0a2`). Do not gate cubic imports behind `try/except` or fall back to scipy/skimage paths. **The cubic version is part of the numeric contract, not just a dep.** Metric @@ -23,6 +23,10 @@ values move across pins — measured 0.8.0a2 → 0.9.0a1: `Z_FSC_Resolution` Unstamped means "written before the stamp existed", i.e. an unknown pin — it must recompute, unlike the per-extractor `preprocess_version` bootstrap which treats an untagged entry as unconstrained. +- A bump measured to move no value beyond a stated tolerance lists the old + version under the new one in `CUBIC_VERSIONS_EQUIVALENT_TO` so its caches are still reused + (0.9.0a1 under 0.9.0a2). The map is keyed by the declared version, so the + next bump reuses nothing until someone measures an equivalence against it. The GPU-resident Cellpose-SAM entry point is `cubic.segmentation.segment_cpsam` (single host→device upload, masks diff --git a/applications/dynacell/src/dynacell/evaluation/provenance.py b/applications/dynacell/src/dynacell/evaluation/provenance.py index 3710d141f..3b447b1c3 100644 --- a/applications/dynacell/src/dynacell/evaluation/provenance.py +++ b/applications/dynacell/src/dynacell/evaluation/provenance.py @@ -29,6 +29,11 @@ and :func:`metrics_provenance_matches` lets the final-metrics cache gate refuse a cache built by a different ``cubic``. +A version bump measured to move no value beyond a stated tolerance lists the old +version under the new one in :data:`CUBIC_VERSIONS_EQUIVALENT_TO`, so its caches +stay reusable instead of forcing every leaf to recompute. 0.9.0a1 -> 0.9.0a2 is such a +bump: it rewrites only the MicroSSIM RI-factor fit. + The same sidecar stamps the content hash of the CP reference the CP feature metrics were scored in (:mod:`dynacell.evaluation.cp_reference`). CP KID/FID/ cosine move whenever the reference is rebuilt, so a cache stamped with another @@ -43,7 +48,25 @@ #: The ``cubic`` version this repo is built against. Must equal the pin in #: ``applications/dynacell/pyproject.toml``; ``provenance_test.py`` asserts #: they cannot drift apart. -REQUIRED_CUBIC_VERSION = "0.9.0a1" +REQUIRED_CUBIC_VERSION = "0.9.0a2" + +#: For each declared ``cubic`` version, the earlier versions whose metric values it +#: reproduces within a stated tolerance, so their caches stay reusable. Keyed by the +#: declared version so a bump starts with no equivalents until one is measured +#: against it. +#: +#: 0.9.0a1 -> 0.9.0a2 changes only the MicroSSIM RI-factor fit: it reduces the +#: objective in bounded chunks, accumulating in float64 where 0.9.0a1 took a +#: float32 ``.mean()``, and raises ``ValueError`` on an empty pool. Tolerance: MicroMS3IM relative delta <= 1e-6, far +#: below the tables' 2-decimal rounding. Measured 2026-09-28 on the A549-trained +#: ``fnet3d_paper`` ER ``a549__denv`` leaf at the production calibration settings +#: (``max_pairs=12``, seed 42, a 576x640x960 float32 pool): the fitted alpha is +#: 17.883591651916504 under both versions, and MicroMS3IM over all 108 (FOV, t) +#: differs by 0 abs / 0 rel. The fit objective itself moves ~7e-8 relative between +#: the versions; the root finder lands on the same alpha. The other +#: ``pixel_metrics`` / ``mask_metrics`` columns do not use the RI-factor code and +#: were bit-identical on 1 FOV x 7 timepoints. +CUBIC_VERSIONS_EQUIVALENT_TO = {"0.9.0a2": frozenset({"0.9.0a1"})} #: Sidecar written next to ``pixel_metrics.csv`` by :func:`write_metrics_provenance`. PROVENANCE_FILENAME = "metrics_provenance.json" @@ -142,15 +165,20 @@ def metrics_provenance_matches(save_dir: Path, *, cp_space_sha256: str | None) - Returns ------- bool - True when the recorded ``cubic`` version equals the installed one and, - if ``cp_space_sha256`` is given, the recorded binding equals it. + True when the recorded ``cubic`` version equals the installed one (or is + in :data:`CUBIC_VERSIONS_EQUIVALENT_TO` for the declared pin while the installed one is + the declared pin) and, if ``cp_space_sha256`` is given, the recorded binding + equals it. """ path = save_dir / PROVENANCE_FILENAME if not path.is_file(): return False payload = json.loads(path.read_text()) recorded = payload.get("versions", {}).get("cubic") - if recorded != version("cubic"): + installed = version("cubic") + equivalents = CUBIC_VERSIONS_EQUIVALENT_TO.get(REQUIRED_CUBIC_VERSION, frozenset()) + reproduced = installed == REQUIRED_CUBIC_VERSION and recorded in equivalents + if recorded != installed and not reproduced: return False if cp_space_sha256 is None: return True diff --git a/applications/dynacell/src/dynacell/evaluation/provenance_test.py b/applications/dynacell/src/dynacell/evaluation/provenance_test.py index 7ae32db37..978f3a604 100644 --- a/applications/dynacell/src/dynacell/evaluation/provenance_test.py +++ b/applications/dynacell/src/dynacell/evaluation/provenance_test.py @@ -8,6 +8,7 @@ import pytest from dynacell.evaluation.provenance import ( + CUBIC_VERSIONS_EQUIVALENT_TO, PROVENANCE_FILENAME, REQUIRED_CUBIC_VERSION, check_cubic_pin, @@ -82,12 +83,66 @@ def test_missing_sidecar_is_not_a_match(tmp_path): assert not metrics_provenance_matches(tmp_path, cp_space_sha256=None) -def test_foreign_cubic_version_is_not_a_match(tmp_path): - write_metrics_provenance(tmp_path, cp_reference_sha256=None, cp_space_sha256=None) - path = tmp_path / PROVENANCE_FILENAME +def _stamp_cubic(save_dir, cubic, *, cp_space_sha256): + """Write a sidecar as if a run under ``cubic`` had produced it.""" + cp_reference_sha256 = None if cp_space_sha256 is None else f"ref-{cp_space_sha256}" + write_metrics_provenance(save_dir, cp_reference_sha256=cp_reference_sha256, cp_space_sha256=cp_space_sha256) + path = save_dir / PROVENANCE_FILENAME payload = json.loads(path.read_text()) - payload["versions"]["cubic"] = "0.8.0a2" + payload["versions"]["cubic"] = cubic path.write_text(json.dumps(payload)) + + +def test_foreign_cubic_version_is_not_a_match(tmp_path): + _stamp_cubic(tmp_path, "0.8.0a2", cp_space_sha256=None) + assert not metrics_provenance_matches(tmp_path, cp_space_sha256=None) + + +@pytest.fixture +def running_the_declared_pin(monkeypatch): + """Report the declared cubic as installed, whatever the test venv holds.""" + monkeypatch.setattr("dynacell.evaluation.provenance.version", lambda _name: REQUIRED_CUBIC_VERSION) + + +def test_equivalent_versions_exclude_the_declared_one(): + equivalents = CUBIC_VERSIONS_EQUIVALENT_TO[REQUIRED_CUBIC_VERSION] + assert "0.9.0a1" in equivalents + assert REQUIRED_CUBIC_VERSION not in equivalents + + +def test_equivalent_cubic_version_is_a_match(tmp_path, running_the_declared_pin): + """A 0.9.0a1 cache holds the values the declared pin reproduces, so it is reused.""" + _stamp_cubic(tmp_path, "0.9.0a1", cp_space_sha256=None) + assert metrics_provenance_matches(tmp_path, cp_space_sha256=None) + + +def test_older_cubic_version_is_still_not_a_match(tmp_path, running_the_declared_pin): + """0.8.0a2 moved FSC/FRC/Spectral_PCC, so its caches stay refused.""" + _stamp_cubic(tmp_path, "0.8.0a2", cp_space_sha256=None) + assert not metrics_provenance_matches(tmp_path, cp_space_sha256=None) + + +def test_equivalent_cubic_version_keeps_the_cp_rules(tmp_path, running_the_declared_pin): + """Accepting the cubic stamp does not relax the CP-space binding check.""" + _stamp_cubic(tmp_path, "0.9.0a1", cp_space_sha256="abc") + assert metrics_provenance_matches(tmp_path, cp_space_sha256="abc") + assert not metrics_provenance_matches(tmp_path, cp_space_sha256="other") + _stamp_cubic(tmp_path, "0.9.0a1", cp_space_sha256=None) + assert not metrics_provenance_matches(tmp_path, cp_space_sha256="abc") + + +def test_equivalence_only_applies_under_the_declared_pin(tmp_path, monkeypatch): + """An environment off the pin gets no equivalence: only its own version matches.""" + monkeypatch.setattr("dynacell.evaluation.provenance.version", lambda _name: "0.8.0a2") + _stamp_cubic(tmp_path, "0.9.0a1", cp_space_sha256=None) + assert not metrics_provenance_matches(tmp_path, cp_space_sha256=None) + + +def test_unmeasured_pin_has_no_equivalents(tmp_path, monkeypatch): + """A newly declared pin with no measured entry reuses no older cache.""" + monkeypatch.setattr("dynacell.evaluation.provenance.REQUIRED_CUBIC_VERSION", "0.9.0a3") + monkeypatch.setattr("dynacell.evaluation.provenance.version", lambda _name: "0.9.0a3") + _stamp_cubic(tmp_path, "0.9.0a1", cp_space_sha256=None) assert not metrics_provenance_matches(tmp_path, cp_space_sha256=None) diff --git a/applications/dynacell/tools/README.md b/applications/dynacell/tools/README.md index a7a6cbe45..a4d28c268 100644 --- a/applications/dynacell/tools/README.md +++ b/applications/dynacell/tools/README.md @@ -40,6 +40,8 @@ builders. Not part of the installable package — run with `uv run python applic - `smoke_joint_leaf.py` — smoke a joint (multi-dataset) training leaf. - `nccl_smoke_test.py` — probe multi-GPU NCCL connectivity before a DDP fit. +- `scan_nan_microssim_caches.py` — read-only: list eval caches stamped with a cubic version (default 0.9.0a1) + whose MicroMS3IM is entirely NaN, i.e. caches a cubic-equivalence reuse would carry a NaN forward in. - `*_test.py` — pytest for the generators (`generate_grouped_eval_configs_test.py`, `generate_instance_ap_eval_configs_test.py`); further tool tests live in [`../tests/`](../tests/README.md). - `slurm_logs/` — captured job logs (runtime output, not source). diff --git a/applications/dynacell/tools/run_eval_direct.slurm b/applications/dynacell/tools/run_eval_direct.slurm index cc92946c9..e00550a77 100755 --- a/applications/dynacell/tools/run_eval_direct.slurm +++ b/applications/dynacell/tools/run_eval_direct.slurm @@ -74,8 +74,10 @@ ml uv # the 0.9.0a1 venv existed, so jobs that did not export DYNACELL_EVAL_VENV kept # silently producing 0.8.0a2 FSC/FRC/Spectral_PCC values — 62 of the 121 # surviving eval logs. ``check_cubic_pin`` now fails such a run loudly, but the -# default still has to be right. -export UV_PROJECT_ENVIRONMENT="${DYNACELL_EVAL_VENV:-/hpc/mydata/alex.kalinin/VisCy-venvs/cpdino-eval-cubic090a1}" +# default still has to be right. ``cpdino-eval-cubic090a2`` is the +# ``cpdino-eval-cubic090a1`` freeze with only cubic changed; 0.9.0a1 caches stay +# reusable under it (``provenance.CUBIC_VERSIONS_EQUIVALENT_TO``). +export UV_PROJECT_ENVIRONMENT="${DYNACELL_EVAL_VENV:-/hpc/mydata/alex.kalinin/VisCy-venvs/cpdino-eval-cubic090a2}" export HF_HUB_CACHE="${HF_HUB_CACHE:-/hpc/projects/comp.micro/virtual_staining/models/dynacell/evaluation/hf_cache}" export PYTHONUNBUFFERED=1 diff --git a/applications/dynacell/tools/scan_nan_microssim_caches.py b/applications/dynacell/tools/scan_nan_microssim_caches.py new file mode 100644 index 000000000..65d08bcd7 --- /dev/null +++ b/applications/dynacell/tools/scan_nan_microssim_caches.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +r"""List eval caches stamped with a given cubic whose MicroMS3IM column is entirely NaN. + +``provenance.CUBIC_VERSIONS_EQUIVALENT_TO`` lets 0.9.0a2 reuse 0.9.0a1 caches. That +holds only where the 0.9.0a1 cache holds a value. When the leaf-level MicroSSIM +calibration ran out of memory under 0.9.0a1, ``_calibrate_microssim``'s caller +wrote ``MicroMS3IM = NaN`` for every FOV. 0.9.0a2's bounded-memory fit would score +those leaves, but the reused cache keeps the NaN. This scan finds them so they can +be re-run with ``force_recompute.final_metrics=true``. + +Mito A549 mock caches are all-NaN for a reason that predates this bump (the +MicroSSIM ``data_range`` case), so they are reported as known rather than +unexplained. Read-only: the tool never writes. + +Exit status: 0 when no unexplained cache is found, 1 otherwise. + +Usage:: + + uv run --no-sync python applications/dynacell/tools/scan_nan_microssim_caches.py \ + --root /hpc/projects/virtual_staining/training/dynacell +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from pathlib import Path + +import pandas as pd + +from dynacell.evaluation.provenance import PROVENANCE_FILENAME + +#: Save dirs whose all-NaN MicroMS3IM is the mito A549 mock ``data_range`` case: +#: canonical ``mito///a549__mock`` and legacy +#: ``a549/evaluations_with_embeddings/eval__mitochondria_mock``. +_KNOWN_ALL_NAN = re.compile(r"(/mito/[^/]+/[^/]+/a549__mock|/a549/[^/]+/eval_[^/]*mitochondria_mock)$") + + +def find_sidecars(root: Path, max_depth: int) -> list[Path]: + """Return every provenance sidecar under ``root``, not descending into zarr stores.""" + found = [] + base = len(root.parts) + for dirpath, dirnames, filenames in os.walk(root): + depth = len(Path(dirpath).parts) - base + dirnames[:] = [d for d in dirnames if not d.endswith((".zarr", ".ozx")) and depth < max_depth] + if PROVENANCE_FILENAME in filenames: + found.append(Path(dirpath) / PROVENANCE_FILENAME) + return sorted(found) + + +def all_nan_microssim(sidecars: list[Path], cubic: str) -> list[Path]: + """Return the save dirs among ``sidecars`` stamped ``cubic`` whose MicroMS3IM is entirely NaN.""" + hits = [] + for sidecar in sidecars: + if json.loads(sidecar.read_text())["versions"]["cubic"] != cubic: + continue + column = pd.read_csv(sidecar.parent / "pixel_metrics.csv")["MicroMS3IM"] + if column.isna().all(): + hits.append(sidecar.parent) + return hits + + +def is_known_all_nan(save_dir: Path) -> bool: + """Whether ``save_dir`` is the mito A549 mock case, all-NaN independent of cubic.""" + return _KNOWN_ALL_NAN.search(save_dir.as_posix()) is not None + + +def main(argv: list[str] | None = None) -> int: + """Run the scan and print one line per all-NaN cache.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--cubic", default="0.9.0a1", help="cubic version stamp to scan (default: 0.9.0a1)") + parser.add_argument("--max-depth", type=int, default=6, help="directory depth below --root to search") + args = parser.parse_args(argv) + + sidecars = find_sidecars(args.root, args.max_depth) + hits = all_nan_microssim(sidecars, args.cubic) + unexplained = [d for d in hits if not is_known_all_nan(d)] + for save_dir in hits: + print(f"{'known' if is_known_all_nan(save_dir) else 'UNEXPLAINED'}\t{save_dir}") + print( + f"{len(sidecars)} sidecars, {len(hits)} all-NaN MicroMS3IM stamped cubic {args.cubic}, " + f"{len(unexplained)} unexplained" + ) + return 1 if unexplained else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/applications/dynacell/tools/scan_nan_microssim_caches_test.py b/applications/dynacell/tools/scan_nan_microssim_caches_test.py new file mode 100644 index 000000000..1e30abc52 --- /dev/null +++ b/applications/dynacell/tools/scan_nan_microssim_caches_test.py @@ -0,0 +1,64 @@ +"""Integration tests for ``scan_nan_microssim_caches.py``. + +Each test lays out real eval save dirs in ``tmp_path`` (sidecar written by +``write_metrics_provenance``, then restamped) and runs the tool's ``main``. +""" + +from __future__ import annotations + +import json + +import numpy as np +import pandas as pd +from scan_nan_microssim_caches import main # noqa: E402 + +from dynacell.evaluation.provenance import PROVENANCE_FILENAME, write_metrics_provenance + + +def _save_dir(root, rel, microssim, cubic="0.9.0a1"): + save_dir = root / rel + save_dir.mkdir(parents=True) + write_metrics_provenance(save_dir, cp_reference_sha256=None, cp_space_sha256=None) + sidecar = save_dir / PROVENANCE_FILENAME + payload = json.loads(sidecar.read_text()) + payload["versions"]["cubic"] = cubic + sidecar.write_text(json.dumps(payload)) + pd.DataFrame({"FOV": ["a", "b"], "MicroMS3IM": microssim}).to_csv(save_dir / "pixel_metrics.csv", index=False) + return save_dir + + +def test_unexplained_all_nan_cache_fails(tmp_path, capsys): + """An all-NaN cache outside the known case is reported and fails the scan; a partial NaN is not.""" + hit = _save_dir(tmp_path, "er/fnet3d_paper/a549/a549__denv", [np.nan, np.nan]) + _save_dir(tmp_path, "er/fnet3d_paper/a549/a549__zikv", [0.8, np.nan]) + assert main(["--root", str(tmp_path)]) == 1 + out = capsys.readouterr().out + assert f"UNEXPLAINED\t{hit}" in out + assert "a549__zikv" not in out + assert "2 sidecars, 1 all-NaN MicroMS3IM stamped cubic 0.9.0a1, 1 unexplained" in out + + +def test_mito_a549_mock_is_known(tmp_path, capsys): + """Canonical and legacy mito A549 mock dirs are reported as known and pass.""" + canonical = _save_dir(tmp_path, "mito/fnet2d/joint/a549__mock", [np.nan, np.nan]) + legacy = _save_dir(tmp_path, "a549/evaluations_with_embeddings/eval_phase_mitochondria_mock", [np.nan, np.nan]) + assert main(["--root", str(tmp_path)]) == 0 + out = capsys.readouterr().out + assert f"known\t{canonical}" in out + assert f"known\t{legacy}" in out + assert "0 unexplained" in out + + +def test_mito_other_conditions_are_not_known(tmp_path, capsys): + """The known case is mito A549 mock only, not every mito condition.""" + hit = _save_dir(tmp_path, "mito/fnet2d/joint/a549__denv", [np.nan, np.nan]) + assert main(["--root", str(tmp_path)]) == 1 + assert f"UNEXPLAINED\t{hit}" in capsys.readouterr().out + + +def test_other_cubic_stamps_and_zarr_contents_are_skipped(tmp_path, capsys): + """Only the requested cubic stamp is scanned, and zarr stores are not descended into.""" + _save_dir(tmp_path, "er/fnet3d_paper/a549/a549__denv", [np.nan, np.nan], cubic="0.9.0a2") + _save_dir(tmp_path, "er/fnet3d_paper/a549/prediction.zarr/inner", [np.nan, np.nan]) + assert main(["--root", str(tmp_path)]) == 0 + assert "1 sidecars, 0 all-NaN" in capsys.readouterr().out diff --git a/uv.lock b/uv.lock index d4d7885da..5b8a01d11 100644 --- a/uv.lock +++ b/uv.lock @@ -941,8 +941,8 @@ wheels = [ [[package]] name = "cubic" -version = "0.9.0a1" -source = { git = "https://github.com/alxndrkalinin/cubic.git?rev=v0.9.0a1#8b6d5090426b8b6511e6251063109c957ff33715" } +version = "0.9.0a2" +source = { git = "https://github.com/alxndrkalinin/cubic.git?rev=v0.9.0a2#2efdeb0e68d2e7c1389a3039c33bfa2024dcb0da" } dependencies = [ { name = "numpy" }, { name = "scikit-image" }, @@ -1401,8 +1401,8 @@ requires-dist = [ { name = "boto3", marker = "extra == 'distribution'" }, { name = "cellpose", marker = "extra == 'eval'", specifier = ">=4.2" }, { name = "cellpose", marker = "extra == 'preprocess'", specifier = ">=4.2" }, - { name = "cubic", marker = "extra == 'eval'", git = "https://github.com/alxndrkalinin/cubic.git?rev=v0.9.0a1" }, - { name = "cubic", marker = "extra == 'preprocess'", git = "https://github.com/alxndrkalinin/cubic.git?rev=v0.9.0a1" }, + { name = "cubic", marker = "extra == 'eval'", git = "https://github.com/alxndrkalinin/cubic.git?rev=v0.9.0a2" }, + { name = "cubic", marker = "extra == 'preprocess'", git = "https://github.com/alxndrkalinin/cubic.git?rev=v0.9.0a2" }, { name = "cucim-cu13", marker = "extra == 'eval-gpu'", index = "https://pypi.nvidia.com/" }, { name = "cupy-cuda13x", marker = "extra == 'eval-gpu'" }, { name = "dinov3", marker = "extra == 'eval'", git = "https://github.com/facebookresearch/dinov3" },