diff --git a/src/backend/services/docker_utils.py b/src/backend/services/docker_utils.py index 0daf596b35..9a26381f86 100644 --- a/src/backend/services/docker_utils.py +++ b/src/backend/services/docker_utils.py @@ -573,11 +573,35 @@ async def containers_run( loop = asyncio.get_event_loop() def _run(): + _cap_nano_cpus_to_host(kwargs) return docker_client.containers.run(image, command=command, **kwargs) return await loop.run_in_executor(_docker_executor, _run) +def _cap_nano_cpus_to_host(kwargs: Dict[str, Any]) -> None: + """#2318: Docker rejects NanoCpus above the host's CPU count with a 400, so a + template asking for more CPUs than the host has (trinity-system asks for 4) + could never be created on a smaller host. Cap the limit here, the one seam + every agent create/recreate goes through. The `trinity.cpu` label keeps the + requested value: the resource drift check compares it with the DB setting, + and a capped label would recreate the agent on every start.""" + requested = kwargs.get("nano_cpus") + if not requested: + return + try: + host_nano = int(docker_client.info()["NCPU"]) * 1_000_000_000 + except Exception as e: # noqa: BLE001 — unknown host size: let Docker decide + logger.warning(f"[#2318] could not read host CPU count, CPU limit not capped: {e}") + return + if requested > host_nano: + logger.warning( + f"[#2318] CPU limit {requested / 1e9:g} exceeds the host's " + f"{host_nano / 1e9:g} CPUs; capping to {host_nano / 1e9:g}" + ) + kwargs["nano_cpus"] = host_nano + + # ============================================================================= # Container Exec Operations # ============================================================================= diff --git a/tests/unit/test_2318_cpu_capped_to_host.py b/tests/unit/test_2318_cpu_capped_to_host.py new file mode 100644 index 0000000000..6f1257d1c7 --- /dev/null +++ b/tests/unit/test_2318_cpu_capped_to_host.py @@ -0,0 +1,59 @@ +""" +#2318: containers_run caps the requested CPU limit at the Docker host's CPU +count. Docker rejects NanoCpus above the host's CPUs with a 400, so a template +asking for cpu "4" (the trinity-system template does) could never be created on +a 2-CPU host. + +Module: src/backend/services/docker_utils.py +""" + +import sys +from pathlib import Path + +import pytest + +# Sibling import — the unit dir is not implicitly importable. +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) +from test_docker_utils import get_docker_utils # noqa: E402 + +GB = 1_000_000_000 + + +async def _run_with(host_cpus, **kwargs): + docker_utils, client = get_docker_utils() + client.info.return_value = {"NCPU": host_cpus} + await docker_utils.containers_run("img", detach=True, **kwargs) + return client.containers.run.call_args.kwargs + + +@pytest.mark.unit +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested,host,expected", + [ + (4, 2, 2), # the trinity-system template on a t3a.large + (4, 4, 4), # fits: unchanged + (2, 8, 2), # below the host: unchanged + (8, 3, 3), # an odd host count is a valid Docker limit + ], +) +async def test_nano_cpus_capped_to_host(requested, host, expected): + kwargs = await _run_with(host, nano_cpus=requested * GB) + assert kwargs["nano_cpus"] == expected * GB + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_no_cpu_limit_is_left_alone(): + kwargs = await _run_with(2) + assert "nano_cpus" not in kwargs + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_unreadable_host_count_passes_request_through(): + docker_utils, client = get_docker_utils() + client.info.side_effect = RuntimeError("daemon unreachable") + await docker_utils.containers_run("img", detach=True, nano_cpus=4 * GB) + assert client.containers.run.call_args.kwargs["nano_cpus"] == 4 * GB