From 0292f04d26bad918fd521d2ecc2f1acd6115735c Mon Sep 17 00:00:00 2001 From: obasilakis Date: Fri, 2 Oct 2026 16:51:08 +0200 Subject: [PATCH 1/2] fix(agents): cap container CPU limit at the Docker host's CPU count (#2318) Docker rejects NanoCpus above the host's CPU count with a 400. The trinity-system template asks for 4 CPUs, so on any 2-CPU host (a t3a.large, the AWS Marketplace default) the system agent was never created, failing again on every backend start. A user agent configured above the host's CPUs failed the same way. containers_run, the one seam every agent create and recreate goes through, now caps nano_cpus at docker info's NCPU and logs a warning. The trinity.cpu label keeps the requested value: the resource drift check compares it with the DB setting, and a capped label would recreate the agent on every start. If the host count can't be read, the request passes through unchanged. Verified on a 2-vCPU EC2 instance: with this file in place, trinity-system was created and runs with NanoCpus=2e9, label trinity.cpu=4. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/backend/services/docker_utils.py | 24 ++++++++++ tests/unit/test_2318_cpu_capped_to_host.py | 53 ++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 tests/unit/test_2318_cpu_capped_to_host.py diff --git a/src/backend/services/docker_utils.py b/src/backend/services/docker_utils.py index 0daf596b35..9a26381f86 100644 --- a/src/backend/services/docker_utils.py +++ b/src/backend/services/docker_utils.py @@ -573,11 +573,35 @@ async def containers_run( loop = asyncio.get_event_loop() def _run(): + _cap_nano_cpus_to_host(kwargs) return docker_client.containers.run(image, command=command, **kwargs) return await loop.run_in_executor(_docker_executor, _run) +def _cap_nano_cpus_to_host(kwargs: Dict[str, Any]) -> None: + """#2318: Docker rejects NanoCpus above the host's CPU count with a 400, so a + template asking for more CPUs than the host has (trinity-system asks for 4) + could never be created on a smaller host. Cap the limit here, the one seam + every agent create/recreate goes through. The `trinity.cpu` label keeps the + requested value: the resource drift check compares it with the DB setting, + and a capped label would recreate the agent on every start.""" + requested = kwargs.get("nano_cpus") + if not requested: + return + try: + host_nano = int(docker_client.info()["NCPU"]) * 1_000_000_000 + except Exception as e: # noqa: BLE001 — unknown host size: let Docker decide + logger.warning(f"[#2318] could not read host CPU count, CPU limit not capped: {e}") + return + if requested > host_nano: + logger.warning( + f"[#2318] CPU limit {requested / 1e9:g} exceeds the host's " + f"{host_nano / 1e9:g} CPUs; capping to {host_nano / 1e9:g}" + ) + kwargs["nano_cpus"] = host_nano + + # ============================================================================= # Container Exec Operations # ============================================================================= diff --git a/tests/unit/test_2318_cpu_capped_to_host.py b/tests/unit/test_2318_cpu_capped_to_host.py new file mode 100644 index 0000000000..06763e78d1 --- /dev/null +++ b/tests/unit/test_2318_cpu_capped_to_host.py @@ -0,0 +1,53 @@ +""" +#2318: containers_run caps the requested CPU limit at the Docker host's CPU +count. Docker rejects NanoCpus above the host's CPUs with a 400, so a template +asking for cpu "4" (the trinity-system template does) could never be created on +a 2-CPU host. + +Module: src/backend/services/docker_utils.py +""" + +import pytest + +from test_docker_utils import get_docker_utils + +GB = 1_000_000_000 + + +async def _run_with(host_cpus, **kwargs): + docker_utils, client = get_docker_utils() + client.info.return_value = {"NCPU": host_cpus} + await docker_utils.containers_run("img", detach=True, **kwargs) + return client.containers.run.call_args.kwargs + + +@pytest.mark.unit +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested,host,expected", + [ + (4, 2, 2), # the trinity-system template on a t3a.large + (4, 4, 4), # fits: unchanged + (2, 8, 2), # below the host: unchanged + (8, 3, 3), # an odd host count is a valid Docker limit + ], +) +async def test_nano_cpus_capped_to_host(requested, host, expected): + kwargs = await _run_with(host, nano_cpus=requested * GB) + assert kwargs["nano_cpus"] == expected * GB + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_no_cpu_limit_is_left_alone(): + kwargs = await _run_with(2) + assert "nano_cpus" not in kwargs + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_unreadable_host_count_passes_request_through(): + docker_utils, client = get_docker_utils() + client.info.side_effect = RuntimeError("daemon unreachable") + await docker_utils.containers_run("img", detach=True, nano_cpus=4 * GB) + assert client.containers.run.call_args.kwargs["nano_cpus"] == 4 * GB From 11950b153c3fa24e5249056f90c6517bb07e480f Mon Sep 17 00:00:00 2001 From: trinity-ability <309458136+trinity-ability@users.noreply.github.com> Date: Sun, 4 Oct 2026 21:07:20 +0100 Subject: [PATCH 2/2] merge-train: make the sibling test import resolvable (#3184) test_2318_cpu_capped_to_host imported get_docker_utils from test_docker_utils by bare name, but tests/unit is not on sys.path, so the file failed at collection and 'regression diff' went red on the PR's own test. Add the path preamble the other sibling-importing unit tests use (test_2392, test_3185). Mechanical, per the merge-train note on the PR. Co-Authored-By: Claude Fable 5.1 --- tests/unit/test_2318_cpu_capped_to_host.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_2318_cpu_capped_to_host.py b/tests/unit/test_2318_cpu_capped_to_host.py index 06763e78d1..6f1257d1c7 100644 --- a/tests/unit/test_2318_cpu_capped_to_host.py +++ b/tests/unit/test_2318_cpu_capped_to_host.py @@ -7,9 +7,15 @@ Module: src/backend/services/docker_utils.py """ +import sys +from pathlib import Path + import pytest -from test_docker_utils import get_docker_utils +# Sibling import — the unit dir is not implicitly importable. +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) +from test_docker_utils import get_docker_utils # noqa: E402 GB = 1_000_000_000