-
Notifications
You must be signed in to change notification settings - Fork 7
Expand file tree
/
Copy pathcbf_engine.py
More file actions
1971 lines (1747 loc) · 89.8 KB
/
Copy pathcbf_engine.py
File metadata and controls
1971 lines (1747 loc) · 89.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Stateful Redis-backed financial invariant enforcer.
Implements a discrete-time Control Barrier Function (CBF) enforcing
h(S(t+1)) >= (1 - gamma) * h(S(t)) >= 0 for all t >= 0, gamma in (0, 1).
Uses Redis for state persistence so that distributed gateway instances share a
consistent cash-balance view within a single-primary epoch.
Phase 1 fix: imports now resolve against the canonical gateway-internal
infrastructure package (``src.gateway.infrastructure.*``) instead of the
cross-package ``src.governed_financial_advisor.*`` path.
In CAGE v3.0.0, ``atomic_verify_and_commit()`` executes the CBF condition
evaluation and state deduction in a single Redis Lua script, eliminating
TOCTOU windows between check and commit.
"""
import asyncio
import inspect
import json
import logging
import math
# ---------------------------------------------------------------------------
# Canonical gateway-internal imports (Phase 1.1)
# ---------------------------------------------------------------------------
import os
import time
from dataclasses import dataclass
from typing import Any, Optional
from src.gateway.governance.constants import ControlRegistry, GovernanceControl
from src.gateway.governance.contracts import InvariantModel
# ---------------------------------------------------------------------------
# Threshold singleton (Phase 2.3)
# ---------------------------------------------------------------------------
from src.gateway.governance.schemas.thresholds import THRESHOLDS
from src.gateway.infrastructure.redis_client import redis_client, sync_redis_client
from src.gateway.infrastructure.telemetry import get_tracer
logger = logging.getLogger("SafetyLayer")
# ---------------------------------------------------------------------------
# Environment detection (module-level so tests can patch it)
# ---------------------------------------------------------------------------
_cage_env_cbf = (
os.environ.get("CAGE_ENV") or os.environ.get("ENVIRONMENT", "production")
).lower()
_IS_PRODUCTION: bool = _cage_env_cbf not in ("development", "test", "dev", "ci")
# ---------------------------------------------------------------------------
# Feature flag: CBF strict mode - fail-closed on ground truth unavailability
# ---------------------------------------------------------------------------
# When CAGE_CBF_STRICT_MODE=true (default), the CBF rejects transactions when
# KMS verification of reconciled balance fails. This prevents fail-open behavior
# where an attacker could exhaust KMS quota to force unverified balance usage.
# Set to "false" only in isolated development/test environments.
_CBF_STRICT_MODE: bool = os.environ.get(
"CAGE_CBF_STRICT_MODE", "true" if _IS_PRODUCTION else "false"
).lower() in ("true", "1", "yes")
# ---------------------------------------------------------------------------
# Feature flag: Replay defense (R-04 mitigation, §2.10)
# ---------------------------------------------------------------------------
# Stage 1 (read-side): When enabled, CBF enforces sequence validation.
_REPLAY_DEFENSE_ENABLED: bool = os.environ.get(
"CAGE_RECONCILIATION_REPLAY_DEFENSE", "false"
).lower() in ("true", "1", "yes")
# Redis key for tracking last accepted sequence (never TTL'd)
_REDIS_KEY_SEQUENCE_LAST_ACCEPTED = "reconciliation:sequence:last_accepted"
# ---------------------------------------------------------------------------
# Feature flag: Fence epoch validation (R-05 mitigation, §2.6)
# ---------------------------------------------------------------------------
# When enabled, CBF validates fence epoch hasn't regressed after failover.
# This detects stale reads from replicas that haven't caught up to primary.
# DEFAULT CHANGED (peer review Fix A2): Enabled by default to provide failover
# protection out-of-box. Operators can disable with CAGE_REDIS_SYNCHRONOUS_REPLICATION=false.
# Cross-region impact: US_FED, EU_ECB, APAC_MAS all benefit from failover safety.
_FENCE_EPOCH_ENABLED: bool = os.environ.get(
"CAGE_REDIS_SYNCHRONOUS_REPLICATION", "true"
).lower() in ("true", "1", "yes")
# Redis key for fence epoch counter (never TTL'd)
_REDIS_KEY_FENCE_EPOCH = "safety:fence_epoch"
# ---------------------------------------------------------------------------
# Local debits tracking (POAM-023 remediation)
# ---------------------------------------------------------------------------
# Redis key for tracking debits within a reconciliation cycle to prevent
# double-spend when using reconciled balance with TTL window.
_REDIS_KEY_LOCAL_DEBITS = "cbf:local_debits"
# ---------------------------------------------------------------------------
# Feature flag: WAIT command replication (Phase 4.3)
# ---------------------------------------------------------------------------
# When CAGE_REDIS_WAIT_REPLICAS > 0, CBF will call Redis WAIT after fence
# epoch increment to ensure the epoch is replicated before returning.
# https://redis.io/commands/wait/
# DEFAULT CHANGED (peer review Fix A1): Enabled by default (1 replica) to ensure
# durability before returning success to caller. Set CAGE_REDIS_WAIT_REPLICAS=0 to disable.
# Cross-region impact: US_FED, EU_ECB, APAC_MAS all benefit from replication guarantee.
_WAIT_REPLICAS: int = int(os.environ.get("CAGE_REDIS_WAIT_REPLICAS", "1"))
_WAIT_TIMEOUT_MS: int = int(os.environ.get("CAGE_REDIS_WAIT_TIMEOUT_MS", "1000"))
# ---------------------------------------------------------------------------
# Feature flag: Strict replication mode (P0 security hardening)
# ---------------------------------------------------------------------------
# When CAGE_STRICT_REPLICATION=true (default in production), a WAIT timeout
# triggers a fail-closed rollback rather than logging-only. This prevents
# financial actions from succeeding when async replication cannot confirm
# the mutation reached replicas — if the primary crashes before replication
# and Sentinel promotes a replica, that replica would be missing the mutation.
# Cross-region impact: US_FED, EU_ECB, APAC_MAS all benefit from fail-closed safety.
_STRICT_REPLICATION: bool = os.environ.get(
"CAGE_STRICT_REPLICATION", "true" if _IS_PRODUCTION else "false"
).lower() in ("true", "1", "yes")
# Sentinel awareness (Phase 4.3 stretch goal)
# When set, connection should be Sentinel-aware for automatic failover handling.
_REDIS_SENTINEL_MASTER_NAME: str | None = os.environ.get("REDIS_SENTINEL_MASTER_NAME")
# ---------------------------------------------------------------------------
# Prometheus telemetry for replay defense (§2.10) and WAIT replication (§4.3)
# ---------------------------------------------------------------------------
try:
from prometheus_client import REGISTRY, Counter, Gauge, Histogram
def _get_or_create_metric(
metric_cls: Any, name: str, *args: Any, **kwargs: Any
) -> Any:
"""Thread-safe metric registry lookup with fallback to singleton.
Race-safe: If metric creation fails due to duplicate registration
(ValueError from Prometheus), falls back to the shared singleton
from REGISTRY._names_to_collectors. Concurrent calls will all
return the same collector instance (intended behavior).
The broad exception handler catches both:
- ValueError: Raised by Prometheus for duplicate metric names
- Exception: Any unexpected Prometheus internal errors
Args:
metric_cls: Prometheus metric class (Counter, Gauge, Histogram)
name: Metric name
*args, **kwargs: Arguments passed to metric constructor
Returns:
Prometheus collector instance (new or existing singleton)
Raises:
Exception: If metric creation fails and no existing collector found
"""
try:
return metric_cls(name, *args, **kwargs)
except (ValueError, Exception):
collector = REGISTRY._names_to_collectors.get(name)
if collector is not None:
return collector
raise
_REPLAY_REJECTED_COUNTER = _get_or_create_metric(
Counter,
"cage_reconciliation_replay_rejected_total",
"Number of reconciliation payloads rejected due to non-advancing sequence (R-04 replay defense)",
["source"],
)
# R-05 fence epoch telemetry
_EPOCH_REGRESSION_COUNTER = _get_or_create_metric(
Counter,
"cage_cbf_epoch_regression_detected_total",
"Number of CBF reads rejected due to fence epoch regression (R-05 double-spend defense)",
)
_CURRENT_FENCE_EPOCH_GAUGE = _get_or_create_metric(
Gauge,
"cage_cbf_current_fence_epoch",
"Current value of the CBF fence epoch counter",
)
# Phase 4.3: WAIT command telemetry
_WAIT_LATENCY_HISTOGRAM = _get_or_create_metric(
Histogram,
"cage_cbf_wait_latency_seconds",
"Latency of Redis WAIT command for replication synchronization (Phase 4.3)",
buckets=(0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5),
)
_WAIT_TIMEOUT_COUNTER = _get_or_create_metric(
Counter,
"cage_cbf_wait_timeout_total",
"Number of Redis WAIT commands that timed out before reaching replica count (Phase 4.3)",
)
# P0 hardening: Strict replication rollback counter
_STRICT_REPLICATION_ROLLBACK_COUNTER = _get_or_create_metric(
Counter,
"cage_cbf_strict_replication_rollback_total",
"Number of CBF commits rolled back due to WAIT timeout in strict replication mode (P0 hardening)",
)
except ImportError:
_REPLAY_REJECTED_COUNTER = None # type: ignore[assignment]
_EPOCH_REGRESSION_COUNTER = None # type: ignore[assignment]
_CURRENT_FENCE_EPOCH_GAUGE = None # type: ignore[assignment]
_WAIT_LATENCY_HISTOGRAM = None # type: ignore[assignment]
_WAIT_TIMEOUT_COUNTER = None # type: ignore[assignment]
_STRICT_REPLICATION_ROLLBACK_COUNTER = None # type: ignore[assignment]
# ---------------------------------------------------------------------------
# CBFInitializationError — Fail-closed exception for epoch seeding (B3a)
# ---------------------------------------------------------------------------
class GroundTruthUnavailableError(RuntimeError):
"""Raised when CBF cannot verify ground truth balance (fail-closed behavior).
This exception is raised when:
- KMS signature verification fails on the reconciled balance
- Reconciled balance read fails entirely
- Production environment receives unsigned balance
In strict mode (CAGE_CBF_STRICT_MODE=true, default), this error is raised
to prevent transactions from proceeding with unverified self-reported balance.
This is a security control to prevent attackers from exhausting KMS quota
to force unverified balance usage.
"""
def __init__(self, message: str, cause: Exception | None = None):
super().__init__(message)
self.cause = cause
class CBFInitializationError(RuntimeError):
"""Raised when CBF cannot seed its initial fence epoch from Redis.
§B3a: A newly spawned gateway instance (after restart, redeploy, or
autoscale) must seed its fence epoch from Redis before accepting
requests. If Redis is unavailable at initialization time, the CBF
MUST fail-closed rather than starting with epoch=0, which would
create a window for stale-read attacks.
This exception should propagate to the pod readiness probe, preventing
the instance from joining the load-balancer pool until Redis is
reachable and the epoch is successfully seeded.
"""
async def _get_raw_redis(r: Any) -> Any:
"""Helper to extract raw redis client from wrapper or mock."""
if r is None:
return None
getter = getattr(r, "get_raw_client", None)
if callable(getter):
client = getter()
if inspect.isawaitable(client):
client = await client
return client
return r
# ---------------------------------------------------------------------------
# ControlBarrierFunction
# ---------------------------------------------------------------------------
@dataclass(frozen=True)
class _DefaultKernelBarrier:
"""Kernel fallback barrier for backward compatibility and test isolation.
Implements the InvariantModel protocol without importing domain plugins.
"""
invariant_id: str = "default.cash_balance"
state_key: str = "safety:current_cash"
threshold_key: str = "cbf.min_cash_balance"
gamma: float = 0.5
class ControlBarrierFunction:
"""Discrete-time Control Barrier Function (CBF).
Uses Redis for state persistence so that stateless Cloud Run instances
share a consistent cash-balance view.
Phase 4.1: ``update_state()`` and ``rollback_state()`` wrap all
read-modify-write operations in a Redis WATCH / MULTI / EXEC optimistic-
locking pipeline. If another process mutates the key between the WATCH
and the EXEC, the transaction is aborted and retried up to
``_MAX_RETRIES`` times before raising ``RuntimeError``.
"""
_MAX_RETRIES: int = 5
LUA_ATOMIC_CBF: str = """
-- PR C (Stage 2): Parameterized affine barrier script.
-- Driven by InvariantModel: h(x) = state[state_key] - thresholds[threshold_key]
--
-- KEYS[1]: <InvariantModel.state_key> — barrier state variable (e.g., "safety:current_cash")
-- KEYS[2]: audit:state_ledger
-- KEYS[3]: safety:fence_epoch (R-05)
-- ARGV[1]: magnitude (float string) — deduction amount (domain-neutral; was "cost")
-- ARGV[2]: threshold (float string) — <InvariantModel.threshold_key> resolved floor
-- ARGV[3]: gamma (float string) — <InvariantModel.gamma>
-- ARGV[4]: governance_signature (string, may be empty)
-- ARGV[5]: ground_truth_balance (float string) -- POAM-023: KMS-verified balance from Python
-- Returns: array {status_code, message, new_balance_str, new_epoch}
-- status_code 1 = COMMITTED, 0 = UNSAFE (envelope violation)
-- POAM-023: Ground truth balance passed from Python after KMS verification
local current = tonumber(ARGV[5])
if not current then
return {0, "Ground truth balance unavailable", "0", 0}
end
local cost = tonumber(ARGV[1]) or 0.0
local min_cash = tonumber(ARGV[2])
local gamma = tonumber(ARGV[3])
local sig = ARGV[4]
local next_cash = current - cost
local h_t = current - min_cash
local h_next = next_cash - min_cash
local required_h_next = (1.0 - gamma) * h_t
-- Read current epoch for return (even on UNSAFE)
local current_epoch_raw = redis.call('GET', KEYS[3])
local current_epoch = current_epoch_raw and tonumber(current_epoch_raw) or 0
if h_next < required_h_next or h_next < 0 then
return {0, "UNSAFE: h_next=" .. tostring(h_next) .. " < required=" .. tostring(required_h_next), tostring(current), current_epoch}
end
redis.call('SET', KEYS[1], tostring(next_cash))
-- R-05: Increment fence epoch on every mutating write
local new_epoch = redis.call('INCR', KEYS[3])
if sig ~= "" then
redis.call('RPUSH', KEYS[2], sig .. ":" .. tostring(next_cash))
end
return {1, "COMMITTED", tostring(next_cash), new_epoch}
"""
def __init__(
self,
invariant: "InvariantModel | None" = None,
cost_resolver: Any = None,
skip_epoch_seed: bool = False,
): # type: ignore[no-untyped-def]
"""Initialize the ControlBarrierFunction.
PR C (Stage 2): One engine instance enforces exactly one affine barrier.
Multi-barrier domains construct multiple engines and register multiple
CBF tiers. A single engine multiplexing barriers would require a multi-key
Lua script, which is a distinct proof obligation (cross-key atomicity) not
covered by DistributedCBF.tla.
Args:
invariant: InvariantModel instance defining the barrier (state_key,
threshold_key, gamma). If None, defaults to kernel
fallback barrier with legacy cost resolver.
cost_resolver: Callable[[str, dict], float] that computes the cost
for a given (action_name, payload). Domain plugins must
inject their resolver at registration. Default is zero
cost for all actions (domain-agnostic kernel).
skip_epoch_seed: If True, skip Redis epoch seeding at init time.
Used only for testing; production instances must
seed from Redis.
Raises:
CBFInitializationError: If Redis is unavailable and skip_epoch_seed
is False. This prevents the instance from
accepting requests with an unseeded epoch.
"""
# W1 (Post-v3): Mandatory value object — single source of truth.
# Domain plugins provide both invariant and cost_resolver at registration.
if invariant is None:
invariant = _DefaultKernelBarrier()
if cost_resolver is None:
cost_resolver = self._legacy_finance_cost_resolver
self._invariant = invariant
self._cost_resolver = cost_resolver or self._default_cost_resolver
self._gamma_override: float | None = None
# Backward-compatibility attributes (deprecated; use _invariant)
self.min_cash_balance: float = THRESHOLDS.cbf.min_cash_balance
self.tracer = get_tracer("src.gateway.governance.safety")
self._lua_sha: str | None = None
# Reviewer note H53: local intra-window debits subtracted from snapshot to prevent double-spend within TTL window.
self._local_debits: float = 0.0
# R-05 fence epoch: track last seen epoch to detect regression after failover
# B3a: Seed from Redis on startup — fail-closed if unavailable
self._last_seen_epoch: int = self._fetch_initial_fence_epoch_sync(
skip_epoch_seed
)
# POAM-023: Track last verified fence epoch to detect regression on commit path
self._last_verified_fence_epoch: int | None = None
@property
def threshold_key(self) -> str:
"""Threshold key derived from invariant model."""
return self._invariant.threshold_key
@property
def gamma(self) -> float:
"""CBF gamma derived from invariant model."""
if self._gamma_override is not None:
return self._gamma_override
return self._invariant.gamma
@gamma.setter
def gamma(self, value: float) -> None:
"""Allow test harness and configuration override of gamma.
Warning: This setter is NOT thread-safe and is intended for test
harnesses and single-threaded configuration only. Do not call
during concurrent request processing in production.
"""
cage_env = os.getenv("CAGE_ENV", "dev").lower()
if cage_env in ("production", "prod"):
logger.warning(
"gamma override in production is unsafe for concurrent requests; "
"use static configuration (InvariantModel.gamma) instead"
)
self._gamma_override = value
@property
def redis_key(self) -> str:
"""State key derived from invariant model."""
return self._invariant.state_key
def _fetch_initial_fence_epoch_sync(self, skip_epoch_seed: bool) -> int:
"""Fetch the current fence epoch from Redis at construction time.
§B3a: A newly spawned gateway instance must have an external anchor
for its fence epoch. Without this, a fresh instance would accept
whatever epoch it first observes as baseline, creating a window
for stale-read attacks after a Redis failover.
This method uses the synchronous Redis client because __init__ is
synchronous. The sync client is safe to call from module-load time.
Args:
skip_epoch_seed: If True, return 0 without contacting Redis.
Used for testing only.
Returns:
The current fence epoch from Redis, or 0 if this is the first-ever
startup (no epoch key exists — we initialize it to 0 and write it).
Raises:
CBFInitializationError: If Redis is unavailable and skip_epoch_seed
is False.
"""
if skip_epoch_seed:
logger.debug("B3a: Skipping epoch seed (skip_epoch_seed=True)")
return 0
if sync_redis_client is None:
# Behavior depends on environment:
# - Production: fail-closed (raise CBFInitializationError)
# - Dev/test: warn but proceed with epoch=0 (backward compatibility)
if _IS_PRODUCTION:
raise CBFInitializationError(
"Cannot initialize CBF: sync Redis client unavailable. "
"Fence epoch cannot be seeded from external anchor. "
"Failing closed to prevent stale-read attack window."
)
else:
logger.warning(
"B3a: sync Redis client unavailable in dev/test mode — "
"proceeding with epoch=0. Set CAGE_ENV=prod to enforce "
"fail-closed behavior."
)
return 0
try:
epoch_raw = sync_redis_client.get(_REDIS_KEY_FENCE_EPOCH)
if epoch_raw is None:
# First-ever startup — initialize epoch to 0 and write it.
# This is the only case where epoch=0 is acceptable.
sync_redis_client._get().set(_REDIS_KEY_FENCE_EPOCH, "0")
logger.info(
"B3a: First-ever startup — initialized fence epoch to 0 in Redis"
)
return 0
epoch = int(epoch_raw)
logger.info("B3a: Seeded fence epoch from Redis: %d", epoch)
if _CURRENT_FENCE_EPOCH_GAUGE is not None:
_CURRENT_FENCE_EPOCH_GAUGE.set(epoch)
return epoch
except Exception as exc:
# Behavior depends on environment:
# - Production: fail-closed (raise CBFInitializationError)
# - Dev/test: warn but proceed with epoch=0 (backward compatibility)
if _IS_PRODUCTION:
raise CBFInitializationError(
f"Cannot initialize CBF: fence epoch unavailable from Redis. "
f"Error: {exc}. Failing closed to prevent stale-read attack window."
) from exc
else:
logger.warning(
"B3a: Redis unavailable in dev/test mode — proceeding with "
"epoch=0. Set CAGE_ENV=prod to enforce fail-closed behavior. "
"Error: %s",
exc,
)
return 0
async def setup(self) -> None:
"""Bootstrap Redis state if the key is absent (first run)."""
if redis_client is None:
logger.error("Redis client unavailable — cannot bootstrap CBF state.")
return
if await redis_client.get(self.redis_key) is None:
await redis_client.set(self.redis_key, "100000.0")
# Initialize fence epoch if absent (R-05)
client = await _get_raw_redis(redis_client)
if client is not None:
epoch_raw = await client.get(_REDIS_KEY_FENCE_EPOCH)
if epoch_raw is None:
await client.set(_REDIS_KEY_FENCE_EPOCH, "0")
logger.info("R-05: Initialized fence epoch to 0")
async def _get_current_cash(self) -> float:
if redis_client is None:
raise RuntimeError("Redis client unavailable.")
return await redis_client.get_float(self.redis_key, 100000.0)
# ------------------------------------------------------------------
# R-05 Fence Epoch: Double-spend detection across Redis failover
# ------------------------------------------------------------------
async def _increment_fence_epoch(self, pipeline: Any) -> int:
"""Increment the fence epoch atomically within the pipeline.
§2.6 R-05 mitigation: The fence epoch is a monotonically increasing
counter that increments on every CBF-mutating write. After a Redis
failover, if a replica hasn't replicated the latest epoch, reads
from that replica will return a regressed epoch, which we detect
and reject (fail-closed).
Args:
pipeline: Redis pipeline object to queue the INCR command.
Returns:
The new epoch value after increment.
Note:
The INCR command is atomic and creates the key with value 1 if
it doesn't exist. The epoch is never TTL'd.
"""
# Queue INCR in the pipeline — returns new value after increment
pipeline.incr(_REDIS_KEY_FENCE_EPOCH)
# The actual value is returned when pipeline.execute() is called
# Caller must extract from execute() results
return 0 # Placeholder; actual value comes from pipeline results
async def _get_fence_epoch(self) -> int:
"""Read the current fence epoch from Redis.
Returns:
The current epoch value, or 0 if the key doesn't exist.
"""
if redis_client is None:
raise RuntimeError("Redis client unavailable.")
client = await _get_raw_redis(redis_client)
raw = await client.get(_REDIS_KEY_FENCE_EPOCH)
if raw is None:
return 0
return int(raw)
async def _check_fence_epoch(self, current_epoch: int) -> tuple[bool, str]:
"""Validate that current fence epoch hasn't regressed.
§2.6 R-05 mitigation: After a Redis primary-to-replica failover,
the replica may not have replicated the latest fence epoch. If the
epoch we read is less than the last epoch we saw, we've likely
switched to a stale replica. This is a double-spend vulnerability.
Args:
current_epoch: The epoch value just read from Redis.
Returns:
(True, "OK") if epoch is valid (>= last seen).
(False, reason) if epoch has regressed (< last seen).
Side effects:
- On regression: logs CRITICAL, increments Prometheus counter
- On valid: updates _last_seen_epoch, updates Prometheus gauge
"""
if current_epoch < self._last_seen_epoch:
reason = (
f"epoch={current_epoch} < last_seen={self._last_seen_epoch} "
"(possible failover to stale replica)"
)
logger.critical(
json.dumps(
{
"event": "CBF_EPOCH_REGRESSION_DETECTED",
"severity": "CRITICAL",
"current_epoch": current_epoch,
"last_seen_epoch": self._last_seen_epoch,
"audit_note": (
"R-05: Fence epoch regression detected. This indicates "
"a possible failover to a Redis replica that hasn't "
"replicated the latest writes. Rejecting read to prevent "
"double-spend vulnerability. Fail-closed."
),
}
)
)
if _EPOCH_REGRESSION_COUNTER is not None:
_EPOCH_REGRESSION_COUNTER.inc()
return (False, reason)
# Epoch is valid — update tracking state
self._last_seen_epoch = current_epoch
if _CURRENT_FENCE_EPOCH_GAUGE is not None:
_CURRENT_FENCE_EPOCH_GAUGE.set(current_epoch)
return (True, "OK")
# ------------------------------------------------------------------
# Phase 4.3: WAIT command for synchronous replication
# ------------------------------------------------------------------
async def _sync_to_replicas(
self,
num_replicas: int | None = None,
timeout_ms: int | None = None,
) -> bool:
"""Block until fence epoch is replicated to at least num_replicas.
Phase 4.3: Uses Redis WAIT command to ensure the fence epoch increment
(and any preceding writes) is replicated to the specified number of
replicas before returning. This provides stronger durability guarantees
for deployments using Redis replication.
See: https://redis.io/commands/wait/
Args:
num_replicas: Number of replicas to wait for. Defaults to
CAGE_REDIS_WAIT_REPLICAS env var (default 0 = disabled).
timeout_ms: Timeout in milliseconds to wait for replication.
Defaults to CAGE_REDIS_WAIT_TIMEOUT_MS env var (default 1000).
Returns:
True if replication confirmed to num_replicas within timeout.
False if timeout elapsed before replication confirmed.
True (no-op) if num_replicas == 0 (WAIT disabled).
Note:
- WAIT returns the number of replicas that acknowledged the write.
- A return value < num_replicas means some replicas are lagging.
- Timeout is not an error condition for WAIT; it simply means we
waited the full duration without reaching the replica count.
"""
# Use provided values or fall back to module-level config
replicas = num_replicas if num_replicas is not None else _WAIT_REPLICAS
timeout = timeout_ms if timeout_ms is not None else _WAIT_TIMEOUT_MS
# No-op if WAIT is disabled (replicas=0 is the default)
if replicas <= 0:
return True
if redis_client is None:
logger.warning("Redis client unavailable — cannot execute WAIT command.")
return False
raw_client_getter = getattr(redis_client, "get_raw_client", None)
if callable(raw_client_getter):
client = raw_client_getter()
if inspect.isawaitable(client):
client = await client
else:
client = redis_client
start_time = time.time()
try:
# WAIT numreplicas timeout
# Returns: number of replicas that acknowledged the write
cmd = client.execute_command("WAIT", replicas, timeout)
if inspect.isawaitable(cmd):
acks = await cmd
else:
acks = int(cmd) if cmd is not None else 0
elapsed = time.time() - start_time
# Record latency in Prometheus histogram
if _WAIT_LATENCY_HISTOGRAM is not None:
_WAIT_LATENCY_HISTOGRAM.observe(elapsed)
if acks >= replicas:
logger.debug(
"Phase 4.3: WAIT confirmed replication to %d/%d replicas in %.3fs",
acks,
replicas,
elapsed,
)
return True
else:
# Timeout elapsed before reaching replica count
logger.warning(
json.dumps(
{
"event": "CBF_WAIT_TIMEOUT",
"severity": "WARNING",
"requested_replicas": replicas,
"acknowledged_replicas": acks,
"timeout_ms": timeout,
"elapsed_seconds": round(elapsed, 3),
"audit_note": (
"Phase 4.3: Redis WAIT timed out before reaching "
f"requested replica count. {acks}/{replicas} replicas "
"acknowledged. Proceeding with degraded replication."
),
}
)
)
if _WAIT_TIMEOUT_COUNTER is not None:
_WAIT_TIMEOUT_COUNTER.inc()
return False
except Exception as exc:
elapsed = time.time() - start_time
logger.error(
"Phase 4.3: WAIT command failed after %.3fs: %s",
elapsed,
exc,
)
# Record the latency even on error
if _WAIT_LATENCY_HISTOGRAM is not None:
_WAIT_LATENCY_HISTOGRAM.observe(elapsed)
return False
async def _validate_sequence(
self, incoming_sequence: int, source: str, sync_redis: Any
) -> tuple[bool, str]:
"""Validate incoming sequence is strictly greater than last accepted.
§2.10 R-04 Replay defense: monotonic sequence number validation.
Prevents replay of stale balance data by rejecting payloads with
non-advancing sequence numbers.
Args:
incoming_sequence: The sequence number in the incoming payload.
source: Provider source name (for logging).
sync_redis: Synchronous Redis client for reading/writing sequence.
Returns:
(True, "OK") if sequence is advancing.
(False, reason) if sequence is non-advancing (replay detected).
"""
try:
# Read last accepted sequence
last_accepted_raw = await asyncio.to_thread(
sync_redis.get,
_REDIS_KEY_SEQUENCE_LAST_ACCEPTED, # type: ignore[attr-defined]
)
last_accepted = int(last_accepted_raw) if last_accepted_raw else 0
if incoming_sequence <= last_accepted:
reason = (
f"sequence={incoming_sequence} <= last_accepted={last_accepted}"
)
return (False, reason)
# Update last_accepted atomically
await asyncio.to_thread(
sync_redis.set,
_REDIS_KEY_SEQUENCE_LAST_ACCEPTED,
str(incoming_sequence), # type: ignore[attr-defined]
)
logger.debug(
"[R-04] Sequence validated: incoming=%d > last_accepted=%d, updated",
incoming_sequence,
last_accepted,
)
return (True, "OK")
except Exception as exc:
# On error, fail-open to allow balance through (conservative)
# but log a warning so the issue is visible
logger.warning(
"[R-04] Sequence validation error: %s — allowing payload (fail-open)",
exc,
)
return (True, f"validation error (fail-open): {exc}")
async def _read_cbf_state_atomic(self) -> dict[str, float | str]:
"""Read the CBF cash balance, preferring externally reconciled ground truth.
Priority order (POAM-023):
1. ``reconciliation:verified_balance`` — written by the isolated
reconciliation-worker daemon, KMS-signed, TTL-gated. When present
and signature-valid, this is the authoritative balance.
2. ``safety:current_cash`` — self-reported by the execution system.
Used only when the reconciled balance is absent or invalid.
A CRITICAL audit log is emitted so the fallback is always visible
in Langfuse and SIEM.
Returns:
dict with keys:
``current_cash`` (float) — the balance to use in the CBF formula
``source`` (str) — ``"reconciled"`` | ``"reconciled_unsigned"``
| ``"self_reported"``
"""
if redis_client is None:
raise RuntimeError("Redis client unavailable.")
# ── Attempt 1: externally reconciled balance (POAM-023) ──────────────
try:
# LOW-6 fix: removed inline `import asyncio as _asyncio` — asyncio is
# already imported at module level.
from src.gateway.governance.reconciliation.daemon import (
read_verified_balance,
)
# read_verified_balance is synchronous (redis-py sync client).
# Use sync_redis_client (blocking redis.Redis) — NOT redis_client._get()
# which returns an aioredis.Redis (async) client whose .get() returns a
# coroutine instead of a value, causing the JSON parse to fail with:
# "the JSON object must be str, bytes or bytearray, not coroutine"
# (CBF_USING_SELF_REPORTED_BALANCE log sentinel — POAM-023 async bug).
from src.gateway.infrastructure.redis_client import sync_redis_client
verified = await asyncio.to_thread(read_verified_balance, sync_redis_client)
if verified is not None and verified.is_valid:
if verified.signature:
# Verify KMS signature before trusting the balance.
try:
from src.gateway.governance.kms_signer import (
get_governance_signer,
)
signer = get_governance_signer()
payload_dict = {
"source": verified.source,
"balance_usd": verified.balance_usd,
"verified_at": verified.verified_at,
"sequence": verified.sequence, # §2.10: in signed payload
}
sig_valid = signer.verify(payload_dict, verified.signature)
if sig_valid:
# ── §2.10 R-04 Replay defense: sequence validation ────
if _REPLAY_DEFENSE_ENABLED and verified.sequence > 0:
(
sequence_valid,
seq_reason,
) = await self._validate_sequence(
verified.sequence,
verified.source,
sync_redis_client,
)
if not sequence_valid:
# Replay detected — fall through to self-reported
logger.critical(
json.dumps(
{
"event": "CBF_RECONCILED_BALANCE_SEQUENCE_REPLAY_DETECTED",
"severity": "CRITICAL",
"source": verified.source,
"balance_usd": verified.balance_usd,
"sequence": verified.sequence,
"reason": seq_reason,
"audit_note": (
"R-04 Replay defense: monotonic sequence "
"validation FAILED. Payload sequence is "
"non-advancing. Falling back to self-reported "
"balance. Possible TTL reset attack or stale replay."
),
}
)
)
# Increment Prometheus counter for replay rejection
if _REPLAY_REJECTED_COUNTER is not None:
_REPLAY_REJECTED_COUNTER.labels(
source=verified.source
).inc()
# Fall through to self-reported balance below
else:
# Sequence valid — update last_accepted and proceed
logger.info(
"CBF: using externally reconciled balance=%.2f "
"source=%s verified_at=%.0f sequence=%d (KMS signature valid, sequence advancing)",
verified.balance_usd,
verified.source,
verified.verified_at,
verified.sequence,
)
return {
"current_cash": verified.balance_usd,
"source": "reconciled",
"sequence": verified.sequence,
}
else:
# Replay defense disabled or sequence=0 (backward compat)
logger.info(
"CBF: using externally reconciled balance=%.2f "
"source=%s verified_at=%.0f sequence=%d (KMS signature valid)",
verified.balance_usd,
verified.source,
verified.verified_at,
verified.sequence,
)
return {
"current_cash": verified.balance_usd,
"source": "reconciled",
"sequence": verified.sequence,
}
else:
logger.critical(
json.dumps(
{
"event": "CBF_RECONCILED_BALANCE_SIGNATURE_INVALID",
"severity": "CRITICAL",
"source": verified.source,
"balance_usd": verified.balance_usd,
"audit_note": (
"KMS signature on reconciled balance is INVALID. "
"Falling back to self-reported balance. "
"POAM-023: CBF ground truth unverified."
),
}
)
)
except Exception as sig_exc:
logger.critical(
json.dumps(
{
"event": "CBF_KMS_VERIFY_FAILED",
"severity": "CRITICAL",
"error": str(sig_exc),
"strict_mode": _CBF_STRICT_MODE,
"audit_note": (
"KMS signature verification raised an exception. "
+ (
"FAIL-CLOSED: Transaction rejected (strict mode). "
if _CBF_STRICT_MODE
else "Falling back to self-reported balance. "
)
+ "POAM-023: CBF ground truth unverified."
),
}
)
)
# SECURITY FIX: Fail-closed in strict mode to prevent attackers
# from exhausting KMS quota to force unverified balance usage
if _CBF_STRICT_MODE:
logger.error(
"[SECURITY] CBF ground truth unavailable — rejecting transaction "
"(strict mode enabled). error_type=%s error=%s source=%s",
type(sig_exc).__name__,
str(sig_exc)[:100],
verified.source if verified else "unknown",
)
raise GroundTruthUnavailableError(
f"Cannot verify balance — transaction rejected. "
f"KMS verification failed: {type(sig_exc).__name__}",
cause=sig_exc,
) from sig_exc
# Non-strict mode: fall through to self-reported balance (legacy behavior)
logger.warning(
"[DIAG-FAILOPEN] cbf_kms_verify_failed decision=continue_with_fallback "
"error_type=%s error=%s source=%s balance_usd=%.2f "
"poam_ref=POAM-023 security_impact=HIGH strict_mode=false",
type(sig_exc).__name__,
str(sig_exc)[:100],
verified.source if verified else "unknown",
verified.balance_usd if verified else 0.0,
)
else:
# Unsigned reconciled balance — accept only in dev/test.
if _IS_PRODUCTION:
logger.critical(
json.dumps(
{
"event": "CBF_RECONCILED_BALANCE_UNSIGNED_IN_PRODUCTION",
"severity": "CRITICAL",
"source": verified.source,
"balance_usd": verified.balance_usd,
"audit_note": (
"Reconciled balance has no KMS signature in production. "
"Falling back to self-reported balance. "
"POAM-023: CBF ground truth unverified."
),
}
)