diff --git a/codex-rs/ext/guardian-reviewer/src/completion.rs b/codex-rs/ext/guardian-reviewer/src/completion.rs index 72befe0bd1d0..ee037c25f0b1 100644 --- a/codex-rs/ext/guardian-reviewer/src/completion.rs +++ b/codex-rs/ext/guardian-reviewer/src/completion.rs @@ -82,21 +82,22 @@ pub fn complete_review( } GuardianReviewOutcome::Error(error) => { analytics.failure_reason = Some(error.failure_reason()); - if matches!(error, GuardianReviewError::InputBudgetExceeded) && !require_guardian { - let rationale = format!("Automatic approval review failed: {INPUT_BUDGET_MESSAGE}"); - analytics.decision = GuardianReviewDecision::Aborted; - analytics.terminal_status = GuardianReviewTerminalStatus::Aborted; - event.status = GuardianAssessmentStatus::Aborted; - event.rationale = Some(rationale.clone()); - return ReviewCompletion { - decision: None, - event, - warning: Some(rationale), - analytics, - assessment_outcome: None, - }; - } match error { + GuardianReviewError::InputBudgetExceeded if !require_guardian => { + let rationale = + format!("Automatic approval review failed: {INPUT_BUDGET_MESSAGE}"); + analytics.decision = GuardianReviewDecision::Aborted; + analytics.terminal_status = GuardianReviewTerminalStatus::Aborted; + event.status = GuardianAssessmentStatus::Aborted; + event.rationale = Some(rationale.clone()); + return ReviewCompletion { + decision: None, + event, + warning: Some(rationale), + analytics, + assessment_outcome: None, + }; + } GuardianReviewError::Timeout => { let rationale = "Automatic approval review timed out while evaluating the requested approval.".to_string(); analytics.decision = GuardianReviewDecision::Denied; diff --git a/codex-rs/ext/guardian-reviewer/src/retry.rs b/codex-rs/ext/guardian-reviewer/src/retry.rs index 2dbb61a45b9c..dd98638fe52c 100644 --- a/codex-rs/ext/guardian-reviewer/src/retry.rs +++ b/codex-rs/ext/guardian-reviewer/src/retry.rs @@ -56,8 +56,15 @@ where } _ => None, }; - if let Some(error) = - wait_before_guardian_retry(attempt_count, retry_at, deadline, external_cancel).await + if let Some(error) = wait_before_guardian_retry( + attempt_count, + max_attempts, + &outcome, + retry_at, + deadline, + external_cancel, + ) + .await { return (GuardianReviewOutcome::Error(error), analytics_result, None); } @@ -67,6 +74,8 @@ where async fn wait_before_guardian_retry( attempt_count: i64, + max_attempts: i64, + outcome: &GuardianReviewOutcome, retry_not_before: Option, deadline: Instant, external_cancel: Option<&CancellationToken>, @@ -74,9 +83,34 @@ async fn wait_before_guardian_retry( let exponential_delay = 200.0 * 2.0_f64.powi(attempt_count.saturating_sub(1) as i32); let jitter = rand::rng().random_range(0.9..1.1); let retry_delay = Duration::from_millis((exponential_delay * jitter) as u64); - let retry_at = (Instant::now() + retry_delay) - .max(retry_not_before.unwrap_or_else(Instant::now)) - .min(deadline); + let now = Instant::now(); + let requested_retry_at = (now + retry_delay).max(retry_not_before.unwrap_or(now)); + let retry_at = requested_retry_at.min(deadline); + let (failure_reason, error_info) = match outcome { + GuardianReviewOutcome::Error(error) => ( + Some(error.failure_reason()), + if let GuardianReviewError::Session { error_info, .. } = error { + error_info.as_ref() + } else { + None + }, + ), + GuardianReviewOutcome::CachedApproval | GuardianReviewOutcome::Completed(_) => (None, None), + }; + tracing::debug!( + attempt_count, + next_attempt = attempt_count + 1, + max_attempts, + ?failure_reason, + ?error_info, + wait_ms = retry_at.saturating_duration_since(now).as_millis() as u64, + backoff_ms = retry_delay.as_millis() as u64, + retry_not_before_ms = retry_not_before + .map(|instant| instant.saturating_duration_since(now).as_millis() as u64), + remaining_budget_ms = deadline.saturating_duration_since(now).as_millis() as u64, + deadline_limited = requested_retry_at >= deadline, + "Guardian review retry scheduled" + ); tokio::select! { _ = sleep_until(retry_at) => { (Instant::now() >= deadline).then_some(GuardianReviewError::Timeout) diff --git a/codex-rs/ext/guardian-reviewer/src/retry_tests.rs b/codex-rs/ext/guardian-reviewer/src/retry_tests.rs index 4f998b878212..3ee4d74922d2 100644 --- a/codex-rs/ext/guardian-reviewer/src/retry_tests.rs +++ b/codex-rs/ext/guardian-reviewer/src/retry_tests.rs @@ -137,6 +137,8 @@ async fn guardian_review_retry_wait_honors_cancellation() { let error = wait_before_guardian_retry( /*attempt_count*/ 1, + /*max_attempts*/ 3, + &GuardianReviewOutcome::Error(GuardianReviewError::StaleAuthorization), /*retry_not_before*/ None, Instant::now() + Duration::from_secs(/*secs*/ 1), Some(&cancel_token), @@ -150,6 +152,8 @@ async fn guardian_review_retry_wait_honors_cancellation() { async fn guardian_review_retry_wait_honors_deadline() { let error = wait_before_guardian_retry( /*attempt_count*/ 1, + /*max_attempts*/ 3, + &GuardianReviewOutcome::Error(GuardianReviewError::StaleAuthorization), /*retry_not_before*/ None, Instant::now(), /*external_cancel*/ None, diff --git a/codex-rs/ext/guardian-reviewer/src/review.rs b/codex-rs/ext/guardian-reviewer/src/review.rs index e8bcd4bb2ae4..10e3e65fd026 100644 --- a/codex-rs/ext/guardian-reviewer/src/review.rs +++ b/codex-rs/ext/guardian-reviewer/src/review.rs @@ -78,6 +78,16 @@ pub trait ReviewHost: Send + Sync { } impl SynchronousApprovalReviewer for ReviewRequest<'_, H> { + #[tracing::instrument( + name = "guardian_review", + level = "debug", + skip_all, + fields( + review_id = self.approval_id, + thread_id = %self.thread_id, + tool_call_id = self.tool_call_id, + ) + )] fn review<'a>( &'a self, reason: GuardianReviewReason, diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/classification.rs b/codex-rs/ext/guardian-v2/src/async_scorer/classification.rs index 55a4bab5ca00..08d68d4ff0f3 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/classification.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/classification.rs @@ -106,6 +106,19 @@ enum ClassificationOutcome { } impl Classification { + #[tracing::instrument( + name = "guardian_background_scoring", + level = "debug", + skip_all, + fields( + thread_id = %self.thread_id, + turn_id = %self.turn_id, + call_id = %self.call_id, + tool_call_index = self.tool_call_index, + context_mode = self.context_mode.as_str(), + retained_conversation = self.reservation.is_some(), + ) + )] pub(super) async fn run(self) { let Self { reservation, diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs index 25b50494c751..de96c9ed9b1e 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs @@ -1,4 +1,4 @@ -//! Low-cardinality metrics for Guardian V2 classification, connections, and approval decisions. +//! Structured failure diagnostics and low-cardinality metrics for Guardian V2 scoring and decisions. use std::time::Duration; @@ -69,6 +69,14 @@ pub(super) fn record_classification( outcome: &str, failure_reason: Option<&str>, ) { + if outcome == "failure" { + tracing::warn!( + failure_reason, + context_mode = context_mode.as_str(), + duration_ms = u64::try_from(duration.as_millis()).unwrap_or(u64::MAX), + "Guardian V2 background scoring failed" + ); + } let Some(metrics) = metrics else { return; }; diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/observation.rs b/codex-rs/ext/guardian-v2/src/async_scorer/observation.rs index 65eafd52009b..e1eaf45d1408 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/observation.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/observation.rs @@ -37,6 +37,18 @@ use super::score::GuardianV2ScoreProgress; use codex_protocol::openai_models::GuardianUnscoredAction as UnscoredAction; impl GuardianV2Extension { + #[tracing::instrument( + name = "guardian_scoring_observation", + level = "debug", + skip_all, + fields( + thread_id = input.thread_store.level_id(), + turn_id = input.turn_id, + call_id = input.call_id, + tool_name = %input.tool_name, + tool_call_index = tracing::field::Empty, + ) + )] pub(super) async fn score_tool(&self, input: ToolStartInput<'_>) { // Polling a code cell does not introduce another action or age its score. if input.tool_name.is_default_namespace() && input.tool_name.name == "wait" { @@ -129,6 +141,7 @@ impl GuardianV2Extension { } None => (score_progress.observe(&input), Ok(None)), }; + tracing::Span::current().record("tool_call_index", tool_call_index); let reservation = match reservation { Ok(reservation) => reservation, Err(error) => {