Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 15 additions & 14 deletions codex-rs/ext/guardian-reviewer/src/completion.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,21 +82,22 @@ pub fn complete_review(
}
GuardianReviewOutcome::Error(error) => {
analytics.failure_reason = Some(error.failure_reason());
if matches!(error, GuardianReviewError::InputBudgetExceeded) && !require_guardian {
let rationale = format!("Automatic approval review failed: {INPUT_BUDGET_MESSAGE}");
analytics.decision = GuardianReviewDecision::Aborted;
analytics.terminal_status = GuardianReviewTerminalStatus::Aborted;
event.status = GuardianAssessmentStatus::Aborted;
event.rationale = Some(rationale.clone());
return ReviewCompletion {
decision: None,
event,
warning: Some(rationale),
analytics,
assessment_outcome: None,
};
}
match error {
GuardianReviewError::InputBudgetExceeded if !require_guardian => {
let rationale =
format!("Automatic approval review failed: {INPUT_BUDGET_MESSAGE}");
analytics.decision = GuardianReviewDecision::Aborted;
analytics.terminal_status = GuardianReviewTerminalStatus::Aborted;
event.status = GuardianAssessmentStatus::Aborted;
event.rationale = Some(rationale.clone());
return ReviewCompletion {
decision: None,
event,
warning: Some(rationale),
analytics,
assessment_outcome: None,
};
}
GuardianReviewError::Timeout => {
let rationale = "Automatic approval review timed out while evaluating the requested approval.".to_string();
analytics.decision = GuardianReviewDecision::Denied;
Expand Down
44 changes: 39 additions & 5 deletions codex-rs/ext/guardian-reviewer/src/retry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,15 @@ where
}
_ => None,
};
if let Some(error) =
wait_before_guardian_retry(attempt_count, retry_at, deadline, external_cancel).await
if let Some(error) = wait_before_guardian_retry(
attempt_count,
max_attempts,
&outcome,
retry_at,
deadline,
external_cancel,
)
.await
{
return (GuardianReviewOutcome::Error(error), analytics_result, None);
}
Expand All @@ -67,16 +74,43 @@ where

async fn wait_before_guardian_retry(
attempt_count: i64,
max_attempts: i64,
outcome: &GuardianReviewOutcome,
retry_not_before: Option<Instant>,
deadline: Instant,
external_cancel: Option<&CancellationToken>,
) -> Option<GuardianReviewError> {
let exponential_delay = 200.0 * 2.0_f64.powi(attempt_count.saturating_sub(1) as i32);
let jitter = rand::rng().random_range(0.9..1.1);
let retry_delay = Duration::from_millis((exponential_delay * jitter) as u64);
let retry_at = (Instant::now() + retry_delay)
.max(retry_not_before.unwrap_or_else(Instant::now))
.min(deadline);
let now = Instant::now();
let requested_retry_at = (now + retry_delay).max(retry_not_before.unwrap_or(now));
let retry_at = requested_retry_at.min(deadline);
let (failure_reason, error_info) = match outcome {
GuardianReviewOutcome::Error(error) => (
Some(error.failure_reason()),
if let GuardianReviewError::Session { error_info, .. } = error {
error_info.as_ref()
} else {
None
},
),
GuardianReviewOutcome::CachedApproval | GuardianReviewOutcome::Completed(_) => (None, None),
};
tracing::debug!(
attempt_count,
next_attempt = attempt_count + 1,
max_attempts,
?failure_reason,
?error_info,
wait_ms = retry_at.saturating_duration_since(now).as_millis() as u64,
backoff_ms = retry_delay.as_millis() as u64,
retry_not_before_ms = retry_not_before
.map(|instant| instant.saturating_duration_since(now).as_millis() as u64),
remaining_budget_ms = deadline.saturating_duration_since(now).as_millis() as u64,
deadline_limited = requested_retry_at >= deadline,
"Guardian review retry scheduled"
);
tokio::select! {
_ = sleep_until(retry_at) => {
(Instant::now() >= deadline).then_some(GuardianReviewError::Timeout)
Expand Down
4 changes: 4 additions & 0 deletions codex-rs/ext/guardian-reviewer/src/retry_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,8 @@ async fn guardian_review_retry_wait_honors_cancellation() {

let error = wait_before_guardian_retry(
/*attempt_count*/ 1,
/*max_attempts*/ 3,
&GuardianReviewOutcome::Error(GuardianReviewError::StaleAuthorization),
/*retry_not_before*/ None,
Instant::now() + Duration::from_secs(/*secs*/ 1),
Some(&cancel_token),
Expand All @@ -150,6 +152,8 @@ async fn guardian_review_retry_wait_honors_cancellation() {
async fn guardian_review_retry_wait_honors_deadline() {
let error = wait_before_guardian_retry(
/*attempt_count*/ 1,
/*max_attempts*/ 3,
&GuardianReviewOutcome::Error(GuardianReviewError::StaleAuthorization),
/*retry_not_before*/ None,
Instant::now(),
/*external_cancel*/ None,
Expand Down
10 changes: 10 additions & 0 deletions codex-rs/ext/guardian-reviewer/src/review.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,16 @@ pub trait ReviewHost: Send + Sync {
}

impl<H: ReviewHost> SynchronousApprovalReviewer for ReviewRequest<'_, H> {
#[tracing::instrument(
name = "guardian_review",
level = "debug",
skip_all,
fields(
review_id = self.approval_id,
thread_id = %self.thread_id,
tool_call_id = self.tool_call_id,
)
)]
fn review<'a>(
&'a self,
reason: GuardianReviewReason,
Expand Down
13 changes: 13 additions & 0 deletions codex-rs/ext/guardian-v2/src/async_scorer/classification.rs
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,19 @@ enum ClassificationOutcome {
}

impl Classification {
#[tracing::instrument(
name = "guardian_background_scoring",
level = "debug",
skip_all,
fields(
thread_id = %self.thread_id,
turn_id = %self.turn_id,
call_id = %self.call_id,
tool_call_index = self.tool_call_index,
context_mode = self.context_mode.as_str(),
retained_conversation = self.reservation.is_some(),
)
)]
pub(super) async fn run(self) {
let Self {
reservation,
Expand Down
10 changes: 9 additions & 1 deletion codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! Low-cardinality metrics for Guardian V2 classification, connections, and approval decisions.
//! Structured failure diagnostics and low-cardinality metrics for Guardian V2 scoring and decisions.

use std::time::Duration;

Expand Down Expand Up @@ -69,6 +69,14 @@ pub(super) fn record_classification(
outcome: &str,
failure_reason: Option<&str>,
) {
if outcome == "failure" {
tracing::warn!(
failure_reason,
context_mode = context_mode.as_str(),
duration_ms = u64::try_from(duration.as_millis()).unwrap_or(u64::MAX),
"Guardian V2 background scoring failed"
);
}
let Some(metrics) = metrics else {
return;
};
Expand Down
13 changes: 13 additions & 0 deletions codex-rs/ext/guardian-v2/src/async_scorer/observation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,18 @@ use super::score::GuardianV2ScoreProgress;
use codex_protocol::openai_models::GuardianUnscoredAction as UnscoredAction;

impl GuardianV2Extension {
#[tracing::instrument(
name = "guardian_scoring_observation",
level = "debug",
skip_all,
fields(
thread_id = input.thread_store.level_id(),
turn_id = input.turn_id,
call_id = input.call_id,
tool_name = %input.tool_name,
tool_call_index = tracing::field::Empty,
)
)]
pub(super) async fn score_tool(&self, input: ToolStartInput<'_>) {
// Polling a code cell does not introduce another action or age its score.
if input.tool_name.is_default_namespace() && input.tool_name.name == "wait" {
Expand Down Expand Up @@ -129,6 +141,7 @@ impl GuardianV2Extension {
}
None => (score_progress.observe(&input), Ok(None)),
};
tracing::Span::current().record("tool_call_index", tool_call_index);
let reservation = match reservation {
Ok(reservation) => reservation,
Err(error) => {
Expand Down
Loading