diff --git a/.gitignore b/.gitignore index 3fb88623..df987a07 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ MANIFEST secrets/ *.json !package.json +.cursorrules # Virtual environments venv/ @@ -34,7 +35,6 @@ env/ ENV/ .venv config.yml -config.docker.yml # uv .python-version uv.lock @@ -109,3 +109,5 @@ frontend/dist/ # Enterprise license private keys enterprise/keys/*.pem + +# \ No newline at end of file diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py index f73744ad..1b7c91ce 100644 --- a/app/api/v1/routes/call_import_evaluations.py +++ b/app/api/v1/routes/call_import_evaluations.py @@ -1,8877 +1,9250 @@ -"""Evaluation routes scoped to a Call Import batch.""" - -from __future__ import annotations - -import asyncio -import csv -import base64 -import io -import json -import math -import re -import statistics -from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple -from uuid import UUID - -from datetime import date, datetime, timedelta, timezone - -from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status -from fastapi.responses import StreamingResponse -from loguru import logger -from pydantic import BaseModel, Field, field_validator -from sqlalchemy import desc, func, or_, text -from sqlalchemy.orm import Session -from sqlalchemy.orm.attributes import flag_modified - -from app.core.auth import Principal, get_principal -from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message -from app.database import get_db -from app.dependencies import ( - get_api_key, - get_organization_id, - get_workspace_id, - require_enterprise_feature, -) -from app.services.workspace_rbac import resolve_workspace_capabilities -from app.models.database import ( - AIProvider, - CallImport, - CallImportEvaluation, - CallImportEvaluationReportSnapshot, - CallImportEvaluationRow, - CallImportRow, - Metric, - PromptPartial, - Workspace, -) -from app.models.enums import CallImportRowStatus, ModelProvider -from app.models.schemas import ( - CallImportEvaluationAggregateResponse, - CallImportEvaluationBulkDelete, - CallImportEvaluationBulkActionResponse, - CallImportEvaluationCreate, - CallImportEvaluationListResponse, - CallImportEvaluationResponse, - CallImportEvaluationRetryRequest, - CallImportEvaluationRetryResponse, - CallImportEvaluationRetrySkippedItem, - CallImportEvaluationRowListResponse, - CallImportEvaluationRowResponse, - CallImportEvaluationUpdate, - CallImportMetricAggregate, - CallImportMetricHistogramBucket, - CallImportMetricLabelPair, - CallImportMetricSummary, - CallImportMetricValueCount, - DiscoveredLabelDeleteRequest, - DiscoveredLabelItem, - DiscoveredLabelMergeRequest, - DiscoveredLabelsResponse, - DiscoveredMetricDeleteRequest, - DiscoveredMetricItem, - DiscoveredMetricMergeRequest, - DiscoveredMetricsResponse, - EvaluationInsightsRequest, - EvaluationTldrSummary, - EvaluationMetricClustersRequest, - EvaluationMetricClustersState, - EvaluationPromptImprovementsRequest, - EvaluationPromptImprovementsState, - MetricFailurePoliciesResponse, - MetricFailurePoliciesSaveRequest, - MetricFailurePolicy, - MetricClusterEligibleRow, - MetricClusterEligibleRowsResponse, - EvaluationUserInsightsRequest, - EvaluationUserInsightsState, - MetricFlowEdge, - MetricPeriodDelta, - MetricFlowNode, - MetricFlowResponse, -) -from app.services.reporting.call_import_evaluation_pdf_report import ( - call_import_evaluation_pdf_report_service, -) -from app.services.call_import_metric_clusters import ( - METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, - estimate_metric_clusters_llm_calls, - filter_completed_row_pairs, - list_eligible_cluster_rows, - metric_clusters_raw_is_cancelled, - metric_clusters_state_from_raw, - metric_clusters_state_to_db, -) -from app.services.metric_failure_policy import ( - aggregate_primary_percent, - build_failure_policy_previews, - effective_policies, - failure_rate_percent_from_rows, - failure_policies_to_db, - has_clusterable_metrics, - merge_clustering_policies, - merge_failure_policies_into_raw, - policies_from_evaluation_raw, - validate_failure_policies_for_metrics, -) -from app.services.call_import_user_insights import ( - normalize_max_llm_calls, - total_llm_calls_for_rows, - user_insights_state_from_raw, -) - -router = APIRouter( - prefix="/call-imports/{call_import_id}/evaluations", - tags=["Call Import Evaluations"], - dependencies=[Depends(require_enterprise_feature("call_imports"))], -) - - -class CallImportEvaluationPdfReportRequest(BaseModel): - vendor_name: str = Field(..., min_length=1, max_length=120) - report_type: Literal["external", "internal"] = "external" - include_weekly_delta: bool = False - include_period_delta: bool = False - baseline_evaluation_id: Optional[str] = None - period_label: Optional[str] = Field(default=None, max_length=64) - use_case: Optional[str] = Field(default=None, max_length=120) - internal_brand_image_id: Optional[str] = None - external_brand_image_id: Optional[str] = None - report_config: Dict[str, Any] = Field(default_factory=dict) - platform_base_url: Optional[str] = Field( - default=None, - max_length=512, - description="Frontend origin for deep links to example calls in internal PDFs.", - ) - - @field_validator("vendor_name") - @classmethod - def _clean_vendor_name(cls, value: str) -> str: - cleaned = value.strip() - if not cleaned: - raise ValueError("Vendor name is required.") - return cleaned - - -class CallImportEvaluationBaselineCandidate(BaseModel): - evaluation_id: str - name: str - dataset: str - period_label: Optional[str] = None - period_start: Optional[date] = None - period_end: Optional[date] = None - period_display: str - completed_rows: int - created_at: datetime - is_default: bool = False - - -class CallImportEvaluationBaselineCandidatesResponse(BaseModel): - items: List[CallImportEvaluationBaselineCandidate] - default_evaluation_id: Optional[str] = None - - -def _require_import( - db: Session, - call_import_id: UUID, - organization_id: UUID, -) -> CallImport: - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException(status_code=404, detail="Call import not found") - return call_import - - -def require_call_import_capability(capability: str): - """Ensure the caller has *capability* in the call import's workspace (not just the header).""" - - def _dep( - call_import_id: UUID, - principal: Principal = Depends(get_principal), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), - ) -> CallImport: - call_import = _require_import(db, call_import_id, organization_id) - caps, _, role = resolve_workspace_capabilities( - db, - principal=principal, - workspace_id=call_import.workspace_id, - organization_id=organization_id, - ) - if capability not in caps: - raise HTTPException( - status_code=403, - detail=capability_denied_message( - capability, - role_name=role.name if role else None, - workspace_label="the active workspace", - ), - ) - return call_import - - return _dep - - -def _flatten_transcript(text: Optional[str]) -> str: - """Collapse a multi-line transcript onto a single line for spreadsheet export. - - The diarised transcript is stored as ``: `` lines joined - by ``\\n`` because the in-app ``TranscriptView`` parses those line - breaks to render chat bubbles. In Excel / Google Sheets that same - newline-per-turn formatting causes each cell to balloon vertically, - which the user reads as "lots of empty space on top of the cell". - Flattening at export time keeps the DB shape intact while giving the - spreadsheet a single-line cell per row. - """ - if not text: - return "" - parts = [ - segment.strip() - for segment in text.replace("\r\n", "\n").replace("\r", "\n").split("\n") - ] - return " ".join(p for p in parts if p) - - -def _evaluated_transcript_source_label( - evaluation: CallImportEvaluation, - source_row: CallImportRow, -) -> str: - """Label which transcript source this row was scored against.""" - source = (evaluation.transcript_source or "diarised").strip().lower() - if source == "production": - if not (source_row.transcript or "").strip(): - return "" - return "Production" - if not (source_row.diarised_transcript or "").strip(): - return "" - return "Diarised" - - -def _pick_evaluation_row_transcript( - source_row: Optional[CallImportRow], - evaluation: Optional[CallImportEvaluation] = None, -) -> Optional[str]: - """Transcript shown in evaluation row detail for the run's source.""" - if source_row is None: - return None - source = ( - (evaluation.transcript_source or "diarised").strip().lower() - if evaluation is not None - else "diarised" - ) - if source == "production": - raw = (source_row.transcript or "").strip() - return raw or None - diarised = (source_row.diarised_transcript or "").strip() - if diarised: - return diarised - raw = (source_row.transcript or "").strip() - return raw or None - - -def _to_evaluation_row_response( - eval_row_obj: CallImportEvaluationRow, - source_row: Optional[CallImportRow], - evaluation: Optional[CallImportEvaluation] = None, -) -> CallImportEvaluationRowResponse: - """Serialize one evaluation row plus joined source-row metadata.""" - return CallImportEvaluationRowResponse( - id=eval_row_obj.id, - evaluation_id=eval_row_obj.evaluation_id, - call_import_row_id=eval_row_obj.call_import_row_id, - row_index=source_row.row_index if source_row else None, - conversation_id=source_row.conversation_id if source_row else None, - transcript=_pick_evaluation_row_transcript(source_row, evaluation), - raw_columns=source_row.raw_columns if source_row else None, - recording_url=source_row.recording_url if source_row else None, - recording_date=source_row.recording_date if source_row else None, - recording_s3_key=source_row.recording_s3_key if source_row else None, - diarised_transcript_status=( - source_row.diarised_transcript_status if source_row else None - ), - diarised_transcript_error=( - source_row.diarised_transcript_error if source_row else None - ), - status=eval_row_obj.status, - metric_scores=eval_row_obj.metric_scores or {}, - error_message=eval_row_obj.error_message, - started_at=eval_row_obj.started_at, - finished_at=eval_row_obj.finished_at, - created_at=eval_row_obj.created_at, - updated_at=eval_row_obj.updated_at, - ) - - -def _serialize_selected_metric_ids(value) -> List[UUID]: - result: List[UUID] = [] - if not isinstance(value, list): - return result - for item in value: - try: - result.append(UUID(str(item))) - except (TypeError, ValueError): - continue - return result - - -def _metrics_for_ids(db: Session, org_id: UUID, ids: List[UUID]) -> List[Metric]: - if not ids: - return [] - rows = ( - db.query(Metric) - .filter( - Metric.organization_id == org_id, - Metric.id.in_(ids), - ) - .all() - ) - by_id = {row.id: row for row in rows} - return [by_id[mid] for mid in ids if mid in by_id] - - -def _expand_metric_selection( - db: Session, - org_id: UUID, - selected_ids: List[UUID], -) -> Tuple[List[Metric], Dict[UUID, List[Metric]]]: - """Resolve user-supplied metric ids into actual leaves + parent grouping. - - Rules: - * If a parent id is in ``selected_ids`` and no specific children of - that parent are also listed, include EVERY enabled child of that - parent. - * If a parent id AND some of its children are listed, include only - the listed children (treat the parent selection as the - "container" so users can deselect labels). - * Standalone metrics (no parent, no children) pass through - unchanged. - * Disabled metrics are filtered out at this layer so the caller - doesn't have to repeat the check. - - Returns: - (effective_metrics, parent_to_children) - - ``effective_metrics`` is the deduplicated list of metrics the - worker will actually score (children + standalone). Order is - preserved from ``selected_ids`` for display stability. - - ``parent_to_children`` maps each parent metric id (UUID) to the - list of its selected children. Useful for grouping in the LLM - prompt builder. - """ - if not selected_ids: - return [], {} - - requested = list(selected_ids) - initial_rows = ( - db.query(Metric) - .filter( - Metric.organization_id == org_id, - Metric.id.in_(requested), - ) - .all() - ) - initial_by_id = {row.id: row for row in initial_rows} - - parent_ids_requested = { - m.id for m in initial_rows if m.selection_mode and not m.parent_metric_id - } - # Map parent id -> children explicitly requested by the user. - explicit_children_by_parent: Dict[UUID, List[Metric]] = {} - for m in initial_rows: - if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: - explicit_children_by_parent.setdefault( - m.parent_metric_id, [] - ).append(m) - - # For parents without explicit children, hydrate every enabled child. - parents_needing_full_expansion = [ - pid - for pid in parent_ids_requested - if pid not in explicit_children_by_parent - ] - auto_expanded_children: Dict[UUID, List[Metric]] = {} - if parents_needing_full_expansion: - for pid in parents_needing_full_expansion: - child_rows = ( - db.query(Metric) - .filter( - Metric.organization_id == org_id, - Metric.parent_metric_id == pid, - Metric.enabled.is_(True), - ) - .order_by(Metric.created_at.asc()) - .all() - ) - auto_expanded_children[pid] = child_rows - - parent_to_children: Dict[UUID, List[Metric]] = {} - for pid in parent_ids_requested: - children = explicit_children_by_parent.get( - pid - ) or auto_expanded_children.get(pid, []) - # Drop disabled children so the worker doesn't waste a slot on - # them. Empty parents (no enabled children) are still tracked - # because the UI may want to show "0 of 0" rather than swallow - # them silently. - parent_to_children[pid] = [c for c in children if c.enabled] - - effective: List[Metric] = [] - seen: set[UUID] = set() - for mid in requested: - m = initial_by_id.get(mid) - if m is None: - continue - if m.selection_mode and not m.parent_metric_id: - # Parent row itself is not scored — only its children. - for child in parent_to_children.get(m.id, []): - if child.id in seen or not child.enabled: - continue - seen.add(child.id) - effective.append(child) - continue - if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: - # Already accounted for via the parent expansion above. - continue - if not m.enabled: - continue - if m.id in seen: - continue - seen.add(m.id) - effective.append(m) - - return effective, parent_to_children - - -def _evaluation_bulk_operation_for_response( - evaluation_id: UUID, -) -> Optional[str]: - from app.services.call_imports.evaluation_bulk_op import ( - get_evaluation_bulk_operation, - ) - - return get_evaluation_bulk_operation(evaluation_id) - - -def _serialize_eval( - db: Session, - row: CallImportEvaluation, - *, - sibling_evaluation_ids: Optional[List[UUID]] = None, -) -> CallImportEvaluationResponse: - selected_ids = _serialize_selected_metric_ids(row.selected_metric_ids) - - # Pull every metric referenced anywhere in the run's grouping (leaves, - # standalone, AND parents from selected_metric_groups) so the UI can - # render parent labels even when only children were materialized into - # selected_metric_ids. - groups_raw: Dict[str, List[str]] = {} - if isinstance(row.selected_metric_groups, dict): - for parent_str, children in row.selected_metric_groups.items(): - if not isinstance(children, list): - continue - cleaned: List[str] = [] - for c in children: - try: - UUID(str(c)) - cleaned.append(str(c)) - except (TypeError, ValueError): - continue - try: - UUID(parent_str) - groups_raw[parent_str] = cleaned - except (TypeError, ValueError): - continue - - metric_ids_for_lookup: List[UUID] = list(selected_ids) - for parent_str in groups_raw.keys(): - try: - pid = UUID(parent_str) - if pid not in metric_ids_for_lookup: - metric_ids_for_lookup.append(pid) - except (TypeError, ValueError): - continue - - metrics = _metrics_for_ids( - db, row.organization_id, metric_ids_for_lookup - ) - - from app.services.call_imports.progress_counters import merge_eval_counters_for_ui - - ui_completed_raw, ui_failed_raw = merge_eval_counters_for_ui(row) - total = int(row.total_rows or 0) - ui_completed = ( - min(ui_completed_raw, total) if total else ui_completed_raw - ) - ui_failed = min(ui_failed_raw, total) if total else ui_failed_raw - - return CallImportEvaluationResponse( - id=row.id, - call_import_id=row.call_import_id, - organization_id=row.organization_id, - name=row.name, - selected_metric_ids=selected_ids, - selected_metric_groups=groups_raw or None, - metrics=[ - CallImportMetricSummary( - id=metric.id, - name=metric.name, - metric_type=metric.metric_type, - description=metric.description, - parent_metric_id=metric.parent_metric_id, - selection_mode=metric.selection_mode, - # Required by the Flow tab to know whether a parent - # opted into discovery; without it the - # DiscoveredLabelsPanel stays hidden even when the - # worker is actively producing discovered_labels. - allow_discovery=bool( - getattr(metric, "allow_discovery", False) - ), - ) - for metric in metrics - ], - status=row.status, - total_rows=row.total_rows, - completed_rows=ui_completed, - failed_rows=ui_failed, - error_message=row.error_message, - llm_provider=row.llm_provider, - llm_model=row.llm_model, - llm_credential_id=row.llm_credential_id, - llm_config=( - row.llm_config if isinstance(getattr(row, "llm_config", None), dict) else None - ), - metric_llm_overrides=( - row.metric_llm_overrides - if isinstance(row.metric_llm_overrides, dict) - else None - ), - stt_provider=row.stt_provider, - stt_model=row.stt_model, - stt_credential_id=row.stt_credential_id, - diarisation_llm_provider=getattr(row, "diarisation_llm_provider", None), - diarisation_llm_model=getattr(row, "diarisation_llm_model", None), - diarisation_llm_credential_id=getattr( - row, "diarisation_llm_credential_id", None - ), - diarisation_prompt=getattr(row, "diarisation_prompt", None), - transcribe_mode=( - (getattr(row, "transcribe_mode", None) or "stt_llm") - ), - transcript_source=(row.transcript_source or "diarised"), - sibling_evaluation_ids=list(sibling_evaluation_ids or []), - started_at=row.started_at, - finished_at=row.finished_at, - created_at=row.created_at, - updated_at=row.updated_at, - tldr_summary=_tldr_summary_payload(row), - user_insights=_user_insights_payload(row), - metric_clusters=_metric_clusters_payload(row), - discover_new_metrics=bool( - getattr(row, "discover_new_metrics", False) - ), - bulk_operation=_evaluation_bulk_operation_for_response(row.id), - ) - - -def _normalize_name(value: Optional[str]) -> Optional[str]: - """Trim user-supplied name; empty string becomes ``NULL``.""" - if value is None: - return None - trimmed = value.strip() - return trimmed or None - - -def _rollup_evaluation_status(evaluation: CallImportEvaluation, db: Session) -> None: - """Recompute counters + terminal status after rows are added/removed. - - Uses a single aggregate query instead of loading every row status. - """ - from app.workers.tasks.evaluate_call_import_row_core import ( - _apply_parent_status_from_counters, - reconcile_evaluation_counters, - ) - - reconcile_evaluation_counters(db, evaluation) - _apply_parent_status_from_counters(evaluation) - db.flush() - - if evaluation.status in {"completed", "failed", "partial"}: - from app.models.database import CallImport - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - - call_import = ( - db.query(CallImport) - .filter(CallImport.id == evaluation.call_import_id) - .first() - ) - if call_import is not None: - rollup_call_import_batch_status(db, call_import) - - -@router.post( - "", - response_model=CallImportEvaluationResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="createCallImportEvaluation", -) -async def create_call_import_evaluation( - call_import_id: UUID, - payload: CallImportEvaluationCreate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationResponse: - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - metric_ids = payload.metric_ids - if not metric_ids: - raise HTTPException( - status_code=400, - detail="Select at least one metric to run the evaluation against.", - ) - - org_metrics = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.id.in_(metric_ids), - ) - .all() - ) - by_id = {metric.id: metric for metric in org_metrics} - unknown_ids = [mid for mid in metric_ids if mid not in by_id] - if unknown_ids: - raise HTTPException( - status_code=400, - detail=( - "These metric ids do not exist in your organization: " - f"{', '.join(str(mid) for mid in unknown_ids)}. " - "Refresh the metrics list and try again." - ), - ) - # Parents themselves are containers, not scored rows, so a disabled - # parent shouldn't block the run as long as it has enabled children. - # We only reject disabled rows that the worker will actually try to - # evaluate (children + standalone leaves). - disabled_leaves = [ - metric - for metric in org_metrics - if not metric.enabled - and not (metric.selection_mode and not metric.parent_metric_id) - ] - if disabled_leaves: - names = ", ".join(metric.name for metric in disabled_leaves) - raise HTTPException( - status_code=400, - detail=( - f"These metrics are disabled and cannot be evaluated: {names}. " - "Enable them on the Metrics page (or pick different ones) and " - "try again." - ), - ) - - # Expand hierarchical selection: parents auto-include their enabled - # children, mixed parent+child selections respect the user's subset. - effective_metrics, parent_to_children = _expand_metric_selection( - db, organization_id, metric_ids - ) - if not effective_metrics: - raise HTTPException( - status_code=400, - detail=( - "None of the selected metrics yielded an enabled leaf to " - "evaluate. Check that parent categories have enabled " - "children, then try again." - ), - ) - - # The effective list (children + standalone leaves) is what gets - # persisted to ``selected_metric_ids`` and scored by the worker. - # The original parents are preserved in ``selected_metric_groups`` - # so the UI can rebuild the tree later. - leaf_metric_ids: List[UUID] = [m.id for m in effective_metrics] - selected_metric_groups: Dict[str, List[str]] = { - str(pid): [str(c.id) for c in children] - for pid, children in parent_to_children.items() - } - metric_rows = effective_metrics - valid_metric_id_strs = {str(m.id) for m in metric_rows} - - # ----- Validate run-level + per-metric LLM config ----- - llm_provider_norm: Optional[str] = None - llm_model_norm: Optional[str] = None - if payload.llm_provider or payload.llm_model: - if not (payload.llm_provider and payload.llm_model): - raise HTTPException( - status_code=400, - detail="Both llm_provider and llm_model are required when overriding the run LLM.", - ) - try: - llm_provider_norm = ModelProvider( - payload.llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Unknown LLM provider '{payload.llm_provider}'. " - "Valid keys are documented in ModelProvider." - ), - ) - llm_model_norm = payload.llm_model.strip() or None - if not llm_model_norm: - raise HTTPException( - status_code=400, detail="llm_model cannot be empty." - ) - - if payload.llm_credential_id is not None: - cred = ( - db.query(AIProvider) - .filter( - AIProvider.id == payload.llm_credential_id, - AIProvider.organization_id == organization_id, - ) - .first() - ) - if not cred: - raise HTTPException( - status_code=400, - detail=( - "The provided llm_credential_id does not exist in this " - "organization." - ), - ) - - # Per-metric overrides: keys can be either a leaf metric id (applies - # to that metric only) or a parent metric id (applies to every - # child of that parent). Parent keys are expanded to their - # children so the worker only sees concrete leaf ids. - metric_overrides_payload: Optional[Dict[str, Dict[str, Any]]] = None - if payload.metric_llm_overrides: - metric_overrides_payload = {} - for metric_id, override in payload.metric_llm_overrides.items(): - target_leaf_ids: List[str] = [] - if metric_id in valid_metric_id_strs: - target_leaf_ids = [metric_id] - else: - # Maybe it's a parent id — expand to the children that - # are part of THIS run. - try: - parent_uuid = UUID(metric_id) - except (TypeError, ValueError): - raise HTTPException( - status_code=400, - detail=( - "metric_llm_overrides references metric " - f"{metric_id} which is not a valid UUID." - ), - ) - children_for_parent = parent_to_children.get(parent_uuid) - if not children_for_parent: - raise HTTPException( - status_code=400, - detail=( - "metric_llm_overrides references metric " - f"{metric_id} which is not in metric_ids." - ), - ) - target_leaf_ids = [str(c.id) for c in children_for_parent] - - override_dict: Dict[str, Any] = {} - if override.provider is not None: - if not override.model: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a provider " - "but no model." - ), - ) - try: - override_dict["provider"] = ModelProvider( - override.provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} uses unknown " - f"provider '{override.provider}'." - ), - ) - override_dict["model"] = override.model.strip() - elif override.model: - # Model without provider doesn't make sense — treat as 400 - # so the UI can fix it instead of silently falling back. - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a model but " - "no provider." - ), - ) - if override.credential_id is not None: - override_dict["credential_id"] = str(override.credential_id) - if override.llm_config is not None: - override_dict["llm_config"] = override.llm_config - if override_dict: - for leaf_id in target_leaf_ids: - metric_overrides_payload[leaf_id] = override_dict - - # ----- Validate auto-transcribe settings ----- - # Diarised runs auto-diarise rows missing a diarised transcript and - # require STT + diariser LLM config. Production runs score the CSV - # transcript directly and skip diarisation entirely. - use_diarised = payload.transcript_sources[0] == "diarised" - auto_transcribe = use_diarised - - transcribe_mode_norm: Optional[str] = None - stt_provider_norm: Optional[str] = None - stt_model_norm: Optional[str] = None - diarisation_llm_provider_norm: Optional[str] = None - diarisation_llm_model_norm: Optional[str] = None - diarisation_prompt_norm: Optional[str] = None - - if use_diarised: - transcribe_mode_norm = (payload.transcribe_mode or "stt_llm").strip().lower() - if transcribe_mode_norm not in {"stt_llm", "llm_only"}: - raise HTTPException( - status_code=400, - detail=( - f"Unknown transcribe_mode '{payload.transcribe_mode}'. " - "Expected 'stt_llm' or 'llm_only'." - ), - ) - - if transcribe_mode_norm == "stt_llm": - if not payload.stt_provider: - raise HTTPException( - status_code=400, - detail=( - "stt_provider is required when " - "transcribe_mode='stt_llm': every evaluation run " - "auto-diarises rows that are missing a diarised " - "transcript." - ), - ) - if not payload.stt_model: - raise HTTPException( - status_code=400, - detail=( - "stt_model is required when transcribe_mode='stt_llm'." - ), - ) - try: - stt_provider_norm = ModelProvider( - payload.stt_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=f"Unknown STT provider '{payload.stt_provider}'.", - ) - stt_model_norm = payload.stt_model.strip() or None - if not stt_model_norm: - raise HTTPException( - status_code=400, detail="stt_model cannot be empty." - ) - else: - # llm_only — explicitly reject lingering STT inputs so the - # contract is unambiguous (the worker would ignore them but - # silent acceptance hides accidental misconfiguration). - if (payload.stt_provider or "").strip() or ( - payload.stt_model or "" - ).strip(): - raise HTTPException( - status_code=400, - detail=( - "stt_provider / stt_model must be omitted when " - "transcribe_mode='llm_only'; the LLM consumes the " - "audio directly." - ), - ) - - # --- Validate LLM diariser settings ----- - if not payload.diarization_llm_provider: - raise HTTPException( - status_code=400, - detail=( - "diarization_llm_provider is required: every evaluation " - "run diarises STT output with an LLM." - ), - ) - if not payload.diarization_llm_model: - raise HTTPException( - status_code=400, - detail=( - "diarization_llm_model is required: every evaluation " - "run diarises STT output with an LLM." - ), - ) - try: - diarisation_llm_provider_norm = ModelProvider( - payload.diarization_llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Unknown diarisation LLM provider " - f"'{payload.diarization_llm_provider}'." - ), - ) - diarisation_llm_model_norm = ( - payload.diarization_llm_model.strip() or None - ) - if not diarisation_llm_model_norm: - raise HTTPException( - status_code=400, - detail="diarization_llm_model cannot be empty.", - ) - diarisation_prompt_norm = ( - payload.diarization_prompt.strip() - if isinstance(payload.diarization_prompt, str) - else None - ) or None - - from app.models.enums import CallImportParameterType, CallImportStatus - from app.services.call_imports.bulk_ops import ( - count_all_source_rows, - count_completed_source_rows, - count_source_rows_with_production_transcript, - ) - - starting_from_mapped = False - if call_import.status == CallImportStatus.MAPPED: - if not call_import.source_s3_key or not call_import.source_format: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This batch has no staged source file. Upload and map " - "a CSV/Excel file before running evaluation." - ), - ) - if not call_import.schema_id: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Cannot run evaluation without a mapped schema.", - ) - from app.api.v1.routes.call_imports import ( - _ensure_blob_storage_enabled, - _resolve_schema, - _resolve_telephony_integration, - _validate_direct_url_import_ready, - ) - - workspace_id = call_import.workspace_id - schema = _resolve_schema( - db, organization_id, workspace_id, call_import.schema_id - ) - parameters = list(schema.parameters) - if not use_diarised: - transcript_mapped = any( - param.type == CallImportParameterType.TRANSCRIPT - and (call_import.parameter_mapping or {}).get(param.name) - for param in parameters - ) - if not transcript_mapped: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "No transcript column is mapped in this batch. " - "Map a schema transcript parameter to a CSV column, " - "or choose 'Diarize then evaluate'." - ), - ) - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - else: - _validate_direct_url_import_ready( - parameters, dict(call_import.parameter_mapping or {}) - ) - integration = None - - _ensure_blob_storage_enabled() - - if integration is not None: - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - else: - call_import.provider = None - call_import.telephony_integration_id = None - - call_import.total_rows = 0 - call_import.completed_rows = 0 - call_import.failed_rows = 0 - call_import.error_message = None - call_import.status = CallImportStatus.PROCESSING - db.commit() - db.refresh(call_import) - starting_from_mapped = True - - if use_diarised: - total_row_count = count_completed_source_rows(db, call_import.id) - else: - # Production runs score CSV text — rows need not wait for - # recording fetch to finish before they are evaluable. - total_row_count = count_source_rows_with_production_transcript( - db, call_import.id - ) - - requested_sources: List[str] = list(payload.transcript_sources) - - if ( - not use_diarised - and not starting_from_mapped - and count_all_source_rows(db, call_import.id) > 0 - and total_row_count == 0 - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "No rows have a production transcript. " - "Choose 'Diarize then evaluate' or import rows with " - "a transcript column." - ), - ) - - base_name = _normalize_name(payload.name) - - def _name_for_source(source: str) -> Optional[str]: - # Single-source runs preserve the user's chosen name verbatim. - del source - return base_name - - created_evaluations: List[CallImportEvaluation] = [] - - for source in requested_sources: - evaluation = CallImportEvaluation( - call_import_id=call_import.id, - organization_id=organization_id, - # Mirror the parent CallImport's workspace so listings can - # filter on workspace_id directly without joining. - workspace_id=call_import.workspace_id, - name=_name_for_source(source), - selected_metric_ids=[ - str(metric_id) for metric_id in leaf_metric_ids - ], - selected_metric_groups=selected_metric_groups or None, - status="pending", - total_rows=total_row_count, - completed_rows=0, - failed_rows=0, - llm_provider=llm_provider_norm, - llm_model=llm_model_norm, - llm_credential_id=payload.llm_credential_id, - llm_config=payload.llm_config, - metric_llm_overrides=metric_overrides_payload, - stt_provider=stt_provider_norm, - stt_model=stt_model_norm, - stt_credential_id=( - payload.stt_credential_id if auto_transcribe else None - ), - diarisation_llm_provider=diarisation_llm_provider_norm, - diarisation_llm_model=diarisation_llm_model_norm, - diarisation_llm_credential_id=( - payload.diarization_llm_credential_id if auto_transcribe else None - ), - diarisation_prompt=diarisation_prompt_norm, - transcribe_mode=transcribe_mode_norm, - transcript_source=source, - discover_new_metrics=bool( - getattr(payload, "discover_new_metrics", False) - ), - ) - db.add(evaluation) - db.flush() - created_evaluations.append(evaluation) - - db.commit() - for evaluation in created_evaluations: - db.refresh(evaluation) - - primary_evaluation = created_evaluations[0] - sibling_ids = [e.id for e in created_evaluations[1:]] - - if not total_row_count and not starting_from_mapped: - for evaluation in created_evaluations: - evaluation.status = "completed" - db.commit() - for evaluation in created_evaluations: - db.refresh(evaluation) - return _serialize_eval( - db, primary_evaluation, sibling_evaluation_ids=sibling_ids - ) - - if starting_from_mapped: - from app.workers.tasks.call_import_bulk_ops import ( - materialize_mapped_call_import_evaluation_task, - ) - - for evaluation in created_evaluations: - materialize_mapped_call_import_evaluation_task.delay( - str(call_import.id), - str(organization_id), - str(call_import.workspace_id), - str(evaluation.id), - transcribe_overwrite=payload.transcribe_overwrite, - ) - else: - from app.workers.tasks.call_import_bulk_ops import ( - materialize_call_import_evaluation_task, - ) - - for evaluation in created_evaluations: - materialize_call_import_evaluation_task.delay( - str(evaluation.id), - transcribe_overwrite=payload.transcribe_overwrite, - ) - - for evaluation in created_evaluations: - db.refresh(evaluation) - - return _serialize_eval( - db, primary_evaluation, sibling_evaluation_ids=sibling_ids - ) - - -@router.get( - "", - response_model=CallImportEvaluationListResponse, - operation_id="listCallImportEvaluations", -) -async def list_call_import_evaluations( - call_import_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationListResponse: - del api_key - _require_import(db, call_import_id, organization_id) - rows = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .order_by(desc(CallImportEvaluation.created_at)) - .all() - ) - return CallImportEvaluationListResponse( - items=[_serialize_eval(db, row) for row in rows], - total=len(rows), - ) - - -@router.get( - "/{eval_id}", - response_model=CallImportEvaluationResponse, - operation_id="getCallImportEvaluation", -) -async def get_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationResponse: - del api_key - _require_import(db, call_import_id, organization_id) - row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - return _serialize_eval(db, row) - - -@router.get( - "/{eval_id}/rows", - response_model=CallImportEvaluationRowListResponse, - operation_id="listCallImportEvaluationRows", -) -async def list_call_import_evaluation_rows( - call_import_id: UUID, - eval_id: UUID, - page: int = Query(1, ge=1), - page_size: int = Query(100, ge=1, le=500), - q: Optional[str] = Query( - None, - description=( - "Free-text search across conversation_id and transcript " - "(case-insensitive substring match)." - ), - ), - metric_id: Optional[UUID] = Query( - None, - description=( - "If set, only return rows whose ``metric_scores[metric_id].value`` " - "exactly matches ``metric_value`` (string-compared). " - "Use together with ``metric_value``." - ), - ), - metric_value: Optional[str] = Query( - None, - description="Value to match against metric_id (string compare).", - ), - status_filter: Optional[str] = Query( - None, - alias="status", - description="Restrict to rows with this evaluation row status.", - ), - flow_parent_id: Optional[UUID] = Query( - None, - description=( - "Parent (category) metric whose ``sequence`` array should be " - "checked against ``flow_node`` and ``flow_edge_target``. Used " - "to drill into the calls behind a flow-chart node or edge." - ), - ), - flow_node: Optional[str] = Query( - None, - description=( - "If set together with ``flow_parent_id``, only return rows " - "whose sequence under that parent contains this step. Accepts " - "either a child metric UUID (resolved to slug(name)), a " - "``disc:`` discovered-label id, or a raw slug." - ), - ), - flow_edge_target: Optional[str] = Query( - None, - description=( - "Optional companion to ``flow_node``: when set, restrict to " - "rows whose sequence contains the directed transition " - "``flow_node -> flow_edge_target`` (immediately adjacent). " - "Same id format as ``flow_node``." - ), - ), - discovered_parent_id: Optional[UUID] = Query( - None, - description=( - "Parent (category) metric that defines the discovery scope " - "for ``discovered_label_key`` / ``has_discovered``." - ), - ), - discovered_label_key: Optional[str] = Query( - None, - description=( - "If set together with ``discovered_parent_id``, only return " - "rows whose ``metric_scores[parent].discovered_labels`` " - "list contains an entry with this slug (after applying " - "evaluation-level merge aliases)." - ), - ), - has_discovered: Optional[bool] = Query( - None, - description=( - "If true together with ``discovered_parent_id``, only return " - "rows that have at least one LLM-discovered label for the " - "parent. Useful to triage which calls produced novel labels." - ), - ), - sort_by: Optional[str] = Query( - None, - description=( - "Column to sort by. Accepted values: ``row_index`` (default " - "when omitted), ``conversation_id``, ``status`` (the " - "evaluation-row status), or ``metric:`` to sort " - "by ``metric_scores[].value``. Metric sorts compare " - "the extracted JSON text — adequate for booleans, enum " - "labels, and 0-1 ratings; large integer values may sort " - "lexicographically (10 before 2)." - ), - ), - sort_dir: Optional[str] = Query( - "asc", - description="Sort direction: ``asc`` (default) or ``desc``.", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationRowListResponse: - del api_key - _require_import(db, call_import_id, organization_id) - - eval_row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not eval_row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - query = ( - db.query(CallImportEvaluationRow, CallImportRow) - .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) - .filter(CallImportEvaluationRow.evaluation_id == eval_id) - ) - - # --- Filters ---------------------------------------------------------- - if q and q.strip(): - needle = f"%{q.strip()}%" - # Search across both transcript columns so a hit in either the - # production or the diarised version surfaces the row, - # independent of which source the evaluation actually scored. - query = query.filter( - or_( - CallImportRow.conversation_id.ilike(needle), - CallImportRow.transcript.ilike(needle), - CallImportRow.diarised_transcript.ilike(needle), - ) - ) - - if status_filter: - # The CallImportEvaluationRow.status column is a string in PG so a - # plain == filter works; we lowercase to match the stored values. - query = query.filter( - CallImportEvaluationRow.status == status_filter.strip().lower() - ) - - if metric_id is not None and metric_value is not None: - # ``metric_scores`` is a JSONB column shaped like - # ``{"": {"value": , "type": "boolean", ...}}``. We - # extract the nested ``value`` as text and compare to the user - # input as a string — that handles bool/int/enum without needing - # per-type casts. ``metric_value`` is matched case-insensitively - # so chart clicks on labels like "True" survive any casing drift - # between worker output and the chart label. - path_value = func.json_extract_path_text( - CallImportEvaluationRow.metric_scores, - str(metric_id), - "value", - ) - query = query.filter(func.lower(path_value) == metric_value.strip().lower()) - - # --- Flow chart drilldown filter ------------------------------------- - # Translates a clicked node (or edge) on the flow chart into a - # SQL filter against ``metric_scores[].sequence``. The - # frontend sends either a child UUID, a ``disc:`` discovered - # node id, or a raw slug — we normalize all three to the slug that - # actually appears in stored ``sequence`` arrays. - if flow_parent_id is not None and flow_node and flow_node.strip(): - parent_id_str_local = str(flow_parent_id) - alias_map_flow = _alias_map_for_parent(eval_row, flow_parent_id) - - def _flow_node_to_slug(raw: str) -> Optional[str]: - raw_clean = raw.strip() - if not raw_clean: - return None - if raw_clean == _FLOW_START_NODE_ID: - # The synthetic START node isn't a real sequence entry; - # filtering on it is meaningless so we skip silently. - return None - if raw_clean.startswith(_DISCOVERED_NODE_PREFIX): - return _resolve_alias( - alias_map_flow, - _slug_label(raw_clean[len(_DISCOVERED_NODE_PREFIX) :]), - ) - # Try to interpret as a child metric UUID first; fall back - # to treating it as a slug. - try: - child_uuid = UUID(raw_clean) - except (TypeError, ValueError): - return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) - child = ( - db.query(Metric.name) - .filter( - Metric.id == child_uuid, - Metric.organization_id == organization_id, - ) - .first() - ) - if child and child[0]: - return _resolve_alias(alias_map_flow, _slug_label(child[0])) - return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) - - from_slug = _flow_node_to_slug(flow_node) - target_slug: Optional[str] = None - if flow_edge_target and flow_edge_target.strip(): - target_slug = _flow_node_to_slug(flow_edge_target) - - if from_slug: - # The ``metric_scores`` column is declared as ``Column(JSON)`` - # in the model so on databases where the table was created - # from the model (rather than the migration) the physical - # type is ``json``, not ``jsonb``. The JSONB-only operators - # below (``jsonb_exists``, ``jsonb_array_elements_text``, - # ``@>``) require a JSONB input — we cast once up front so - # the same SQL works regardless of which path created the - # table. - scores_jsonb = ( - "(call_import_evaluation_rows.metric_scores)::jsonb" - ) - if target_slug: - # Edge filter: rows whose sequence under this parent - # contains ``from_slug`` immediately followed by - # ``target_slug``. Implemented as a correlated EXISTS - # over ``jsonb_array_elements_text`` with ORDINALITY, - # which is the portable way to express "next array - # index" against a JSONB array in Postgres. - edge_filter_sql = text( - f""" - EXISTS ( - SELECT 1 - FROM jsonb_array_elements_text( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ) - ) WITH ORDINALITY AS s1(elem, ord) - JOIN jsonb_array_elements_text( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ) - ) WITH ORDINALITY AS s2(elem, ord) - ON s2.ord = s1.ord + 1 - WHERE s1.elem = :from_slug - AND s2.elem = :to_slug - ) - """ - ).bindparams( - p_id=parent_id_str_local, - from_slug=from_slug, - to_slug=target_slug, - ) - query = query.filter(edge_filter_sql) - else: - # Node filter: rows whose ``metric_scores -> parent -> - # 'sequence'`` array contains ``from_slug``. We use the - # function form ``jsonb_exists`` rather than the ``?`` - # operator to avoid psycopg2 mistaking the question - # mark for a parameter placeholder. - node_filter_sql = text( - f""" - jsonb_exists( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ), - :slug - ) - """ - ).bindparams(p_id=parent_id_str_local, slug=from_slug) - query = query.filter(node_filter_sql) - - # --- Discovered label filters --------------------------------------- - # Surfaces "which calls produced THIS LLM-discovered label" and the - # broader "which calls produced ANY LLM-discovered label". Both - # operate on ``metric_scores[].discovered_labels`` (a list - # of dicts) plus the same ``sequence`` array — covering both legacy - # rows where the slug only made it into ``sequence`` and newer - # rows where it landed in both. - if discovered_parent_id is not None and ( - discovered_label_key or has_discovered - ): - d_parent_str = str(discovered_parent_id) - alias_map_disc = _alias_map_for_parent(eval_row, discovered_parent_id) - # See note above: cast once so the JSONB operators don't reject - # the column when it's typed as ``json`` in the database. - scores_jsonb = "(call_import_evaluation_rows.metric_scores)::jsonb" - if discovered_label_key and discovered_label_key.strip(): - target = _resolve_alias( - alias_map_disc, _slug_label(discovered_label_key) - ) - if target: - # Match rows whose discovered_labels list has an entry - # ``{"key": }`` OR whose sequence array still - # contains the slug. The latter covers older rows that - # were rewritten by a merge in the discovered_labels - # blob but whose sequence may have lagged. - contains_json = json.dumps( - {d_parent_str: {"discovered_labels": [{"key": target}]}} - ) - disc_filter_sql = text( - f""" - ( - {scores_jsonb} @> CAST(:contains AS JSONB) - OR - jsonb_exists( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ), - :slug - ) - ) - """ - ).bindparams( - contains=contains_json, - p_id=d_parent_str, - slug=target, - ) - query = query.filter(disc_filter_sql) - elif has_discovered: - # No specific slug — just rows that surfaced any candidate - # under this parent. We coalesce missing paths to ``[]`` so - # ``jsonb_array_length`` always sees an array (it raises on - # non-array inputs, but our shape guarantees a list when - # the key is present). - has_disc_sql = text( - f""" - jsonb_array_length( - COALESCE( - {scores_jsonb} -> :p_id -> 'discovered_labels', - '[]'::jsonb - ) - ) > 0 - """ - ).bindparams(p_id=d_parent_str) - query = query.filter(has_disc_sql) - - # --- Sorting ---------------------------------------------------------- - # Column-click sorting from the UI. Falls back to ``row_index`` so - # paging stays stable when the user clears the sort. We always add a - # secondary ``row_index`` tiebreaker so duplicate sort keys (e.g. - # many rows with ``status = 'completed'``) keep a deterministic - # order across page boundaries — without this, pagination can - # double-show or skip rows when Postgres picks a different physical - # order on each query. - direction_desc = (sort_dir or "asc").strip().lower() == "desc" - - def _apply_direction(column_expr): - return column_expr.desc() if direction_desc else column_expr.asc() - - # Whether the caller's ``sort_by`` resolved to a known column. We - # use this flag to decide whether ``sort_dir`` is honoured on the - # fallback path: unrecognized columns (typos, stale UI state) fall - # back to the implicit ``row_index ASC`` default and intentionally - # ignore ``sort_dir`` so users don't get a surprise reverse order - # from a typo'd column name. - sort_recognized = False - sort_by_clean = (sort_by or "").strip() - primary_sort = None - metric_uuid: Optional[UUID] = None - if sort_by_clean == "row_index": - sort_recognized = True - # Falls through to the default ``order_by`` below with - # ``primary_sort`` still None — but ``sort_recognized=True`` - # tells the fallback branch to apply the requested direction. - elif sort_by_clean == "conversation_id": - sort_recognized = True - primary_sort = _apply_direction(CallImportRow.conversation_id) - elif sort_by_clean == "status": - sort_recognized = True - primary_sort = _apply_direction(CallImportEvaluationRow.status) - elif sort_by_clean.startswith("metric:"): - raw_metric_id = sort_by_clean.split(":", 1)[1].strip() - try: - metric_uuid = UUID(raw_metric_id) - except (TypeError, ValueError): - metric_uuid = None - if metric_uuid is not None: - sort_recognized = True - # ``metric_scores`` is JSON-typed but the helper functions - # for path extraction differ between Postgres (production) - # and SQLite (default test backend). Branch on the active - # dialect so we can use the right primitive: - # * Postgres → ``json_extract_path_text(col, key, "value")`` - # which returns the value as TEXT for both ``json`` and - # ``jsonb`` columns. - # * SQLite → ``json_extract(col, '$."".value')`` - # using JSONPath syntax. ``metric_uuid`` is already - # validated above (``UUID(raw_metric_id)``), so the - # interpolated path is safe from injection. - # NULL values (rows where the metric wasn't scored) sort - # to the END regardless of direction so un-scored rows - # don't crowd the top of an ascending sort. - dialect_name = ( - db.bind.dialect.name if db.bind is not None else "postgresql" - ) - if dialect_name == "sqlite": - json_path = f'$."{metric_uuid}".value' - path_value = func.json_extract( - CallImportEvaluationRow.metric_scores, - json_path, - ) - else: - path_value = func.json_extract_path_text( - CallImportEvaluationRow.metric_scores, - str(metric_uuid), - "value", - ) - primary_sort = ( - path_value.desc().nullslast() - if direction_desc - else path_value.asc().nullslast() - ) - - if primary_sort is not None: - query = query.order_by(primary_sort, CallImportRow.row_index.asc()) - elif sort_recognized: - # Explicit ``sort_by=row_index`` request — honour direction. - query = query.order_by(_apply_direction(CallImportRow.row_index)) - else: - # No sort requested OR unrecognized column — safe default of - # ``row_index ASC``. We deliberately ignore ``sort_dir`` here - # so a typo'd / stale ``sort_by`` doesn't quietly invert the - # default order. - query = query.order_by(CallImportRow.row_index.asc()) - from app.db_sharding.eval_rows import fetch_evaluation_row_pairs_page - from app.db_sharding.sessions import is_sharding_enabled - - def _pair_row_index( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> int: - return int(pair[1].row_index or 0) - - def _directed_string(value: Optional[str], desc: bool) -> Tuple[int, ...]: - text = value or "" - if not desc: - return (0, *text.encode("utf-8")) - return (1, *(-byte for byte in text.encode("utf-8"))) - - if sort_by_clean == "conversation_id": - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[Any, ...]: - return ( - _directed_string(pair[1].conversation_id, direction_desc), - _pair_row_index(pair), - ) - elif sort_by_clean == "status": - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[Any, ...]: - return ( - _directed_string(pair[0].status, direction_desc), - _pair_row_index(pair), - ) - elif sort_by_clean.startswith("metric:") and metric_uuid is not None: - metric_id_str = str(metric_uuid) - - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[Any, ...]: - scores = pair[0].metric_scores or {} - entry = scores.get(metric_id_str, {}) - raw_value = entry.get("value") if isinstance(entry, dict) else None - null_rank = 1 if raw_value is None else 0 - return ( - null_rank, - _directed_string( - str(raw_value) if raw_value is not None else None, - direction_desc, - ), - _pair_row_index(pair), - ) - elif sort_recognized and sort_by_clean == "row_index": - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[int, ...]: - idx = _pair_row_index(pair) - return (-idx,) if direction_desc else (idx,) - else: - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[int, ...]: - return (_pair_row_index(pair),) - - if is_sharding_enabled(): - def _build_query(session: Session): - return query.with_session(session) - - total, rows = fetch_evaluation_row_pairs_page( - db, - _build_query, - page=page, - page_size=page_size, - sort_key=_pair_sort_key, - bounded_shard_fetch=( - not sort_recognized or sort_by_clean == "row_index" - ), - ) - else: - total = query.count() - rows = query.offset((page - 1) * page_size).limit(page_size).all() - - # Row detail shows the transcript for this run's chosen source. - items: List[CallImportEvaluationRowResponse] = [ - _to_evaluation_row_response(eval_row_obj, source_row, eval_row) - for eval_row_obj, source_row in rows - ] - - return CallImportEvaluationRowListResponse( - items=items, - total=total, - page=page, - page_size=page_size, - ) - - -@router.get( - "/{eval_id}/export", - operation_id="exportCallImportEvaluationCsv", -) -async def export_call_import_evaluation_csv( - call_import_id: UUID, - eval_id: UUID, - format: Literal["csv", "xlsx"] = Query( - "csv", - description=( - "Output format. ``csv`` returns a UTF-8 BOM CSV; ``xlsx`` " - "returns a native Excel workbook (single sheet)." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> StreamingResponse: - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - selected_metric_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) - # Include parent metric ids referenced in selected_metric_groups so - # the export shows a parent "Chosen Label" column next to its - # children's true/false columns. - lookup_ids: List[UUID] = list(selected_metric_ids) - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - for parent_str in groups_raw.keys(): - try: - pid = UUID(parent_str) - if pid not in lookup_ids: - lookup_ids.append(pid) - except (TypeError, ValueError): - continue - metrics = _metrics_for_ids(db, organization_id, lookup_ids) - metric_names = {str(metric.id): metric.name for metric in metrics} - metrics_by_id = {str(metric.id): metric for metric in metrics} - - # Two export-time modes depending on how the batch was uploaded: - # - # * Schema-driven (new): ``call_imports.schema_id`` is set, - # ``parameter_mapping`` records which CSV header fed each - # parameter, and ``raw_columns`` on each row is keyed by - # parameter NAME. Export headers are the parameter names. - # * Legacy (pre-schema): ``column_mapping`` / ``extra_columns`` / - # ``custom_column_mapping`` drive the columns and - # ``raw_columns`` is keyed by the original CSV header. - # - # We bucket entries into ``standard_export_headers`` (raw_columns - # key == export header) and ``custom_export`` (export header - # differs from the raw_columns key) so the row-projection loop - # below stays mode-agnostic. - standard_export_headers: List[str] = [] - custom_export: List[tuple[str, str]] = [] # [(export_header, raw_columns_key)] - - if call_import.schema_id is not None: - # Use the live schema parameter list for column ordering. Falls - # back to whatever's in ``parameter_mapping`` if the schema was - # deleted (defensive - the FK is ON DELETE RESTRICT, but tests - # / future cascades may still hit this branch). - from app.models.database import CallImportSchema as _ImportSchema - - schema_obj = ( - db.query(_ImportSchema) - .filter(_ImportSchema.id == call_import.schema_id) - .first() - ) - if schema_obj is not None: - params_sorted = sorted( - schema_obj.parameters, key=lambda p: p.ordering or 0 - ) - for param in params_sorted: - if param.name and param.name not in standard_export_headers: - standard_export_headers.append(param.name) - else: - for param_name in (call_import.parameter_mapping or {}).keys(): - if param_name and param_name not in standard_export_headers: - standard_export_headers.append(param_name) - else: - mapping = call_import.column_mapping or {} - mapped_headers = [ - mapping.get("external_call_id"), - mapping.get("transcript"), - mapping.get("recording_url"), - ] - for header in [*mapped_headers, *(call_import.extra_columns or [])]: - if ( - isinstance(header, str) - and header - and header not in standard_export_headers - ): - standard_export_headers.append(header) - - custom_mapping = call_import.custom_column_mapping or {} - if isinstance(custom_mapping, dict): - for name, csv_header in custom_mapping.items(): - if not isinstance(name, str) or not isinstance(csv_header, str): - continue - if not name or not csv_header: - continue - if name in standard_export_headers: - continue # would clobber a real column - custom_export.append((name, csv_header)) - - if ( - call_import.source_format == "audio" - and "conversation_id" not in standard_export_headers - ): - standard_export_headers.insert(0, "conversation_id") - - # Build the metric columns: each parent (if any) gets a value column - # and (when capture_rationale=true) a " - LLM Rationale" - # column. The per-child boolean columns are intentionally suppressed - # — categorization metrics now collapse to exactly two columns in - # the export, mirroring the in-app table. - child_ids_in_groups: set[str] = set() - for parent_str, child_strs in groups_raw.items(): - for child_str in child_strs: - if isinstance(child_str, str): - child_ids_in_groups.add(child_str) - - metric_headers: List[str] = [] - rationale_headers: Dict[str, str] = {} # metric_id_str -> rationale column name - seen_metric_ids: set[str] = set() - - def _add_metric_column(metric: Metric) -> None: - mid_str = str(metric.id) - if mid_str in seen_metric_ids: - return - # Skip any child whose parent is part of this run — the parent - # column above already shows the chosen child name as its - # value. - if mid_str in child_ids_in_groups: - return - seen_metric_ids.add(mid_str) - header = metric_names[mid_str] - metric_headers.append(header) - if bool(getattr(metric, "capture_rationale", False)): - rationale_header = f"{header} - LLM Rationale" - metric_headers.append(rationale_header) - rationale_headers[mid_str] = rationale_header - - for parent_str in groups_raw.keys(): - parent = metrics_by_id.get(parent_str) - if parent: - _add_metric_column(parent) - # Children of an in-run parent are deliberately not emitted — - # the ``child_ids_in_groups`` guard inside ``_add_metric_column`` - # is what enforces this. We still iterate the keys above (not - # ``.items()``) so the parent-only emission is explicit. - # Append anything left over (standalone metrics not in any group, or - # legacy runs without ``selected_metric_groups``). - for metric in metrics: - if metric.selection_mode and not metric.parent_metric_id: - continue # already handled above - if str(metric.id) in seen_metric_ids: - continue - _add_metric_column(metric) - - # Three new fixed columns surface the two transcript fields and the - # evaluation's transcript_source as live values pulled from the - # ``CallImportRow`` (not from the frozen ``raw_columns`` snapshot). - # The user can now compare "what was in the CSV" vs "what the - # diarisation worker produced" without round-tripping through the - # UI, and downstream tools can verify which transcript the metrics - # were computed against. - PRODUCTION_TRANSCRIPT_HEADER = "Production Transcript" - DIARISED_TRANSCRIPT_HEADER = "Diarised Transcript" - EVAL_SOURCE_HEADER = "Evaluated Transcript Source" - - fieldnames = [ - *standard_export_headers, - *[h for h, _ in custom_export], - PRODUCTION_TRANSCRIPT_HEADER, - DIARISED_TRANSCRIPT_HEADER, - EVAL_SOURCE_HEADER, - *metric_headers, - ] - - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - rows = sorted( - load_evaluation_row_pairs(db, eval_id), - key=lambda pair: int(pair[1].row_index or 0), - ) - else: - rows = ( - db.query(CallImportEvaluationRow, CallImportRow) - .join( - CallImportRow, - CallImportRow.id == CallImportEvaluationRow.call_import_row_id, - ) - .filter(CallImportEvaluationRow.evaluation_id == eval_id) - .order_by(CallImportRow.row_index.asc()) - .all() - ) - - def _project_rows() -> Iterator[Dict[str, str]]: - for eval_row, source_row in rows: - row_out: Dict[str, str] = {} - raw = ( - source_row.raw_columns - if isinstance(source_row.raw_columns, dict) - else {} - ) - for header in standard_export_headers: - value = raw.get(header) - if value is None and header == "conversation_id": - value = source_row.conversation_id - row_out[header] = "" if value is None else str(value) - for export_header, csv_header in custom_export: - value = raw.get(csv_header) - row_out[export_header] = "" if value is None else str(value) - - # Live transcripts pulled from the row, NOT from raw_columns, - # so re-diarised values are always reflected in the export. - # Both transcript columns are flattened to a single line so the - # spreadsheet cell doesn't balloon vertically — the in-app - # ``TranscriptView`` still has the DB copy with line breaks - # intact for chat-bubble rendering. - row_out[PRODUCTION_TRANSCRIPT_HEADER] = _flatten_transcript( - source_row.transcript - ) - row_out[DIARISED_TRANSCRIPT_HEADER] = _flatten_transcript( - source_row.diarised_transcript - ) - row_out[EVAL_SOURCE_HEADER] = _evaluated_transcript_source_label( - evaluation, - source_row, - ) - - scores = ( - eval_row.metric_scores - if isinstance(eval_row.metric_scores, dict) - else {} - ) - for metric in metrics: - metric_score = ( - scores.get(str(metric.id)) - if isinstance(scores, dict) - else None - ) - value = ( - metric_score.get("value") - if isinstance(metric_score, dict) - else None - ) - # Parent metrics (selection_mode set) render the chosen - # child name for single_choice or the ";"-joined list of - # true child names for multi_label. - if ( - metric.selection_mode - and not metric.parent_metric_id - and isinstance(metric_score, dict) - ): - if metric.selection_mode == "multi_label": - selected = metric_score.get("selected_child_names") - if isinstance(selected, list): - value = ";".join(str(s) for s in selected) - else: - value = ( - metric_score.get("chosen_child_name") - or metric_score.get("value") - ) - row_out[metric.name] = "" if value is None else str(value) - rationale_header = rationale_headers.get(str(metric.id)) - if rationale_header is not None: - rationale = ( - metric_score.get("rationale") - if isinstance(metric_score, dict) - else None - ) - row_out[rationale_header] = ( - "" if rationale is None else str(rationale) - ) - yield row_out - - base_filename = f"call-import-{call_import_id}-evaluation-{eval_id}" - - if format == "xlsx": - # xlsx is unicode-native (Hindi/Devanagari, emoji, etc.) so the - # UTF-8-BOM dance isn't needed here. ``write_only`` mode keeps - # peak memory bounded for large evaluations because openpyxl - # only buffers the current row. - try: - from openpyxl import Workbook # type: ignore - from openpyxl.cell import WriteOnlyCell # type: ignore - from openpyxl.styles import Font # type: ignore - except ImportError as exc: # pragma: no cover - exercised by pyproject lock - raise HTTPException( - status_code=500, - detail=( - "Excel export requires the 'openpyxl' package which is " - "not installed." - ), - ) from exc - - workbook = Workbook(write_only=True) - worksheet = workbook.create_sheet(title="Evaluation") - - bold_font = Font(bold=True) - header_cells = [] - for header in fieldnames: - cell = WriteOnlyCell(worksheet, value=header) - cell.font = bold_font - header_cells.append(cell) - worksheet.append(header_cells) - - for row_dict in _project_rows(): - worksheet.append([row_dict.get(h, "") for h in fieldnames]) - - buffer = io.BytesIO() - workbook.save(buffer) - xlsx_bytes = buffer.getvalue() - filename = f"{base_filename}.xlsx" - return StreamingResponse( - iter([xlsx_bytes]), - media_type=( - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ), - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) - - output = io.StringIO() - writer = csv.DictWriter(output, fieldnames=fieldnames, extrasaction="ignore") - writer.writeheader() - for row_dict in _project_rows(): - writer.writerow(row_dict) - - # Excel on Windows defaults to the system ANSI codepage (Windows-1252) - # when a CSV has no encoding marker, which turns UTF-8 Hindi/Devanagari - # / any non-ASCII text into mojibake (e.g. ``ठीक`` → ``ठीक``). - # A UTF-8 BOM tells Excel to switch to UTF-8 decoding and is silently - # skipped by every other UTF-8-aware reader (pandas, LibreOffice, - # Google Sheets, etc.), so the data round-trips correctly everywhere. - csv_text = output.getvalue() - # ``utf-8-sig`` adds the UTF-8 BOM so Excel on Windows decodes the file - # as UTF-8 instead of the system codepage. We also declare the same - # codec in the Content-Type header so well-behaved HTTP clients (incl. - # ``httpx`` / ``requests`` in our tests) strip the BOM during decode. - csv_bytes = csv_text.encode("utf-8-sig") - filename = f"{base_filename}.csv" - return StreamingResponse( - iter([csv_bytes]), - media_type="text/csv; charset=utf-8-sig", - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) - - -def _report_filename_slug(value: str) -> str: - slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.strip().lower()).strip("-") - return slug or "client" - - -def _report_branding_for_import_workspace( - db: Session, - organization_id: UUID, - workspace_id: UUID, - *, - internal_brand_image_id: Optional[str] = None, - external_brand_image_id: Optional[str] = None, -) -> tuple[dict[str, str] | list[str], Optional[str]]: - workspace = ( - db.query(Workspace) - .filter( - Workspace.id == workspace_id, - Workspace.organization_id == organization_id, - ) - .first() - ) - raw = workspace.report_branding if workspace and isinstance(workspace.report_branding, dict) else {} - images = raw.get("images") if isinstance(raw.get("images"), list) else [] - loaded_images: list[dict[str, str]] = [] - for item in images: - if not isinstance(item, dict) or not item.get("s3_key"): - continue - content_type = str(item.get("content_type") or "image/png") - try: - from app.services.storage.s3_service import s3_service - - image_bytes = s3_service.download_file_by_key(str(item["s3_key"])) - except Exception as exc: # noqa: BLE001 - logger.warning( - "Unable to load report branding image for workspace {}: {}", - workspace_id, - exc, - ) - continue - encoded = base64.b64encode(image_bytes).decode("ascii") - role = str(item.get("role") or "generic") - if role not in {"internal", "external", "generic"}: - role = "generic" - loaded_images.append( - { - "id": str(item.get("id") or ""), - "role": role, - "data_uri": f"data:{content_type};base64,{encoded}", - } - ) - - def _pick(role: str, selected_id: Optional[str]) -> Optional[str]: - if selected_id: - for loaded in loaded_images: - if loaded["id"] == selected_id: - return loaded["data_uri"] - for loaded in loaded_images: - if loaded["role"] == role: - return loaded["data_uri"] - return None - - logo_data_uris: dict[str, str] = {} - internal_uri = _pick("internal", internal_brand_image_id) - external_uri = _pick("external", external_brand_image_id) - if internal_uri: - logo_data_uris["internal"] = internal_uri - if external_uri: - logo_data_uris["external"] = external_uri - if ( - not logo_data_uris - and not internal_brand_image_id - and not external_brand_image_id - ): - # Backward compatibility for workspaces that only had a generic logo - # library before the two-slot report header existed. - generic_uris = [ - loaded["data_uri"] - for loaded in loaded_images - if loaded.get("data_uri") - ] - if generic_uris: - heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None - return generic_uris[:4], heading - heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None - return logo_data_uris, heading - - -def _display_metrics_for_pdf_report( - db: Session, - organization_id: UUID, - evaluation: CallImportEvaluation, -) -> list[Metric]: - selected_metric_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) - lookup_ids: List[UUID] = list(selected_metric_ids) - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - for parent_str in groups_raw.keys(): - try: - parent_id = UUID(parent_str) - except (TypeError, ValueError): - continue - if parent_id not in lookup_ids: - lookup_ids.append(parent_id) - - metrics = _metrics_for_ids(db, organization_id, lookup_ids) - child_ids_in_groups: set[str] = set() - for child_strs in groups_raw.values(): - if not isinstance(child_strs, list): - continue - child_ids_in_groups.update(str(child_id) for child_id in child_strs) - - metrics_by_id = {str(metric.id): metric for metric in metrics} - display: list[Metric] = [] - seen: set[str] = set() - - for parent_str in groups_raw.keys(): - parent = metrics_by_id.get(str(parent_str)) - if parent and str(parent.id) not in seen: - display.append(parent) - seen.add(str(parent.id)) - - for metric in metrics: - metric_id = str(metric.id) - if metric_id in seen or metric_id in child_ids_in_groups: - continue - if metric.selection_mode and not metric.parent_metric_id: - continue - display.append(metric) - seen.add(metric_id) - - return display - - -def _metrics_for_clustering( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], -) -> List[Metric]: - """All enabled quality metrics scored in this run, normalized for clustering. - - Hierarchical children are collapsed to their parent metric so cluster - groups render at the category level (e.g. ``AI reveal``) instead of the - child label level (e.g. ``Yes`` / ``No``). - """ - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - aggregate_metric_ids: List[UUID] = [] - for agg in aggregates: - if (agg.metric_category or "quality") == "user_insight": - continue - try: - aggregate_metric_ids.append(UUID(agg.metric_id)) - except (TypeError, ValueError): - continue - if not aggregate_metric_ids: - return [] - - aggregate_metrics = _metrics_for_ids( - db, evaluation.organization_id, aggregate_metric_ids - ) - by_id = {metric.id: metric for metric in aggregate_metrics} - - normalized_ids: List[UUID] = [] - seen: set[UUID] = set() - for metric_id in aggregate_metric_ids: - metric = by_id.get(metric_id) - target_id = ( - metric.parent_metric_id - if metric is not None and metric.parent_metric_id - else metric_id - ) - if target_id in seen: - continue - seen.add(target_id) - normalized_ids.append(target_id) - - metrics = _metrics_for_ids(db, evaluation.organization_id, normalized_ids) - return [ - metric - for metric in metrics - if getattr(metric, "enabled", True) and not _metric_is_user_insight(metric) - ] - - -def _metric_is_user_insight(metric: Metric) -> bool: - if (getattr(metric, "metric_category", "quality") or "quality") == "user_insight": - return True - text_value = " ".join( - str(part or "").lower() - for part in (getattr(metric, "name", ""), getattr(metric, "description", "")) - ) - normalized = text_value.replace("-", " ").replace("_", " ") - phrases = ( - "call context", - "caller context", - "product identification", - "out of scope", - "identity match", - "user identity", - "caller identity", - "frustration trigger", - "video call offer", - "video call reception", - ) - return any(phrase in normalized for phrase in phrases) - - -def _evaluation_rows_for_period( - db: Session, - evaluation_id: UUID, -) -> list[tuple[CallImportEvaluationRow, CallImportRow]]: - return ( - db.query(CallImportEvaluationRow, CallImportRow) - .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) - .filter(CallImportEvaluationRow.evaluation_id == evaluation_id) - .order_by(CallImportRow.row_index.asc()) - .all() - ) - - -def _baseline_candidate_evaluations( - db: Session, - organization_id: UUID, - workspace_id: UUID, - current_evaluation: CallImportEvaluation, - current_period_start: Optional[date], - *, - limit: int = 20, -) -> list[dict[str, Any]]: - candidates = ( - db.query(CallImportEvaluation, CallImport) - .join(CallImport, CallImport.id == CallImportEvaluation.call_import_id) - .filter( - CallImportEvaluation.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - CallImportEvaluation.id != current_evaluation.id, - CallImportEvaluation.status == "completed", - CallImportEvaluation.completed_rows > 0, - ) - .order_by(desc(CallImportEvaluation.created_at)) - .limit(limit * 3) - .all() - ) - items: list[dict[str, Any]] = [] - for candidate_eval, candidate_import in candidates: - rows = _evaluation_rows_for_period(db, candidate_eval.id) - period_start, period_end, period_label, period_display = _report_period_from_rows(rows) - if current_period_start and period_start and period_start >= current_period_start: - continue - dataset = ( - (candidate_import.dataset or "").strip() - or (candidate_import.original_filename or candidate_import.filename or "").strip() - or "Unknown dataset" - ) - evaluation_name = ( - (candidate_eval.name or "").strip() - or str(candidate_eval.id)[:8] - ) - items.append( - { - "evaluation_id": str(candidate_eval.id), - "name": evaluation_name, - "dataset": dataset, - "period_label": period_label, - "period_start": period_start, - "period_end": period_end, - "period_display": period_display, - "completed_rows": int(candidate_eval.completed_rows or 0), - "created_at": candidate_eval.created_at, - "is_default": False, - } - ) - if len(items) >= limit: - break - items.sort( - key=lambda item: ( - item["period_start"] or date.min, - item["created_at"] or datetime.min.replace(tzinfo=timezone.utc), - ), - reverse=True, - ) - if items: - items[0]["is_default"] = True - return items - - -def _resolve_baseline_evaluation( - db: Session, - organization_id: UUID, - workspace_id: UUID, - current_evaluation: CallImportEvaluation, - current_period_start: Optional[date], - baseline_evaluation_id: Optional[str], -) -> Optional[CallImportEvaluation]: - candidates = _baseline_candidate_evaluations( - db, - organization_id, - workspace_id, - current_evaluation, - current_period_start, - ) - allowed_ids = {item["evaluation_id"] for item in candidates} - if baseline_evaluation_id: - baseline_id = str(baseline_evaluation_id).strip() - if baseline_id not in allowed_ids: - raise HTTPException( - status_code=400, - detail="Selected baseline evaluation is not a valid prior run for this report.", - ) - return ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == UUID(baseline_id), - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not candidates: - return None - default_id = candidates[0]["evaluation_id"] - return ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == UUID(default_id), - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - - -def _benchmark_context_for_evaluation( - db: Session, - baseline_evaluation: Optional[CallImportEvaluation], -) -> Optional[dict[str, str]]: - if baseline_evaluation is None: - return None - baseline_import = ( - db.query(CallImport) - .filter(CallImport.id == baseline_evaluation.call_import_id) - .first() - ) - rows = _evaluation_rows_for_period(db, baseline_evaluation.id) - period_start, _period_end, period_label, _period_display = _report_period_from_rows(rows) - dataset = ( - (baseline_import.dataset or "").strip() - if baseline_import and baseline_import.dataset - else None - ) - filename = ( - (baseline_import.original_filename or baseline_import.filename or "").strip() - if baseline_import - else None - ) - evaluation_label = ( - (baseline_evaluation.name or "").strip() - if baseline_evaluation.name - else str(baseline_evaluation.id)[:8] - ) - period = period_label or ( - period_start.isoformat() if period_start else "previous report" - ) - return { - "dataset": dataset or filename or "Unknown dataset", - "evaluation": evaluation_label, - "evaluation_id": str(baseline_evaluation.id), - "period": period, - } - - -def _period_deltas_from_evaluation( - db: Session, - baseline_evaluation: CallImportEvaluation, - current_metric_aggregates: list[dict[str, Any]], - current_evaluation: CallImportEvaluation, - current_eval_rows: List[CallImportEvaluationRow], -) -> dict[str, dict[str, str]]: - baseline_rows = _evaluation_rows_for_period(db, baseline_evaluation.id) - baseline_eval_rows = [eval_row for eval_row, _source_row in baseline_rows] - baseline_aggregate_models = _compute_metric_aggregates( - db, - baseline_evaluation, - baseline_eval_rows, - ) - baseline_metric_aggregates = [ - _aggregate_to_dict(aggregate) for aggregate in baseline_aggregate_models - ] - _metrics, _aggs, policies, _source, _child_map = _clustering_context( - db, current_evaluation, current_eval_rows - ) - metric_by_id = {str(m.id): m for m in _metrics} - current_by_id = { - str(item.get("metric_id")): item for item in current_metric_aggregates - } - previous_by_id = { - str(item.get("metric_id")): item - for item in baseline_metric_aggregates - if isinstance(item, dict) - } - deltas: dict[str, dict[str, str]] = {} - for metric_id, current in current_by_id.items(): - metric = metric_by_id.get(metric_id) - policy = policies.get(metric_id) - previous_raw = previous_by_id.get(metric_id) - if metric is None or policy is None: - deltas[metric_id] = { - "label": "No previous-week baseline", - "detail": "No comparable prior report snapshot was found.", - } - continue - current_pct = failure_rate_percent_from_rows( - current_eval_rows, metric, policy - ) - previous_pct = failure_rate_percent_from_rows( - baseline_eval_rows, metric, policy - ) - if current_pct is None or previous_pct is None: - current_pct = current_pct or _aggregate_primary_percent(current, policy) - previous_pct = ( - previous_pct or _aggregate_primary_percent(previous_raw, policy) - if previous_raw - else None - ) - if current_pct is None or previous_pct is None: - deltas[metric_id] = { - "label": "No previous-week baseline", - "detail": "No comparable prior report snapshot was found.", - } - continue - delta = current_pct - previous_pct - sign = "+" if delta >= 0 else "" - deltas[metric_id] = { - "label": f"{sign}{delta:.1f} pp", - "detail": ( - f"Current report {current_pct:.1f}% vs previous report " - f"{previous_pct:.1f}%" - ), - } - return deltas - - -_DELTA_EXPLANATION_SYSTEM_PROMPT = ( - "You are a senior conversation-analytics reviewer. You will receive " - "week-over-week metric failure-rate deltas plus reconciled failure " - "cluster context per metric.\n\n" - "Return STRICT JSON only:\n" - "{\n" - ' "explanations": {"": "<1-2 sentence explanation of why the delta likely occurred>"}\n' - "}\n\n" - "Constraints:\n" - "- Only include metrics supplied in the prompt.\n" - "- Cluster labels are generated independently each run and are NOT stable " - "IDs. Never compare an unmatched current label to 0% baseline.\n" - "- Use matched_theme_shifts for label-aligned comparisons, " - "gap_label_shifts for structural shifts, and new_themes_current_period " - "for themes that emerged without a baseline match.\n" - "- If reconciliation is uncertain, explain using the numeric delta and " - "gap_label_shifts only.\n" - "- Keep each explanation to 1-2 short sentences (~220 chars).\n" - "- Vendor-safe, factual language; no markdown." -) - - -def _period_delta_explanation_cache_key( - baseline_evaluation_id: UUID, - *, - completed_rows: int, - baseline_completed_rows: int, -) -> str: - return ( - f"{baseline_evaluation_id}:{completed_rows}:" - f"{baseline_completed_rows}:reconciled-v2" - ) - - -def _normalize_cluster_label(label: str) -> str: - return re.sub(r"[^a-z0-9]+", " ", (label or "").lower()).strip() - - -_CLUSTER_LABEL_STOPWORDS = frozenset( - { - "a", - "an", - "the", - "and", - "or", - "during", - "while", - "with", - "for", - "from", - "into", - "general", - "user", - "bot", - "agent", - } -) - - -def _cluster_label_tokens(label: str) -> set[str]: - return { - token - for token in _normalize_cluster_label(label).split() - if token and token not in _CLUSTER_LABEL_STOPWORDS and len(token) > 2 - } - - -def _cluster_label_similarity(left: str, right: str) -> float: - tokens_left = _cluster_label_tokens(left) - tokens_right = _cluster_label_tokens(right) - if not tokens_left or not tokens_right: - return 0.0 - intersection = tokens_left & tokens_right - if not intersection: - return 0.0 - union = tokens_left | tokens_right - jaccard = len(intersection) / len(union) - smaller = tokens_left if len(tokens_left) <= len(tokens_right) else tokens_right - overlap_ratio = len(intersection) / len(smaller) - return max(jaccard, overlap_ratio * 0.85) - - -def _group_clusters_by_gap_label( - clusters: list[dict[str, Any]], -) -> dict[str, list[dict[str, Any]]]: - grouped: dict[str, list[dict[str, Any]]] = {} - for cluster in clusters: - gap_label = str(cluster.get("gap_label") or "UNKNOWN") - grouped.setdefault(gap_label, []).append(cluster) - return grouped - - -def _append_matched_cluster_pair( - matched: list[dict[str, Any]], - current: dict[str, Any], - baseline: dict[str, Any], - *, - match_confidence: float, - match_method: str, -) -> None: - matched.append( - { - "current_label": current.get("label"), - "baseline_label": baseline.get("label"), - "gap_label": current.get("gap_label") or baseline.get("gap_label"), - "current_share_pct": current.get("share_pct"), - "baseline_share_pct": baseline.get("share_pct"), - "share_delta_pp": round( - float(current.get("share_pct") or 0.0) - - float(baseline.get("share_pct") or 0.0), - 1, - ), - "match_confidence": round(match_confidence, 2), - "match_method": match_method, - } - ) - - -def _aggregate_share_by_gap_label( - clusters: list[dict[str, Any]], -) -> dict[str, float]: - totals: dict[str, float] = {} - for cluster in clusters: - gap_label = str(cluster.get("gap_label") or "UNKNOWN") - totals[gap_label] = totals.get(gap_label, 0.0) + float( - cluster.get("share_pct") or 0.0 - ) - return {gap: round(share, 1) for gap, share in totals.items()} - - -def _reconcile_cluster_periods( - current_clusters: list[dict[str, Any]], - baseline_clusters: list[dict[str, Any]], - *, - similarity_threshold: float = 0.35, -) -> dict[str, Any]: - """Align independently-generated cluster labels before delta explanation.""" - matched: list[dict[str, Any]] = [] - current_unmatched = list(current_clusters) - remaining_baseline = list(baseline_clusters) - - current_by_gap = _group_clusters_by_gap_label(current_unmatched) - baseline_by_gap = _group_clusters_by_gap_label(remaining_baseline) - for gap_label in list(current_by_gap): - current_group = current_by_gap.get(gap_label) or [] - baseline_group = baseline_by_gap.get(gap_label) or [] - if len(current_group) != 1 or len(baseline_group) != 1: - continue - current = current_group[0] - baseline = baseline_group[0] - _append_matched_cluster_pair( - matched, - current, - baseline, - match_confidence=0.75, - match_method="single_cluster_per_gap_label", - ) - current_unmatched.remove(current) - remaining_baseline.remove(baseline) - current_by_gap[gap_label] = [] - baseline_by_gap[gap_label] = [] - - for current in list(current_unmatched): - best_idx: Optional[int] = None - best_score = 0.0 - for idx, baseline in enumerate(remaining_baseline): - score = _cluster_label_similarity( - str(current.get("label") or ""), - str(baseline.get("label") or ""), - ) - if current.get("gap_label") == baseline.get("gap_label"): - score += 0.1 - if score > best_score: - best_score = score - best_idx = idx - - if best_idx is not None and best_score >= similarity_threshold: - baseline = remaining_baseline.pop(best_idx) - _append_matched_cluster_pair( - matched, - current, - baseline, - match_confidence=best_score, - match_method="label_similarity", - ) - - matched_current_labels = { - str(item.get("current_label") or "") for item in matched - } - matched_baseline_labels = { - str(item.get("baseline_label") or "") for item in matched - } - current_unmatched = [ - cluster - for cluster in current_clusters - if str(cluster.get("label") or "") not in matched_current_labels - ] - remaining_baseline = [ - cluster - for cluster in baseline_clusters - if str(cluster.get("label") or "") not in matched_baseline_labels - ] - - new_themes = [ - { - "label": cluster.get("label"), - "gap_label": cluster.get("gap_label"), - "share_pct": cluster.get("share_pct"), - "note": "New theme in current period (no close baseline match).", - } - for cluster in current_unmatched - ] - - retired_themes = [ - { - "label": baseline.get("label"), - "gap_label": baseline.get("gap_label"), - "share_pct": baseline.get("share_pct"), - "note": "Theme present in baseline only (retired or renamed).", - } - for baseline in remaining_baseline - ] - - current_gap = _aggregate_share_by_gap_label(current_clusters) - baseline_gap = _aggregate_share_by_gap_label(baseline_clusters) - gap_label_shifts: dict[str, dict[str, float]] = {} - for gap_label in set(current_gap) | set(baseline_gap): - current_share = current_gap.get(gap_label, 0.0) - baseline_share = baseline_gap.get(gap_label, 0.0) - if abs(current_share - baseline_share) >= 0.5: - gap_label_shifts[gap_label] = { - "current_share_pct": current_share, - "baseline_share_pct": baseline_share, - "share_delta_pp": round(current_share - baseline_share, 1), - } - - return { - "matched_theme_shifts": matched, - "new_themes_current_period": new_themes, - "retired_themes_baseline_period": retired_themes, - "gap_label_shifts": gap_label_shifts, - "reconciliation_note": ( - "Cluster labels are generated independently each run and may " - "rename the same failure mode. Do not treat unmatched current " - "labels as 0% in the baseline period." - ), - } - - -def _load_period_delta_explanations_cache( - evaluation: CallImportEvaluation, - cache_key: str, -) -> Optional[dict[str, str]]: - raw = getattr(evaluation, "period_delta_explanations", None) - if not isinstance(raw, dict): - return None - entry = raw.get(cache_key) - if not isinstance(entry, dict): - return None - explanations_raw = entry.get("explanations") - if not isinstance(explanations_raw, dict): - return None - return { - str(metric_id): str(why).strip() - for metric_id, why in explanations_raw.items() - if str(metric_id).strip() and isinstance(why, str) and why.strip() - } - - -def _save_period_delta_explanations_cache( - db: Session, - evaluation: CallImportEvaluation, - cache_key: str, - explanations: dict[str, str], -) -> None: - raw = evaluation.period_delta_explanations - if not isinstance(raw, dict): - raw = {} - updated = dict(raw) - updated[cache_key] = { - "explanations": explanations, - "generated_at": datetime.now(timezone.utc).isoformat(), - } - evaluation.period_delta_explanations = updated - flag_modified(evaluation, "period_delta_explanations") - db.commit() - - -def _cluster_summary_for_metric( - state: Optional[EvaluationMetricClustersState], - metric_id: str, -) -> list[dict[str, Any]]: - if state is None or state.status != "completed": - return [] - for group in state.groups: - if str(group.metric_id) != metric_id: - continue - return [ - { - "label": cluster.label, - "gap_label": cluster.gap_label, - "share_pct": round(cluster.share_pct, 1), - "count": cluster.count, - } - for cluster in group.clusters[:5] - ] - return [] - - -def _merge_delta_why( - raw_deltas: dict[str, dict[str, str]], - explanations: dict[str, str], -) -> dict[str, dict[str, str]]: - if not explanations: - return raw_deltas - merged: dict[str, dict[str, str]] = {} - for metric_id, delta in raw_deltas.items(): - updated = dict(delta) - why = explanations.get(metric_id) - if why: - updated["why"] = why - merged[metric_id] = updated - return merged - - -def _explain_period_deltas( - db: Session, - organization_id: UUID, - evaluation: CallImportEvaluation, - baseline_evaluation: CallImportEvaluation, - raw_deltas: dict[str, dict[str, str]], - *, - min_delta_pp: float = 0.5, -) -> dict[str, dict[str, str]]: - """Attach ``why`` explanations to period deltas using cached LLM output.""" - if not raw_deltas: - return raw_deltas - - cache_key = _period_delta_explanation_cache_key( - baseline_evaluation.id, - completed_rows=evaluation.completed_rows, - baseline_completed_rows=baseline_evaluation.completed_rows, - ) - cached = _load_period_delta_explanations_cache(evaluation, cache_key) - if cached is not None: - return _merge_delta_why(raw_deltas, cached) - - current_clusters = _metric_clusters_payload(evaluation) - baseline_clusters = _metric_clusters_payload(baseline_evaluation) - metrics_for_prompt: list[dict[str, Any]] = [] - for metric_id, delta in raw_deltas.items(): - label = delta.get("label") or "" - if "No previous-week baseline" in label: - continue - match = re.search(r"([+-]?\d+(?:\.\d+)?)\s*pp", label) - if match and abs(float(match.group(1))) < min_delta_pp: - continue - current_summary = _cluster_summary_for_metric(current_clusters, metric_id) - baseline_summary = _cluster_summary_for_metric(baseline_clusters, metric_id) - if not current_summary and not baseline_summary: - continue - cluster_reconciliation = _reconcile_cluster_periods( - current_summary, - baseline_summary, - ) - metrics_for_prompt.append( - { - "metric_id": metric_id, - "delta_label": label, - "delta_detail": delta.get("detail") or "", - "cluster_reconciliation": cluster_reconciliation, - } - ) - - if not metrics_for_prompt: - return raw_deltas - - provider_hint: Optional[str] = None - model_hint: Optional[str] = None - tldr_raw = evaluation.tldr_summary - if isinstance(tldr_raw, dict): - if isinstance(tldr_raw.get("provider"), str): - provider_hint = tldr_raw["provider"] - if isinstance(tldr_raw.get("model"), str): - model_hint = tldr_raw["model"] - - from app.services.ai.llm_resolver import get_llm_provider_and_model - from app.services.call_import_user_insights import _call_llm, _parse_json_object - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, provider_hint, model_hint - ) - try: - text = _call_llm( - db, - organization_id, - provider_enum, - model_str, - [ - {"role": "system", "content": _DELTA_EXPLANATION_SYSTEM_PROMPT}, - { - "role": "user", - "content": json.dumps( - {"metrics": metrics_for_prompt}, - ensure_ascii=False, - default=str, - ), - }, - ], - temperature=0.3, - max_tokens=900, - ) - except Exception as exc: - logger.warning("[PeriodDeltaExplain] LLM call failed: {}", exc) - return raw_deltas - - parsed = _parse_json_object(text) - explanations_raw = parsed.get("explanations") - explanations: dict[str, str] = {} - if isinstance(explanations_raw, dict): - for metric_id, why in explanations_raw.items(): - if isinstance(why, str) and why.strip(): - explanations[str(metric_id)] = why.strip() - - if explanations: - _save_period_delta_explanations_cache( - db, evaluation, cache_key, explanations - ) - return _merge_delta_why(raw_deltas, explanations) - - -def _period_deltas_with_explanations( - db: Session, - organization_id: UUID, - evaluation: CallImportEvaluation, - baseline_evaluation: CallImportEvaluation, - raw_deltas: dict[str, dict[str, str]], -) -> dict[str, dict[str, str]]: - return _explain_period_deltas( - db, - organization_id, - evaluation, - baseline_evaluation, - raw_deltas, - ) - - -def _benchmark_context_for_snapshot( - db: Session, - previous_snapshot: Optional[CallImportEvaluationReportSnapshot], -) -> Optional[dict[str, str]]: - if previous_snapshot is None: - return None - previous_import = ( - db.query(CallImport) - .filter(CallImport.id == previous_snapshot.call_import_id) - .first() - ) - previous_eval = ( - db.query(CallImportEvaluation) - .filter(CallImportEvaluation.id == previous_snapshot.evaluation_id) - .first() - ) - dataset = ( - (previous_import.dataset or "").strip() - if previous_import and previous_import.dataset - else None - ) - filename = ( - (previous_import.original_filename or previous_import.filename or "").strip() - if previous_import - else None - ) - evaluation_label = ( - (previous_eval.name or "").strip() - if previous_eval and previous_eval.name - else str(previous_snapshot.evaluation_id)[:8] - ) - period = previous_snapshot.period_label or ( - previous_snapshot.period_start.isoformat() - if previous_snapshot.period_start - else "previous report" - ) - return { - "dataset": dataset or filename or "Unknown dataset", - "evaluation": evaluation_label, - "evaluation_id": str(previous_snapshot.evaluation_id), - "period": period, - } - - -def _clamp_prose_to_sentences( - text: str, - *, - max_sentences: int = 3, - max_chars: int = 300, -) -> str: - """Keep concise audit/TLDR prose within sentence and character limits.""" - cleaned = (text or "").strip() - if not cleaned: - return cleaned - cleaned = re.sub(r"\s*\n+\s*", " ", cleaned).strip() - sentences = [ - sentence.strip() - for sentence in re.split(r"(?<=[.!?])\s+", cleaned) - if sentence.strip() - ] - if sentences: - result = " ".join(sentences[:max_sentences]).strip() - else: - result = cleaned - if len(result) > max_chars: - trimmed = result[: max_chars - 3].rsplit(" ", 1)[0].rstrip(".,;:") - result = f"{trimmed}..." if trimmed else result[:max_chars] - return result - - -def _audit_summary_text_from_tldr( - summary: Optional[EvaluationTldrSummary], -) -> Optional[str]: - if summary is None: - return None - narrative = _clamp_prose_to_sentences(summary.narrative.strip()) - return narrative or None - - -def _metric_insights_from_tldr( - summary: Optional[EvaluationTldrSummary], -) -> dict[str, str]: - if summary is None: - return {} - return { - str(metric_id): insight.strip() - for metric_id, insight in summary.metric_insights.items() - if str(metric_id).strip() and insight.strip() - } - - -def _report_period_from_rows( - rows: list[tuple[CallImportEvaluationRow, CallImportRow]], -) -> tuple[Optional[date], Optional[date], Optional[str], str]: - dates = [ - source_row.recording_date - for eval_row, source_row in rows - if eval_row.status == "completed" and source_row.recording_date - ] - if not dates: - return None, None, None, "Not specified" - start = min(dates) - end = max(dates) - week_anchor = max(dates) - week_start = week_anchor - timedelta(days=week_anchor.weekday()) - week_end = week_start + timedelta(days=6) - iso_year, iso_week, _ = week_anchor.isocalendar() - label = f"{iso_year}-W{iso_week:02d}" - if week_start.year == week_end.year: - week_range = f"{week_start.strftime('%b %d')}–{week_end.strftime('%b %d, %Y')}" - else: - week_range = ( - f"{week_start.strftime('%b %d, %Y')}–{week_end.strftime('%b %d, %Y')}" - ) - display = f"W{iso_week:02d} · {week_range}" - return start, end, label, display - - -def _aggregate_to_dict(aggregate: CallImportMetricAggregate) -> dict[str, Any]: - if hasattr(aggregate, "model_dump"): - return aggregate.model_dump(mode="json") - return aggregate.dict() - - -def _aggregate_primary_percent( - raw: dict[str, Any], - policy: Optional[MetricFailurePolicy] = None, -) -> Optional[float]: - return aggregate_primary_percent(raw, policy) - - -def _child_names_by_parent( - db: Session, - organization_id: UUID, - parent_metric_ids: Sequence[UUID], -) -> Dict[str, List[str]]: - if not parent_metric_ids: - return {} - children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id.in_(list(parent_metric_ids)), - ) - .all() - ) - out: Dict[str, List[str]] = {} - for child in children: - pid = str(child.parent_metric_id) - out.setdefault(pid, []).append(child.name) - return out - - -def _clustering_context( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], -) -> Tuple[ - List[Metric], - List[CallImportMetricAggregate], - Dict[str, MetricFailurePolicy], - Literal["inferred", "user"], - Dict[str, List[str]], -]: - metrics = _metrics_for_clustering(db, evaluation, eval_rows) - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) and not getattr(m, "parent_metric_id", None) - ] - child_names_by_parent = _child_names_by_parent( - db, evaluation.organization_id, parent_ids - ) - policies, source = effective_policies( - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - return metrics, aggregates, policies, source, child_names_by_parent - - -def _period_deltas_from_aggregates( - previous_metric_aggregates: list[dict[str, Any]], - current_metric_aggregates: list[dict[str, Any]], - policies: Optional[Dict[str, MetricFailurePolicy]] = None, -) -> dict[str, dict[str, str]]: - current_by_id = {str(item.get("metric_id")): item for item in current_metric_aggregates} - previous_by_id = { - str(item.get("metric_id")): item - for item in previous_metric_aggregates - if isinstance(item, dict) - } - deltas: dict[str, dict[str, str]] = {} - for metric_id, current in current_by_id.items(): - previous_raw = previous_by_id.get(metric_id) - policy = (policies or {}).get(metric_id) - current_pct = _aggregate_primary_percent(current, policy) - previous_pct = ( - _aggregate_primary_percent(previous_raw, policy) - if previous_raw - else None - ) - if current_pct is None or previous_pct is None: - deltas[metric_id] = { - "label": "No previous-week baseline", - "detail": "No comparable prior report snapshot was found.", - } - continue - delta = current_pct - previous_pct - sign = "+" if delta >= 0 else "" - deltas[metric_id] = { - "label": f"{sign}{delta:.1f} pp", - "detail": f"Current report {current_pct:.1f}% vs previous report {previous_pct:.1f}%", - } - return deltas - - -def _period_deltas_from_snapshot( - previous: Optional[CallImportEvaluationReportSnapshot], - current_metric_aggregates: list[dict[str, Any]], -) -> dict[str, dict[str, str]]: - previous_items = ( - previous.metric_aggregates - if previous and isinstance(previous.metric_aggregates, list) - else [] - ) - return _period_deltas_from_aggregates(previous_items, current_metric_aggregates) - - -def _sample_evidence_for_metrics( - rows: list[tuple[CallImportEvaluationRow, CallImportRow]], - metric_ids: set[str], -) -> dict[str, list[dict[str, str]]]: - samples: dict[str, list[dict[str, str]]] = {metric_id: [] for metric_id in metric_ids} - for eval_row, source_row in rows: - scores = eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} - for metric_id in metric_ids: - if len(samples.get(metric_id, [])) >= 4: - continue - score = scores.get(metric_id) - if not isinstance(score, dict): - continue - rationale = score.get("rationale") - transcript = source_row.diarised_transcript or source_row.transcript or "" - quote = rationale if isinstance(rationale, str) and rationale.strip() else transcript[:350] - if quote: - samples.setdefault(metric_id, []).append( - { - "conversation_id": source_row.conversation_id, - "quote": str(quote).strip()[:500], - } - ) - return samples - - -def _fallback_report_narrative( - insight_aggregates: list[dict[str, Any]], - evidence_samples: dict[str, list[dict[str, str]]], -) -> dict[str, Any]: - observations: dict[str, str] = {} - evidence: dict[str, dict[str, str]] = {} - design_notes: list[str] = [] - for aggregate in insight_aggregates: - metric_id = str(aggregate.get("metric_id") or "") - name = str(aggregate.get("metric_name") or "Insight") - counts = aggregate.get("value_counts") if isinstance(aggregate.get("value_counts"), list) else [] - if counts: - top = counts[0] - total = int(aggregate.get("count") or 0) or sum( - int(item.get("count") or 0) for item in counts if isinstance(item, dict) - ) - pct = (int(top.get("count") or 0) / total) * 100 if total else 0 - observations[metric_id] = ( - f"{top.get('label')} is the dominant {name.lower()} category at {pct:.1f}% of classified calls." - ) - design_notes.append( - f"{name}: {top.get('label')} is the largest segment and should be reviewed for workflow or prompt improvements." - ) - sample = (evidence_samples.get(metric_id) or [{}])[0] - if sample: - evidence[metric_id] = sample - return { - "observations": observations, - "evidence": evidence, - "design_notes": design_notes[:7], - "audit_summary": None, - } - - -def _generate_report_narrative( - db: Session, - organization_id: UUID, - *, - metric_aggregates: list[dict[str, Any]], - insight_aggregates: list[dict[str, Any]], - period_delta_by_metric: dict[str, dict[str, str]], - evidence_samples: dict[str, list[dict[str, str]]], - report_config: dict[str, Any], -) -> dict[str, Any]: - if not insight_aggregates: - return {"observations": {}, "evidence": {}, "design_notes": [], "audit_summary": None} - try: - from app.services.ai.llm_resolver import get_llm_provider_and_model - from app.services.ai.llm_service import llm_service - - provider_enum, model_str = get_llm_provider_and_model(organization_id, db, None, None) - prompt = ( - "You are writing a vendor-safe external call quality audit report. " - "Return strict JSON with keys observations (object keyed by metric_id), " - "evidence (object keyed by metric_id with conversation_id and quote), " - "design_notes (array of concise numbered-note strings), and audit_summary (string). " - "Use only the supplied aggregates and evidence samples.\n\n" - + json.dumps( - { - "metric_aggregates": metric_aggregates[:30], - "insight_aggregates": insight_aggregates, - "period_deltas": period_delta_by_metric, - "evidence_samples": evidence_samples, - "report_config": report_config, - }, - default=str, - ) - ) - llm_result = llm_service.generate_response( - messages=[ - {"role": "system", "content": "Return JSON only. No markdown."}, - {"role": "user", "content": prompt}, - ], - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - temperature=0.2, - max_tokens=1200, - ) - parsed = json.loads(str(llm_result.content or "{}")) - if isinstance(parsed, dict): - fallback = _fallback_report_narrative(insight_aggregates, evidence_samples) - return { - "observations": parsed.get("observations") or fallback["observations"], - "evidence": parsed.get("evidence") or fallback["evidence"], - "design_notes": parsed.get("design_notes") or fallback["design_notes"], - "audit_summary": parsed.get("audit_summary") or fallback["audit_summary"], - } - except Exception as exc: # noqa: BLE001 - logger.warning("Report narrative LLM generation fell back to deterministic text: {}", exc) - return _fallback_report_narrative(insight_aggregates, evidence_samples) - - -@router.get( - "/{eval_id}/baseline-candidates", - response_model=CallImportEvaluationBaselineCandidatesResponse, - operation_id="listCallImportEvaluationBaselineCandidates", -) -async def list_call_import_evaluation_baseline_candidates( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationBaselineCandidatesResponse: - del api_key - call_import = _require_import(db, call_import_id, organization_id) - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - rows = _evaluation_rows_for_period(db, evaluation.id) - period_start, _period_end, _derived_period_label, _period_display = _report_period_from_rows( - rows - ) - candidates = _baseline_candidate_evaluations( - db, - organization_id, - call_import.workspace_id, - evaluation, - period_start, - ) - default_evaluation_id = next( - (item["evaluation_id"] for item in candidates if item.get("is_default")), - None, - ) - return CallImportEvaluationBaselineCandidatesResponse( - items=[CallImportEvaluationBaselineCandidate(**item) for item in candidates], - default_evaluation_id=default_evaluation_id, - ) - - -@router.post( - "/{eval_id}/pdf-report", - operation_id="generateCallImportEvaluationPdfReport", - dependencies=[Depends(require_call_import_capability(REPORTS_GENERATE))], -) -async def generate_call_import_evaluation_pdf_report( - call_import_id: UUID, - eval_id: UUID, - payload: CallImportEvaluationPdfReportRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> StreamingResponse: - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - is_internal = payload.report_type == "internal" - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - rows = sorted( - load_evaluation_row_pairs(db, eval_id), - key=lambda pair: int(pair[1].row_index or 0), - ) - else: - rows = ( - db.query(CallImportEvaluationRow, CallImportRow) - .join( - CallImportRow, - CallImportRow.id == CallImportEvaluationRow.call_import_row_id, - ) - .filter(CallImportEvaluationRow.evaluation_id == eval_id) - .order_by(CallImportRow.row_index.asc()) - .all() - ) - report_config = payload.report_config if isinstance(payload.report_config, dict) else {} - metrics = _display_metrics_for_pdf_report(db, organization_id, evaluation) - configured_quality_ids = { - str(item) - for item in report_config.get("quality_metric_ids", []) - if item - } - configured_insight_ids = { - str(item.get("metric_id") or item) - for item in report_config.get("insights", []) - if item - } - if configured_quality_ids or configured_insight_ids: - allowed_ids = configured_quality_ids | configured_insight_ids - metrics = [metric for metric in metrics if str(metric.id) in allowed_ids] - - eval_rows = [eval_row for eval_row, _source_row in rows] - aggregate_models = _compute_metric_aggregates(db, evaluation, eval_rows) - selected_report_metric_ids = {str(metric.id) for metric in metrics} - aggregate_dicts = [ - _aggregate_to_dict(aggregate) - for aggregate in aggregate_models - if aggregate.metric_id in selected_report_metric_ids - ] - insight_metric_ids = { - str(metric.id) - for metric in metrics - if _metric_is_user_insight(metric) - } - metric_aggregates = [ - item for item in aggregate_dicts if str(item.get("metric_id")) not in insight_metric_ids - ] - insight_aggregates = [ - item for item in aggregate_dicts if str(item.get("metric_id")) in insight_metric_ids - ] - period_start, period_end, derived_period_label, period_display = _report_period_from_rows(rows) - period_label = (payload.period_label or derived_period_label or "").strip() or None - include_period_delta = ( - payload.include_period_delta or payload.include_weekly_delta - ) - previous_snapshot = None - period_delta_by_metric: dict[str, dict[str, str]] = {} - baseline_evaluation: Optional[CallImportEvaluation] = None - if include_period_delta and period_start: - baseline_evaluation = _resolve_baseline_evaluation( - db, - organization_id, - call_import.workspace_id, - evaluation, - period_start, - payload.baseline_evaluation_id, - ) - if baseline_evaluation: - period_delta_by_metric = _period_deltas_from_evaluation( - db, - baseline_evaluation, - metric_aggregates, - evaluation, - [eval_row for eval_row, _ in rows], - ) - period_delta_by_metric = _period_deltas_with_explanations( - db, - organization_id, - evaluation, - baseline_evaluation, - period_delta_by_metric, - ) - benchmark_context = _benchmark_context_for_evaluation(db, baseline_evaluation) - evidence_samples = _sample_evidence_for_metrics(rows, insight_metric_ids) - cached_tldr_summary = _tldr_summary_payload(evaluation) - cached_user_insights = _user_insights_payload(evaluation) - cached_metric_clusters = _metric_clusters_payload(evaluation) - cached_prompt_improvements = _prompt_improvements_payload(evaluation) - generated_insights_for_pdf = _selected_generated_user_insights( - cached_user_insights, - report_config, - ) - metric_clusters_for_pdf = _selected_metric_clusters_for_pdf( - cached_metric_clusters, - report_config, - ) - prompt_improvements_for_pdf = _selected_prompt_improvements_for_pdf( - cached_prompt_improvements, - report_config, - ) - narrative = _generate_report_narrative( - db, - organization_id, - metric_aggregates=metric_aggregates, - insight_aggregates=insight_aggregates if is_internal else [], - period_delta_by_metric=period_delta_by_metric, - evidence_samples=evidence_samples if is_internal else {}, - report_config=report_config, - ) - - generated_at = datetime.now(timezone.utc) - branding_images, custom_heading = _report_branding_for_import_workspace( - db, - organization_id, - call_import.workspace_id, - internal_brand_image_id=payload.internal_brand_image_id, - external_brand_image_id=payload.external_brand_image_id, - ) - eval_row_list = [eval_row for eval_row, _ in rows] - pdf_aggregates = _compute_metric_aggregates(db, evaluation, eval_row_list) - pdf_parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) - and not getattr(m, "parent_metric_id", None) - ] - pdf_child_map = _child_names_by_parent( - db, evaluation.organization_id, pdf_parent_ids - ) - failure_policies_for_pdf, _fp_source = effective_policies( - evaluation, - metrics, - pdf_aggregates, - child_names_by_parent=pdf_child_map, - ) - try: - pdf_started = datetime.now(timezone.utc) - pdf_bytes = await asyncio.to_thread( - call_import_evaluation_pdf_report_service.render_pdf, - vendor_name=payload.vendor_name, - call_import=call_import, - evaluation=evaluation, - metrics=metrics, - rows=rows, - failure_policies=failure_policies_for_pdf, - generated_at=generated_at, - internal=is_internal, - logo_data_uris=branding_images, - custom_heading=custom_heading, - include_weekly_delta=include_period_delta, - period_delta_by_metric=period_delta_by_metric, - use_case=payload.use_case, - period_display=period_display, - total_metric_count=db.query(Metric) - .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) - .count(), - report_config=report_config, - narrative=narrative, - audit_summary=_audit_summary_text_from_tldr(cached_tldr_summary), - metric_insights=_metric_insights_from_tldr(cached_tldr_summary), - benchmark_context=benchmark_context, - generated_user_insights=generated_insights_for_pdf, - user_insights_overview=( - cached_user_insights.overview if cached_user_insights else None - ), - metric_clusters=metric_clusters_for_pdf, - metric_clusters_overview=( - cached_metric_clusters.overview if cached_metric_clusters else None - ), - prompt_improvements=prompt_improvements_for_pdf, - platform_base_url=payload.platform_base_url, - ) - logger.info( - "PDF report render finished in {:.1f}s for evaluation {}", - (datetime.now(timezone.utc) - pdf_started).total_seconds(), - eval_id, - ) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to generate PDF report for call import {} evaluation {}", - call_import_id, - eval_id, - ) - raise HTTPException( - status_code=500, - detail=f"Failed to generate PDF report: {exc}", - ) from exc - - snapshot = CallImportEvaluationReportSnapshot( - evaluation_id=evaluation.id, - call_import_id=call_import.id, - organization_id=organization_id, - workspace_id=call_import.workspace_id, - period_label=period_label, - period_start=period_start, - period_end=period_end, - report_config=report_config, - selected_metric_ids=[str(metric.id) for metric in metrics], - metric_aggregates=metric_aggregates, - insight_aggregates=insight_aggregates, - narrative=narrative, - total_calls=evaluation.total_rows, - selected_metric_count=len(metrics), - total_metric_count=db.query(Metric) - .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) - .count(), - ) - db.add(snapshot) - db.commit() - - filename = ( - f"{_report_filename_slug(payload.vendor_name)}-" - f"{payload.report_type}-quality-metric-audit-{eval_id}.pdf" - ) - return StreamingResponse( - iter([pdf_bytes]), - media_type="application/pdf", - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) - - -@router.patch( - "/{eval_id}", - response_model=CallImportEvaluationResponse, - operation_id="updateCallImportEvaluation", -) -async def update_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - payload: CallImportEvaluationUpdate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationResponse: - """Edit metadata on an existing evaluation run (currently just ``name``).""" - - del api_key - _require_import(db, call_import_id, organization_id) - - row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - # Treat unset vs explicit ``None`` differently: unset = leave alone, - # explicit ``None`` or empty string = clear the name. - payload_data = payload.model_dump(exclude_unset=True) - if "name" in payload_data: - row.name = _normalize_name(payload_data["name"]) - - db.commit() - db.refresh(row) - return _serialize_eval(db, row) - - -def _revoke_pending_tasks(evaluation: CallImportEvaluation) -> None: - """Best-effort cancel of any in-flight Celery tasks for an evaluation.""" - - if not evaluation.celery_group_id and not any( - r.celery_task_id for r in evaluation.row_results - ): - return - try: - from app.workers.celery_app import celery_app - - pending_task_ids = [ - eval_row.celery_task_id - for eval_row in evaluation.row_results - if eval_row.celery_task_id - and eval_row.status in {"pending", "running"} - ] - if pending_task_ids: - celery_app.control.revoke(pending_task_ids, terminate=False) - except Exception: - # Best effort — DB delete remains the source of truth. - pass - - -# --------------------------------------------------------------------------- -# User-initiated cancel for in-flight evaluation rows -# --------------------------------------------------------------------------- -# -# Evaluation rows can sit in ``running`` for many minutes when the underlying -# LLM / audio metric call is slow or wedged (the worker carries an 8 min -# soft / 10 min hard time limit). Without a cancel affordance the operator's -# only recourse is to wait for Celery's time limit to fire — or to manually -# mutate the DB. These helpers + the two endpoints below give the UI a -# first-class "Abort" button mirroring the diarisation cancel pattern at -# ``app.api.v1.routes.call_imports`` (``_apply_diarisation_cancel`` etc.). -# -# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` above uses -# ``terminate=False`` because it's called from delete-flow paths where the -# task may simply not get to run (a worker pulls it off the queue and drops -# it). For a user-initiated cancel we want SIGTERM to interrupt the worker -# mid-LLM/audio call so the in-flight HTTP request actually aborts. -# ``terminate=True`` routes the signal to the executing process; we spell -# ``signal="SIGTERM"`` out for clarity even though it's the default. - -# Sentinel error message stamped on cancelled rows. Read by the eval worker's -# ``_was_cancelled_externally`` guard (see -# :mod:`app.workers.tasks.evaluate_call_import_row`) so a worker that's already -# past its slowest operation can't overwrite the cancelled state with its own -# terminal status. Touching either copy means touching both. -EVAL_CANCELLED_BY_USER_ERROR: str = "Evaluation cancelled by user" - - -def _cancellable_eval_states() -> Tuple[str, ...]: - """States that an evaluation row can be cancelled from. - - Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` - state only needs one edit. - """ - return ("pending", "running") - - -def _revoke_eval_task(eval_row: CallImportEvaluationRow) -> None: - """Best-effort revoke of a single eval row's Celery task. - - Always swallows control-plane exceptions — Celery's control bus is - inherently best-effort and a missed revoke is not catastrophic - because the DB row is already flipped to ``failed`` by the caller - before this runs (so the UI immediately reflects the cancel; if - the task happens to finish anyway, the worker's finaliser skips - over the row via :data:`EVAL_CANCELLED_BY_USER_ERROR`). - """ - task_id = (eval_row.celery_task_id or "").strip() - if not task_id: - return - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke( - task_id, terminate=True, signal="SIGTERM" - ) - logger.info( - "Revoked evaluation task {} for eval row {}", - task_id, - eval_row.id, - ) - except Exception as exc: # noqa: BLE001 — revoke is best-effort - logger.warning( - "Failed to revoke evaluation task {} for eval row {}: {}", - task_id, - eval_row.id, - exc, - ) - - -def _apply_evaluation_cancel( - eval_rows: List[CallImportEvaluationRow], -) -> Tuple[int, int]: - """Cancel every cancellable row in ``eval_rows``. - - Returns ``(cancelled, skipped)`` so the caller can build a typed - response without re-querying the DB. The caller is responsible for - ``db.commit()`` after this returns — we deliberately don't commit - here so a batch endpoint can flush all rows in one transaction. - """ - cancellable_states = _cancellable_eval_states() - cancelled = 0 - skipped = 0 - now = datetime.now(timezone.utc) - for eval_row in eval_rows: - if (eval_row.status or "").lower() not in cancellable_states: - skipped += 1 - continue - # Flip the row state BEFORE we revoke so the UI's next poll - # already shows the cancel, even if Celery's control plane is - # slow to ack. - eval_row.status = "failed" - eval_row.error_message = EVAL_CANCELLED_BY_USER_ERROR - eval_row.finished_at = now - _revoke_eval_task(eval_row) - # Drop the task id so a follow-up retry (or a stale poll) can't - # accidentally re-revoke or get confused. - eval_row.celery_task_id = None - cancelled += 1 - return cancelled, skipped - - -def _claim_evaluation_bulk_operation( - evaluation_id: UUID, - operation: str, -) -> None: - """Reserve the run for a single bulk worker pass; 409 if one is active.""" - from app.services.call_imports.evaluation_bulk_op import ( - get_evaluation_bulk_operation, - try_set_evaluation_bulk_operation, - ) - - if try_set_evaluation_bulk_operation(evaluation_id, operation): # type: ignore[arg-type] - return - existing = get_evaluation_bulk_operation(evaluation_id) or operation - raise HTTPException( - status_code=409, - detail=( - f"A bulk {existing.replace('_', ' ')} operation is already in " - "progress for this evaluation. Wait for it to finish before " - "starting another action." - ), - ) - - -def _require_no_evaluation_bulk_operation(evaluation_id: UUID) -> None: - from app.services.call_imports.evaluation_bulk_op import ( - get_evaluation_bulk_operation, - ) - - existing = get_evaluation_bulk_operation(evaluation_id) - if existing: - raise HTTPException( - status_code=409, - detail=( - f"A bulk {existing.replace('_', ' ')} operation is already in " - "progress for this evaluation. Wait for it to finish before " - "starting another action." - ), - ) - - -@router.post( - "/{eval_id}/cancel", - response_model=CallImportEvaluationBulkActionResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="cancelCallImportEvaluation", -) -async def cancel_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationBulkActionResponse: - """Abort all in-flight (or queued) rows in a single evaluation run. - - Idempotent: calling on a run whose rows are already terminal returns - ``target_count=0`` with 202 so the UI can fire this from an - "Abort" button without having to pre-check the state. - - Heavy row resets and Celery revokes run in a background worker so - large batches do not block the API thread. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets - - target_count = count_evaluation_cancel_targets(db, eval_id, mode="abort") - if target_count == 0: - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=0, - evaluation_id=eval_id, - ) - - _claim_evaluation_bulk_operation(eval_id, "abort") - evaluation.status = "cancelled" - db.commit() - - from app.workers.tasks.call_import_bulk_ops import ( - cancel_call_import_evaluation_task, - ) - - cancel_call_import_evaluation_task.delay(str(eval_id), mode="abort") - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=target_count, - evaluation_id=eval_id, - ) - - -@router.post( - "/{eval_id}/force-fail-pending", - response_model=CallImportEvaluationBulkActionResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="forceFailCallImportEvaluationPending", -) -async def force_fail_pending_call_import_evaluation_rows( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationBulkActionResponse: - """Force-fail only rows currently in ``pending`` for a single run. - - This is narrower than :func:`cancel_call_import_evaluation`: it leaves - ``running`` rows untouched so operators can clear permanently queued rows - without interrupting in-flight evaluations. - - Row updates run in a background worker so large batches do not block - the API thread. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets - - target_count = count_evaluation_cancel_targets( - db, eval_id, mode="force_fail_pending" - ) - if target_count == 0: - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=0, - evaluation_id=eval_id, - ) - - _claim_evaluation_bulk_operation(eval_id, "force_fail_pending") - - from app.workers.tasks.call_import_bulk_ops import ( - cancel_call_import_evaluation_task, - ) - - cancel_call_import_evaluation_task.delay( - str(eval_id), mode="force_fail_pending" - ) - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=target_count, - evaluation_id=eval_id, - ) - - -@router.post( - "/{eval_id}/rows/{eval_row_id}/cancel", - response_model=CallImportEvaluationRowResponse, - status_code=status.HTTP_200_OK, - operation_id="cancelCallImportEvaluationRow", -) -async def cancel_call_import_evaluation_row( - call_import_id: UUID, - eval_id: UUID, - eval_row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationRowResponse: - """Abort an in-flight (or queued) evaluation for a single row. - - Idempotent: calling on a row that's already terminal (``completed`` - / ``failed``) returns the row unchanged with a 200 so the UI can - wire this to a "Stop" button without having to pre-check the - state. Updates the parent run's rollup so its counters reflect - the cancel immediately. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - _require_no_evaluation_bulk_operation(eval_id) - - from app.db_sharding.eval_rows import evaluation_row_session - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - try: - with evaluation_row_session(eval_row_id) as ( - row_db, - _catalog_db, - eval_row, - source_row, - _shard_id, - ): - if eval_row.evaluation_id != eval_id: - raise HTTPException( - status_code=404, - detail="Evaluation row not found in this run", - ) - _apply_evaluation_cancel([eval_row]) - row_db.commit() - _rollup_evaluation_status(evaluation, db) - db.commit() - row_db.refresh(eval_row) - return _to_evaluation_row_response(eval_row, source_row, evaluation) - except LookupError as exc: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) from exc - - eval_row = ( - db.query(CallImportEvaluationRow) - .filter( - CallImportEvaluationRow.id == eval_row_id, - CallImportEvaluationRow.evaluation_id == eval_id, - ) - .first() - ) - if not eval_row: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - - _apply_evaluation_cancel([eval_row]) - db.flush() - _rollup_evaluation_status(evaluation, db) - db.commit() - db.refresh(eval_row) - - source_row = ( - db.query(CallImportRow) - .filter(CallImportRow.id == eval_row.call_import_row_id) - .first() - ) - - return _to_evaluation_row_response(eval_row, source_row, evaluation) - - -@router.delete( - "/{eval_id}", - status_code=status.HTTP_204_NO_CONTENT, - operation_id="deleteCallImportEvaluation", -) -async def delete_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Response: - del api_key - _require_import(db, call_import_id, organization_id) - - row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - _revoke_pending_tasks(row) - - db.delete(row) - db.commit() - return Response(status_code=status.HTTP_204_NO_CONTENT) - - -@router.post( - "/bulk-delete", - status_code=status.HTTP_200_OK, - operation_id="bulkDeleteCallImportEvaluations", -) -async def bulk_delete_call_import_evaluations( - call_import_id: UUID, - payload: CallImportEvaluationBulkDelete, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Dict[str, int]: - """Delete multiple evaluation runs scoped to one call import. - - Mirrors :func:`delete_call_import_evaluation` but in bulk so the UI - can clear out a multi-select. Unknown ids (already deleted, or - belonging to a different org/import) are silently skipped — the - response just reports how many actually went away. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - if not payload.evaluation_ids: - return {"deleted": 0} - - rows = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id.in_(payload.evaluation_ids), - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .all() - ) - deleted = 0 - for row in rows: - _revoke_pending_tasks(row) - db.delete(row) - deleted += 1 - db.commit() - return {"deleted": deleted} - - -# --------------------------------------------------------------------------- -# Aggregation: turns per-row metric scores into histograms / value counts. -# -# Designed to be cheap enough to call on every page load: we read each -# evaluation row once, bucket numeric values into a fixed 10-bin -# histogram, and tally the top categorical values. Scaling concerns -# (millions of rows) are deferred — at that point we'd push this into a -# Postgres aggregate query, but for typical CSV imports (<10k rows) the -# Python pass is fast enough and dramatically simpler. -# --------------------------------------------------------------------------- - - -_HISTOGRAM_BUCKETS = 10 -_TOP_VALUE_COUNTS = 10 - - -def _coerce_numeric(value: Any) -> Optional[float]: - """Return ``value`` as ``float`` when it's numeric; ``None`` otherwise.""" - if isinstance(value, bool): - # Booleans are ints in Python; treat them as categorical so - # pass/fail metrics show up in value_counts instead of becoming - # a degenerate {0,1} histogram. - return None - if isinstance(value, (int, float)) and math.isfinite(value): - return float(value) - if isinstance(value, str): - try: - f = float(value) - if math.isfinite(f): - return f - except ValueError: - return None - return None - - -def _coerce_category(value: Any) -> Optional[str]: - """Render ``value`` as a label suitable for a value_counts bucket.""" - if value is None: - return None - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, (int, float)): - return str(value) - if isinstance(value, str): - text = value.strip() - return text or None - # Lists / dicts: stringify so they still group sensibly without - # exploding the cardinality (worst case: everything is "[…]" once). - return str(value) - - -def _build_histogram( - values: List[float], -) -> List[CallImportMetricHistogramBucket]: - """Fixed-bin histogram over ``values``; returns [] for <2 values.""" - if len(values) < 2: - return [] - lo = min(values) - hi = max(values) - if lo == hi: - # All values identical — render a single bucket so the UI shows a - # spike rather than empty space. - return [ - CallImportMetricHistogramBucket(x0=lo, x1=hi, count=len(values)) - ] - width = (hi - lo) / _HISTOGRAM_BUCKETS - buckets: List[List[float]] = [[] for _ in range(_HISTOGRAM_BUCKETS)] - for v in values: - # Right-edge inclusive on the last bucket so ``hi`` doesn't fall - # off into a non-existent bucket index. - idx = int((v - lo) / width) - if idx >= _HISTOGRAM_BUCKETS: - idx = _HISTOGRAM_BUCKETS - 1 - buckets[idx].append(v) - return [ - CallImportMetricHistogramBucket( - x0=lo + i * width, - x1=lo + (i + 1) * width, - count=len(bucket), - ) - for i, bucket in enumerate(buckets) - ] - - -def _percentile(values: List[float], pct: float) -> Optional[float]: - """Linear-interpolated percentile compatible with NumPy default.""" - if not values: - return None - sorted_vals = sorted(values) - if len(sorted_vals) == 1: - return sorted_vals[0] - rank = (pct / 100.0) * (len(sorted_vals) - 1) - lo = int(math.floor(rank)) - hi = int(math.ceil(rank)) - if lo == hi: - return sorted_vals[lo] - frac = rank - lo - return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * frac - - -def _compute_metric_aggregates( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], -) -> List[CallImportMetricAggregate]: - """Collapse per-row ``metric_scores`` into one aggregate per metric. - - Selected metrics are read fresh from the DB so the response always - surfaces the current ``metric.name`` / ``metric_type`` even when a - metric was renamed after the run finished. - """ - - selected_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) - # Include parent metrics from selected_metric_groups so they appear - # alongside their children in the aggregate response. Use ``getattr`` - # with a default so the helper still works for callers that pass - # lightweight objects (tests, in-memory shims) that don't carry the - # attribute at all. - groups_raw_candidate = getattr(evaluation, "selected_metric_groups", None) - groups_raw = ( - groups_raw_candidate if isinstance(groups_raw_candidate, dict) else {} - ) - for parent_str in groups_raw.keys(): - try: - pid = UUID(parent_str) - if pid not in selected_ids: - selected_ids.append(pid) - except (TypeError, ValueError): - continue - - metrics = _metrics_for_ids(db, evaluation.organization_id, selected_ids) - metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} - - # Default to selected metrics, but also include any metric ids that - # surface in row scores even if missing from the metric registry — - # otherwise renaming/deleting a metric mid-run would silently drop - # results from the chart. - discovered_ids: List[str] = list(metric_meta.keys()) - for row in eval_rows: - scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} - for metric_id_str in scores.keys(): - if metric_id_str not in metric_meta and metric_id_str not in discovered_ids: - discovered_ids.append(metric_id_str) - - results: List[CallImportMetricAggregate] = [] - - for metric_id_str in discovered_ids: - meta = metric_meta.get(metric_id_str) - numeric_values: List[float] = [] - category_counts: Dict[str, int] = {} - # For multi-label parents we still need to know how many rows - # were scored (each row votes for >=1 label) so the n-badge in - # the UI shows "n=50" instead of the misleading "n=208" sum. - multi_label_rows_scored = 0 - # Unordered pair tally for the co-occurrence heatmap. Keys are - # ``(label_a, label_b)`` with ``a < b`` so we never double-count - # the same unordered pair. Only populated for multi-label - # parents — every other metric leaves this empty. - multi_label_pair_counts: Dict[Tuple[str, str], int] = {} - skipped = 0 - errored = 0 - observed_metric_type: Optional[str] = None - observed_name: Optional[str] = None - - # ``meta`` is a real ``Metric`` row in production, but tests - # frequently pass a lightweight stub. Pull the two attributes - # we need via ``getattr`` so a stub that only sets ``id`` / - # ``name`` / ``metric_type`` doesn't blow up here. - is_multi_label_parent = bool( - meta - and getattr(meta, "selection_mode", None) == "multi_label" - and not getattr(meta, "parent_metric_id", None) - ) - - for row in eval_rows: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else {} - ) - entry = scores.get(metric_id_str) - if not isinstance(entry, dict): - continue - if entry.get("metric_name"): - observed_name = entry.get("metric_name") - if entry.get("type"): - observed_metric_type = entry.get("type") - if entry.get("skipped"): - skipped += 1 - continue - if entry.get("error"): - errored += 1 - continue - - # Multi-label parents store a comma-joined value that - # isn't useful as a single category; instead tally each - # selected child individually so the chart shows per-label - # counts that mirror the children's own boolean histograms. - if is_multi_label_parent: - selected = entry.get("selected_child_names") - if isinstance(selected, list) and selected: - multi_label_rows_scored += 1 - cleaned: List[str] = [] - for label in selected: - text_label = str(label).strip() or None - if text_label: - cleaned.append(text_label) - category_counts[text_label] = ( - category_counts.get(text_label, 0) + 1 - ) - # Emit one increment per unordered pair of distinct - # labels that fired together on this row. ``cleaned`` - # is deduplicated first because the LLM occasionally - # repeats a label inside ``selected_child_names``. - distinct = sorted(set(cleaned)) - for i in range(len(distinct)): - for j in range(i + 1, len(distinct)): - pair = (distinct[i], distinct[j]) - multi_label_pair_counts[pair] = ( - multi_label_pair_counts.get(pair, 0) + 1 - ) - continue - - value = entry.get("value") - numeric = _coerce_numeric(value) - if numeric is not None: - numeric_values.append(numeric) - continue - category = _coerce_category(value) - if category is not None: - category_counts[category] = category_counts.get(category, 0) + 1 - - # ``count`` is "rows scored". For numeric / single-choice - # metrics that's the same as ``len(numeric) + sum(categories)`` - # because each scored row contributes exactly one observation. - # Multi-label parents however contribute one observation per - # selected child, so summing ``category_counts`` over-counts — - # we tracked rows-scored separately above and use it here. - rows_scored = ( - multi_label_rows_scored - if is_multi_label_parent - else len(numeric_values) + sum(category_counts.values()) - ) - - # Build numeric stats first, then categorical (both can coexist). - agg = CallImportMetricAggregate( - metric_id=metric_id_str, - metric_name=( - (meta.name if meta else observed_name) or "Unknown metric" - ), - metric_type=( - meta.metric_type if meta else observed_metric_type - ), - metric_category=( - "user_insight" - if meta is not None and _metric_is_user_insight(meta) - else "quality" - ) - or "quality", - is_multi_label_parent=is_multi_label_parent, - count=rows_scored, - skipped_count=skipped, - error_count=errored, - ) - if numeric_values: - agg.mean = float(statistics.fmean(numeric_values)) - agg.median = float(statistics.median(numeric_values)) - agg.min = min(numeric_values) - agg.max = max(numeric_values) - agg.stddev = ( - float(statistics.pstdev(numeric_values)) - if len(numeric_values) > 1 - else 0.0 - ) - agg.p25 = _percentile(numeric_values, 25) - agg.p75 = _percentile(numeric_values, 75) - agg.p95 = _percentile(numeric_values, 95) - agg.histogram_buckets = _build_histogram(numeric_values) - if category_counts: - sorted_counts = sorted( - category_counts.items(), key=lambda kv: kv[1], reverse=True - ) - agg.value_counts = [ - CallImportMetricValueCount(label=label, count=count) - for label, count in sorted_counts[:_TOP_VALUE_COUNTS] - ] - # Restrict the heatmap to pairs of labels we actually - # rendered above so the frontend never has to match - # against truncated/missing rows. Sorted desc by pair - # count to keep the most informative cells in the - # response when ``_TOP_VALUE_COUNTS`` clipped the matrix. - if is_multi_label_parent and multi_label_pair_counts: - kept_labels = { - label for label, _ in sorted_counts[:_TOP_VALUE_COUNTS] - } - pair_items = [ - (a, b, count) - for (a, b), count in multi_label_pair_counts.items() - if a in kept_labels and b in kept_labels - ] - pair_items.sort(key=lambda t: t[2], reverse=True) - agg.co_occurrence = [ - CallImportMetricLabelPair(a=a, b=b, count=count) - for a, b, count in pair_items - ] - - results.append(agg) - - # Sort so each parent metric immediately precedes its children. - # The Visualizations grid renders metrics top-to-bottom in this - # order, so multi-label parents (the "summary" chart) sit above - # the per-child boolean histograms that drill into them. Metrics - # whose ``meta`` row was deleted mid-run (``meta is None``) sink - # to the bottom but keep their relative order. - enumerated = list(enumerate(results)) - - def _sort_key(item: Tuple[int, CallImportMetricAggregate]): - original_idx, agg = item - meta = metric_meta.get(agg.metric_id) - if meta is None: - return (1, "", 1, "", original_idx) - parent_id = getattr(meta, "parent_metric_id", None) - # Group key: a child shares its parent's UUID; a parent - # uses its own UUID. Within a group, depth=0 (parent) sorts - # before depth=1 (child); ties break alphabetically by name - # so children render in a stable order regardless of which - # row scored which label first. - if parent_id is None: - group_key = str(meta.id) - depth = 0 - else: - group_key = str(parent_id) - depth = 1 - return ( - 0, - group_key, - depth, - (getattr(meta, "name", "") or "").lower(), - original_idx, - ) - - enumerated.sort(key=_sort_key) - return [agg for _idx, agg in enumerated] - - -@router.get( - "/{eval_id}/aggregate", - response_model=CallImportEvaluationAggregateResponse, - operation_id="getCallImportEvaluationAggregate", -) -async def get_call_import_evaluation_aggregate( - call_import_id: UUID, - eval_id: UUID, - baseline_evaluation_id: Optional[UUID] = Query(None), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationAggregateResponse: - """Return per-metric distributions for the Visualizations tab. - - The shape is intentionally chart-friendly: histograms for numeric - metrics, top-N value counts for categorical/text metrics, plus - summary stats (mean/p50/p95) so the UI can render summary cards - without recomputing on the client. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - - metrics = _compute_metric_aggregates(db, evaluation, eval_rows) - - period_deltas: dict[str, MetricPeriodDelta] = {} - resolved_baseline_id: Optional[UUID] = None - if baseline_evaluation_id is not None: - call_import = _require_import(db, call_import_id, organization_id) - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - - rows = load_evaluation_row_pairs(db, eval_id) - period_start, _, _, _ = _report_period_from_rows(rows) - baseline_evaluation = _resolve_baseline_evaluation( - db, - organization_id, - call_import.workspace_id, - evaluation, - period_start, - str(baseline_evaluation_id), - ) - if baseline_evaluation: - resolved_baseline_id = baseline_evaluation.id - metric_aggregates_dicts = [ - _aggregate_to_dict(agg) for agg in metrics - ] - raw_deltas = _period_deltas_from_evaluation( - db, - baseline_evaluation, - metric_aggregates_dicts, - evaluation, - eval_rows, - ) - raw_deltas = _period_deltas_with_explanations( - db, - organization_id, - evaluation, - baseline_evaluation, - raw_deltas, - ) - period_deltas = { - metric_id: MetricPeriodDelta( - label=delta.get("label") or "", - detail=delta.get("detail") or "", - why=(delta.get("why") or "").strip() or None, - ) - for metric_id, delta in raw_deltas.items() - } - - _fp_stored, failure_policies_source = policies_from_evaluation_raw( - evaluation.metric_clusters - ) - return CallImportEvaluationAggregateResponse( - evaluation_id=eval_id, - total_rows=evaluation.total_rows, - completed_rows=evaluation.completed_rows, - failed_rows=evaluation.failed_rows, - metrics=metrics, - period_deltas=period_deltas, - baseline_evaluation_id=resolved_baseline_id, - failure_policies_source=failure_policies_source, - ) - - -# --------------------------------------------------------------------------- -# TLDR insights: LLM-generated narrative + bullet patterns rendered above -# the Visualizations charts. Cached on ``CallImportEvaluation.tldr_summary`` -# so the page never auto-burns LLM tokens; the user explicitly clicks -# "Generate summary" or "Regenerate" from the empty-state CTA. -# --------------------------------------------------------------------------- - - -_INSIGHTS_SYSTEM_PROMPT = ( - "You are a senior conversation-analytics reviewer. You will be " - "given aggregated metric statistics + a sample of rationales for " - "the rows of a single call-import evaluation. Identify the most " - "useful PATTERNS that hold ACROSS the calls -- not just per-metric " - "numbers. Look for combinations (e.g. `when X happens, Y also " - "tends to happen`), notable outliers, frequent failure modes, and " - "any signal that would change how a reviewer triages the run.\n\n" - "Return STRICT JSON only, with this shape and no extra keys:\n" - "{\n" - ' "narrative": "",\n' - ' "patterns": ["", "", ...],\n' - ' "metric_insights": {"": "<2-3 line business meaning>"}\n' - "}\n\n" - "Constraints:\n" - "- narrative is the ONLY text shown in the external audit summary and " - "Visualizations TLDR; keep it to at most 3 short sentences (~300 chars).\n" - "- patterns are optional supporting notes and are NOT rendered in the " - "audit summary; keep 0 to 3 bullets if supplied, each <= 120 characters.\n" - "- metric_insights must include one entry for each top-level metric id supplied.\n" - "- Each metric insight should explain what the metric means for the business and what the current distribution suggests, not restate the metric rubric.\n" - "- Avoid restating raw counts unless they reveal a pattern.\n" - "- Use neutral, factual language ('frustration appeared in...') " - "rather than judgemental ('the agents failed to...')." -) - - -def _tldr_summary_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationTldrSummary]: - """Return the cached TLDR (with ``is_stale`` set) or ``None``. - - ``CallImportEvaluation.tldr_summary`` is a ``JSON`` column so we - have to validate shape defensively -- a half-written or hand-edited - blob should not break the aggregate response. Returns ``None`` when - no cached summary exists. - """ - raw = evaluation.tldr_summary - if not isinstance(raw, dict): - return None - narrative = raw.get("narrative") - if not isinstance(narrative, str) or not narrative.strip(): - return None - patterns_raw = raw.get("patterns") - patterns = ( - [str(p) for p in patterns_raw if isinstance(p, str) and p.strip()] - if isinstance(patterns_raw, list) - else [] - ) - metric_insights_raw = raw.get("metric_insights") - metric_insights = ( - { - str(metric_id): str(insight).strip() - for metric_id, insight in metric_insights_raw.items() - if str(metric_id).strip() - and isinstance(insight, str) - and insight.strip() - } - if isinstance(metric_insights_raw, dict) - else {} - ) - generated_at_raw = raw.get("generated_at") - try: - generated_at = ( - datetime.fromisoformat(generated_at_raw) - if isinstance(generated_at_raw, str) - else evaluation.updated_at or datetime.now(timezone.utc) - ) - except ValueError: - generated_at = evaluation.updated_at or datetime.now(timezone.utc) - snapshot = raw.get("generated_at_completed_rows") - snapshot_int = int(snapshot) if isinstance(snapshot, (int, float)) else 0 - return EvaluationTldrSummary( - narrative=_clamp_prose_to_sentences(narrative.strip()), - patterns=patterns, - metric_insights=metric_insights, - generated_at=generated_at, - generated_at_completed_rows=snapshot_int, - provider=raw.get("provider") if isinstance(raw.get("provider"), str) else None, - model=raw.get("model") if isinstance(raw.get("model"), str) else None, - is_stale=evaluation.completed_rows > snapshot_int, - ) - - -def _sample_rationales_per_metric( - eval_rows: List[CallImportEvaluationRow], - *, - per_metric_cap: int = 3, - rationale_char_cap: int = 600, -) -> Dict[str, List[str]]: - """Collect up to ``per_metric_cap`` distinct rationales per metric. - - Distinctness is case- and whitespace-insensitive. We truncate each - rationale to ``rationale_char_cap`` so a few unusually verbose rows - can't dominate the prompt budget. Empty / non-string rationales are - skipped. - """ - out: Dict[str, List[str]] = {} - seen: Dict[str, set[str]] = {} - for row in eval_rows: - scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} - for metric_id, entry in scores.items(): - if not isinstance(entry, dict): - continue - rationale = entry.get("rationale") - if not isinstance(rationale, str): - continue - text = rationale.strip() - if not text: - continue - bucket = out.setdefault(metric_id, []) - if len(bucket) >= per_metric_cap: - continue - key = " ".join(text.lower().split()) - seen_set = seen.setdefault(metric_id, set()) - if key in seen_set: - continue - seen_set.add(key) - bucket.append(text[:rationale_char_cap]) - return out - - -def _build_insights_messages( - evaluation: CallImportEvaluation, - aggregate: List[CallImportMetricAggregate], - rationale_samples: Dict[str, List[str]], - metric_meta: Dict[str, Metric], -) -> List[Dict[str, str]]: - """Render the user prompt fed to the LLM. - - The shape is plain markdown-ish text instead of JSON so the LLM can - skim it without us spending tokens on verbose schema delimiters. - Parent metrics surface their child metrics nested underneath so the - model sees the hierarchy and can talk about "X often co-occurred - with Y" rather than treating sub-labels as standalone metrics. - """ - name = evaluation.name or f"Run {str(evaluation.id)[:8]}" - lines: List[str] = [ - f"Evaluation: {name}", - ( - f"Rows: total={evaluation.total_rows} " - f"completed={evaluation.completed_rows} " - f"failed={evaluation.failed_rows}" - ), - "", - "## Per-metric aggregate", - ] - - # Group metrics by parent so the prompt mirrors the hierarchy. Any - # aggregate row whose ``metric_id`` is missing from ``metric_meta`` - # is rendered as a leaf at the top-level list (handles renamed / - # deleted parents). - children_by_parent: Dict[str, List[CallImportMetricAggregate]] = {} - top_level: List[CallImportMetricAggregate] = [] - for agg in aggregate: - meta = metric_meta.get(agg.metric_id) - parent_id = ( - str(meta.parent_metric_id) - if meta is not None and getattr(meta, "parent_metric_id", None) - else None - ) - if parent_id: - children_by_parent.setdefault(parent_id, []).append(agg) - else: - top_level.append(agg) - - def _format_metric_block(agg: CallImportMetricAggregate, indent: int) -> List[str]: - prefix = " " * indent + "- " - bits: List[str] = [f"{prefix}{agg.metric_name} [id={agg.metric_id}] (n={agg.count}"] - if agg.skipped_count: - bits.append(f", skipped={agg.skipped_count}") - if agg.error_count: - bits.append(f", errors={agg.error_count}") - bits.append(")") - meta = metric_meta.get(agg.metric_id) - description = (meta.description or "").strip() if meta else "" - if description: - bits.append(f" | definition={description[:500]}") - if agg.mean is not None: - mean_s = f"{agg.mean:.2f}" - stddev_s = f"{agg.stddev:.2f}" if agg.stddev is not None else "-" - bits.append(f" | mean={mean_s} stddev={stddev_s}") - if agg.min is not None and agg.max is not None: - bits.append(f" range=[{agg.min:.2f}, {agg.max:.2f}]") - if agg.value_counts: - total = sum(v.count for v in agg.value_counts) or 1 - top = agg.value_counts[:3] - shares = ", ".join( - f'"{v.label}"={v.count}/{total}' for v in top - ) - bits.append(f" | top={shares}") - result = ["".join(bits)] - rationales = rationale_samples.get(agg.metric_id, []) - for r in rationales: - result.append(" " * (indent + 1) + f"- rationale: {r}") - return result - - for agg in top_level: - lines.extend(_format_metric_block(agg, indent=0)) - meta = metric_meta.get(agg.metric_id) - children = children_by_parent.get(str(meta.id), []) if meta else [] - for child in children: - lines.extend(_format_metric_block(child, indent=1)) - - lines.append("") - top_level_ids = [agg.metric_id for agg in top_level] - if top_level_ids: - lines.append( - "metric_insights keys must exactly use these top-level metric ids: " - + ", ".join(top_level_ids) - ) - lines.append("") - lines.append( - "Write the JSON object as instructed. Do not include " - "preamble, code fences, or trailing commentary." - ) - - return [ - {"role": "system", "content": _INSIGHTS_SYSTEM_PROMPT}, - {"role": "user", "content": "\n".join(lines)}, - ] - - -def _parse_insights_response(text: str) -> EvaluationTldrSummary: - """Coerce the LLM response into ``narrative`` + ``patterns``. - - Matches the JSON-with-fallback pattern used by - ``app.api.v1.routes.metrics._parse_metric_generation_response``: try - ``json.loads`` first, then fall back to regex extraction of the - first ``{...}`` block. Raises ``HTTPException`` with a 502 when the - response can't be parsed at all. - """ - cleaned = (text or "").strip() - if not cleaned: - raise HTTPException( - status_code=502, detail="LLM returned an empty insights response" - ) - try: - parsed = json.loads(cleaned) - except json.JSONDecodeError: - import re - - match = re.search(r"\{.*\}", cleaned, re.DOTALL) - if not match: - raise HTTPException( - status_code=502, - detail="Could not parse LLM insights response as JSON", - ) - try: - parsed = json.loads(match.group(0)) - except json.JSONDecodeError as e: - raise HTTPException( - status_code=502, - detail=f"Could not parse LLM insights response: {e}", - ) - - if not isinstance(parsed, dict): - raise HTTPException( - status_code=502, detail="LLM insights JSON was not an object" - ) - - narrative = parsed.get("narrative") - if not isinstance(narrative, str) or not narrative.strip(): - raise HTTPException( - status_code=502, - detail="LLM insights JSON missing 'narrative' string", - ) - - patterns_raw = parsed.get("patterns") - if patterns_raw is None: - patterns: List[str] = [] - elif isinstance(patterns_raw, list): - patterns = [ - str(p).strip() - for p in patterns_raw - if isinstance(p, str) and p.strip() - ] - else: - raise HTTPException( - status_code=502, - detail="LLM insights JSON 'patterns' must be a list of strings", - ) - metric_insights_raw = parsed.get("metric_insights") - if metric_insights_raw is None: - metric_insights: Dict[str, str] = {} - elif isinstance(metric_insights_raw, dict): - metric_insights = { - str(metric_id): str(insight).strip() - for metric_id, insight in metric_insights_raw.items() - if str(metric_id).strip() - and isinstance(insight, str) - and insight.strip() - } - else: - raise HTTPException( - status_code=502, - detail="LLM insights JSON 'metric_insights' must be an object", - ) - - return EvaluationTldrSummary( - narrative=_clamp_prose_to_sentences(narrative.strip()), - patterns=patterns, - metric_insights=metric_insights, - generated_at=datetime.now(timezone.utc), - generated_at_completed_rows=0, # filled in by caller - is_stale=False, - ) - - -def _generate_and_persist_tldr_summary( - db: Session, - evaluation: CallImportEvaluation, - *, - organization_id: UUID, - provider: Optional[str] = None, - model: Optional[str] = None, -) -> EvaluationTldrSummary: - """LLM TLDR generation used by the imports-queue Celery worker.""" - eval_id = evaluation.id - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - - pairs = load_evaluation_row_pairs(db, eval_id) - eval_rows = [eval_row for eval_row, _ in pairs] - aggregate = _compute_metric_aggregates(db, evaluation, eval_rows) - if not aggregate: - raise HTTPException( - status_code=400, - detail=( - "No metric data yet. Wait for at least one row to " - "finish scoring before generating a summary." - ), - ) - - metric_ids: List[UUID] = [] - for agg in aggregate: - try: - metric_ids.append(UUID(agg.metric_id)) - except (TypeError, ValueError): - continue - metrics = _metrics_for_ids(db, organization_id, metric_ids) - metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} - - rationale_samples = _sample_rationales_per_metric(eval_rows) - messages = _build_insights_messages( - evaluation, aggregate, rationale_samples, metric_meta - ) - - from app.services.ai.llm_resolver import get_llm_provider_and_model - from app.services.ai.llm_service import llm_service - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, provider, model - ) - - try: - llm_result = llm_service.generate_response( - messages=messages, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - temperature=0.4, - max_tokens=1400, - ) - except Exception as e: - logger.error(f"[CallImportInsights] LLM call failed: {e}") - raise HTTPException( - status_code=502, detail=f"LLM call failed: {e}" - ) from e - - summary = _parse_insights_response(llm_result.get("text", "")) - total = int(evaluation.total_rows or 0) - ui_completed = min(int(evaluation.completed_rows or 0), total) if total else int( - evaluation.completed_rows or 0 - ) - summary.generated_at_completed_rows = ui_completed - summary.provider = provider_enum.value - summary.model = model_str - summary.is_stale = False - - evaluation.tldr_summary = { - "narrative": summary.narrative, - "patterns": summary.patterns, - "metric_insights": summary.metric_insights, - "generated_at": summary.generated_at.isoformat(), - "generated_at_completed_rows": summary.generated_at_completed_rows, - "provider": summary.provider, - "model": summary.model, - } - flag_modified(evaluation, "tldr_summary") - db.commit() - db.refresh(evaluation) - return summary - - -@router.get( - "/{eval_id}/insights", - response_model=Optional[EvaluationTldrSummary], - operation_id="getCallImportEvaluationInsights", -) -async def get_call_import_evaluation_insights( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationTldrSummary]: - """Return the cached TLDR (or ``null``) without contacting the LLM. - - Used by the Visualizations tab on first paint so the empty-state - CTA can show up before the user opts into generation. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _tldr_summary_payload(evaluation) - - -@router.post( - "/{eval_id}/insights", - response_model=EvaluationTldrSummary, - operation_id="generateCallImportEvaluationInsights", -) -async def generate_call_import_evaluation_insights( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationInsightsRequest = Body(default_factory=EvaluationInsightsRequest), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> EvaluationTldrSummary: - """Generate (or return-cached) the LLM TLDR for an evaluation run. - - Behavior: - - * ``body.regenerate=False`` and a cached summary at the current - ``completed_rows`` watermark exists -> return it as-is. - * ``body.regenerate=False`` and a stale cached summary exists - (``generated_at_completed_rows < completed_rows``) -> return it - with ``is_stale=True``; the UI prompts the user to regenerate. - * Otherwise -> resolve provider+model (auto-detect when omitted), - call the LLM, persist the new summary, return it. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not body.regenerate: - cached = _tldr_summary_payload(evaluation) - if cached is not None: - return cached - - # Run the TLDR LLM on the imports worker (not the default worker or API). - from app.workers.tasks.generate_evaluation_tldr_insights import ( - generate_evaluation_tldr_insights_task, - ) - - try: - task_result = generate_evaluation_tldr_insights_task.apply_async( - kwargs={ - "evaluation_id": str(eval_id), - "call_import_id": str(call_import_id), - "organization_id": str(organization_id), - "provider": body.provider, - "model": body.model, - }, - ).get(timeout=25 * 60) - except Exception as exc: - logger.error( - "[CallImportInsights] TLDR task failed for evaluation {}: {}", - eval_id, - exc, - ) - raise HTTPException( - status_code=502, - detail=f"Summary generation failed: {exc}", - ) from exc - - if isinstance(task_result, dict) and task_result.get("error"): - status_code = int(task_result.get("status_code") or 502) - raise HTTPException( - status_code=status_code, - detail=str(task_result["error"]), - ) - - summary = EvaluationTldrSummary.model_validate(task_result) - db.refresh(evaluation) - - from app.services.ai.llm_resolver import get_llm_provider_and_model - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, body.provider, body.model, body.credential_id - ) - - _enqueue_user_insights_job( - evaluation, - provider=summary.provider or provider_enum.value, - model=summary.model or model_str, - force=body.regenerate, - max_llm_calls=body.max_llm_calls, - db=db, - ) - - return summary - - -def _user_insights_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationUserInsightsState]: - raw = getattr(evaluation, "user_insights", None) - if raw is None: - return None - return user_insights_state_from_raw( - raw, - completed_rows=evaluation.completed_rows, - ) - - -def _selected_generated_user_insights( - state: Optional[EvaluationUserInsightsState], - report_config: dict[str, Any], -) -> list[dict[str, Any]]: - """Filter and order generated insights for PDF section 03.""" - if state is None or state.status != "completed" or not state.insights: - return [] - - selected_ids = report_config.get("user_insight_ids") - if isinstance(selected_ids, list) and selected_ids: - allowed = {str(item) for item in selected_ids if item} - items = [item for item in state.insights if item.id in allowed] - else: - items = list(state.insights) - - order_raw = report_config.get("order") - order_ids: list[str] = [] - if isinstance(order_raw, dict): - user_order = order_raw.get("user_insights") - if isinstance(user_order, list): - order_ids = [str(item) for item in user_order if item] - - if order_ids: - by_id = {item.id: item for item in items} - ordered = [by_id[iid] for iid in order_ids if iid in by_id] - seen = set(order_ids) - ordered.extend(item for item in items if item.id not in seen) - items = ordered - - return [item.model_dump(mode="json") for item in items] - - -def _enqueue_user_insights_job( - evaluation: CallImportEvaluation, - *, - provider: Optional[str] = None, - model: Optional[str] = None, - force: bool = False, - max_llm_calls: Optional[int] = None, - db: Optional[Session] = None, -) -> None: - """Enqueue background user-insights generation unless already running.""" - current = _user_insights_payload(evaluation) - if current is not None and current.status == "running" and not force: - return - - llm_budget = normalize_max_llm_calls(max_llm_calls) - - completed_count = ( - _count_completed_eval_rows(db, evaluation.id) - if db is not None - else evaluation.completed_rows - ) - total_calls = total_llm_calls_for_rows(completed_count, max_llm_calls=llm_budget) - evaluation.user_insights = { - "status": "running", - "insights": ( - (evaluation.user_insights or {}).get("insights", []) - if isinstance(evaluation.user_insights, dict) - else [] - ), - "generated_at": datetime.now(timezone.utc).isoformat(), - "generated_at_completed_rows": evaluation.completed_rows, - "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, - "provider": provider, - "model": model, - "max_llm_calls": llm_budget, - "llm_calls_used": 0, - "error_message": None, - } - if db is not None: - flag_modified(evaluation, "user_insights") - db.commit() - - from app.workers.tasks.generate_evaluation_user_insights import ( - generate_evaluation_user_insights_task, - ) - - generate_evaluation_user_insights_task.delay( - str(evaluation.id), - provider=provider, - model=model, - max_llm_calls=llm_budget, - ) - - -@router.get( - "/{eval_id}/user-insights", - response_model=Optional[EvaluationUserInsightsState], - operation_id="getCallImportEvaluationUserInsights", -) -async def get_call_import_evaluation_user_insights( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationUserInsightsState]: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _user_insights_payload(evaluation) - - -@router.post( - "/{eval_id}/user-insights", - response_model=EvaluationUserInsightsState, - operation_id="generateCallImportEvaluationUserInsights", -) -async def generate_call_import_evaluation_user_insights( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationUserInsightsRequest = Body( - default_factory=EvaluationUserInsightsRequest - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> EvaluationUserInsightsState: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not body.regenerate and not body.force: - cached = _user_insights_payload(evaluation) - if cached is not None and cached.status in {"running", "completed"}: - return cached - - eval_rows = _load_eval_rows(db, eval_id) - if not any(row.status == "completed" for row in eval_rows): - raise HTTPException( - status_code=400, - detail=( - "No completed rows yet. Wait for at least one row to " - "finish scoring before generating user insights." - ), - ) - - from app.services.ai.llm_resolver import get_llm_provider_and_model - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, body.provider, body.model, body.credential_id - ) - - _enqueue_user_insights_job( - evaluation, - provider=provider_enum.value, - model=model_str, - force=body.force or body.regenerate, - max_llm_calls=body.max_llm_calls, - db=db, - ) - - db.refresh(evaluation) - return _user_insights_payload(evaluation) or EvaluationUserInsightsState( - status="running" - ) - - -def _metric_clusters_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationMetricClustersState]: - raw = getattr(evaluation, "metric_clusters", None) - if raw is None: - return None - return metric_clusters_state_from_raw( - raw, - completed_rows=evaluation.completed_rows, - ) - - -def _selected_metric_clusters_for_pdf( - state: Optional[EvaluationMetricClustersState], - report_config: dict[str, Any], -) -> dict[str, Any]: - if state is None or state.status != "completed": - return {} - sections = report_config.get("sections") - if isinstance(sections, dict) and sections.get("failure_diagnostics") is False: - return {} - payload: dict[str, Any] = { - "groups": [g.model_dump(mode="json") for g in state.groups], - "discovered_problems": [ - d.model_dump(mode="json") for d in state.discovered_problems - ], - } - if state.rca_summary is not None: - payload["rca_summary"] = state.rca_summary.model_dump(mode="json") - return payload - - -def _prompt_improvements_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationPromptImprovementsState]: - from app.services.call_import_prompt_improvements import ( - prompt_improvements_state_from_raw, - ) - - raw = getattr(evaluation, "prompt_improvements", None) - if raw is None: - return None - return prompt_improvements_state_from_raw( - raw, - completed_rows=evaluation.completed_rows, - ) - - -def _selected_prompt_improvements_for_pdf( - state: Optional[EvaluationPromptImprovementsState], - report_config: dict[str, Any], -) -> dict[str, Any]: - if state is None or state.status != "completed": - return {} - sections = report_config.get("sections") - if isinstance(sections, dict) and sections.get("prompt_improvements") is False: - return {} - return { - "imported_agent_id": state.imported_agent_id, - "imported_agent_name": state.imported_agent_name, - "overview": state.overview, - "suggestions": [s.model_dump(mode="json") for s in state.suggestions], - } - - -def _enqueue_prompt_improvements_job( - evaluation: CallImportEvaluation, - *, - imported_agent_id: UUID, - imported_agent_name: str, - provider: Optional[str] = None, - model: Optional[str] = None, - credential_id: Optional[UUID] = None, - force: bool = False, - db: Optional[Session] = None, -) -> None: - current = _prompt_improvements_payload(evaluation) - if current is not None and current.status == "running" and not force: - return - - evaluation.prompt_improvements = { - "status": "running", - "imported_agent_id": str(imported_agent_id), - "imported_agent_name": imported_agent_name, - "suggestions": [], - "generated_at": datetime.now(timezone.utc).isoformat(), - "generated_at_completed_rows": evaluation.completed_rows, - "provider": provider, - "model": model, - "error_message": None, - } - if db is not None: - flag_modified(evaluation, "prompt_improvements") - db.commit() - - from app.workers.tasks.generate_evaluation_prompt_improvements import ( - generate_evaluation_prompt_improvements_task, - ) - - async_result = generate_evaluation_prompt_improvements_task.apply_async( - kwargs={ - "evaluation_id": str(evaluation.id), - "imported_agent_id": str(imported_agent_id), - "provider": provider, - "model": model, - "credential_id": str(credential_id) if credential_id else None, - }, - queue="imports", - ) - if db is not None and isinstance(evaluation.prompt_improvements, dict): - evaluation.prompt_improvements["celery_task_id"] = async_result.id - flag_modified(evaluation, "prompt_improvements") - db.commit() - - -def _load_eval_rows(db: Session, evaluation_id: UUID) -> List[CallImportEvaluationRow]: - from app.db_sharding.eval_rows import load_evaluation_rows_for_run - - return load_evaluation_rows_for_run(db, evaluation_id) - - -def _count_completed_eval_rows(db: Session, evaluation_id: UUID) -> int: - from app.db_sharding.eval_rows import count_evaluation_rows_for_run - - return count_evaluation_rows_for_run( - db, evaluation_id, statuses=["completed"] - ) - - -def _completed_row_pairs_for_evaluation( - db: Session, - evaluation_id: UUID, -) -> List[Tuple[CallImportEvaluationRow, CallImportRow]]: - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - - row_pairs = load_evaluation_row_pairs(db, evaluation_id) - return [ - (eval_row, source_row) - for eval_row, source_row in row_pairs - if eval_row.status == "completed" - ] - - -def _resolve_metric_cluster_row_selection( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], - evaluation_row_ids: Optional[List[UUID]], - *, - row_limit: Optional[int] = None, - policies: Optional[Dict[str, MetricFailurePolicy]] = None, -) -> Tuple[List[Tuple[CallImportEvaluationRow, CallImportRow]], List[str]]: - """Return filtered completed row pairs and the selected row id strings.""" - completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) - metrics = _metrics_for_clustering(db, evaluation, eval_rows) - if policies is None: - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) - and not getattr(m, "parent_metric_id", None) - ] - child_names_by_parent = _child_names_by_parent( - db, evaluation.organization_id, parent_ids - ) - policies, _ = effective_policies( - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - eligible = list_eligible_cluster_rows( - evaluation, completed_pairs, metrics, policies - ) - eligible_ordered_ids = [str(item["evaluation_row_id"]) for item in eligible] - eligible_id_set = set(eligible_ordered_ids) - - if evaluation_row_ids is None and row_limit is not None: - selected_ids = eligible_ordered_ids[:row_limit] - filtered = filter_completed_row_pairs( - completed_pairs, - [UUID(rid) for rid in selected_ids], - ) - return filtered, selected_ids - - if evaluation_row_ids is None: - selected_ids = eligible_ordered_ids - filtered = filter_completed_row_pairs( - completed_pairs, - [UUID(rid) for rid in selected_ids], - ) - return filtered, selected_ids - - requested = {str(rid) for rid in evaluation_row_ids} - completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} - unknown = sorted(requested - completed_id_set) - if unknown: - raise HTTPException( - status_code=400, - detail=( - "One or more evaluation_row_ids are missing or not completed: " - + ", ".join(unknown[:5]) - + ("…" if len(unknown) > 5 else "") - ), - ) - not_eligible = sorted(requested - eligible_id_set) - if not_eligible: - raise HTTPException( - status_code=400, - detail=( - "Each selected row must have at least one flagged quality metric. " - "Ineligible row(s): " - + ", ".join(not_eligible[:5]) - + ("…" if len(not_eligible) > 5 else "") - ), - ) - selected_ids = sorted(requested) - filtered = filter_completed_row_pairs(completed_pairs, evaluation_row_ids) - return filtered, selected_ids - - -def _enqueue_metric_clusters_job( - evaluation: CallImportEvaluation, - *, - provider: Optional[str] = None, - model: Optional[str] = None, - credential_id: Optional[UUID] = None, - force: bool = False, - max_llm_calls: Optional[int] = None, - evaluation_row_ids: Optional[List[UUID]] = None, - selected_evaluation_row_ids: Optional[List[str]] = None, - failure_policies: Optional[Dict[str, MetricFailurePolicy]] = None, - db: Optional[Session] = None, -) -> None: - current = _metric_clusters_payload(evaluation) - if current is not None and current.status == "running" and not force: - return - - llm_budget = normalize_max_llm_calls(max_llm_calls) - total_calls = 1 - row_ids_for_task: Optional[List[str]] = None - if db is not None: - eval_rows = _load_eval_rows(db, evaluation.id) - if selected_evaluation_row_ids is None: - _, selected_evaluation_row_ids = _resolve_metric_cluster_row_selection( - db, - evaluation, - eval_rows, - evaluation_row_ids, - ) - completed_pairs = filter_completed_row_pairs( - _completed_row_pairs_for_evaluation(db, evaluation.id), - [UUID(rid) for rid in selected_evaluation_row_ids], - ) - metrics = _metrics_for_clustering(db, evaluation, eval_rows) - policies_for_estimate = failure_policies - if policies_for_estimate is None: - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) - and not getattr(m, "parent_metric_id", None) - ] - child_names_by_parent = _child_names_by_parent( - db, evaluation.organization_id, parent_ids - ) - policies_for_estimate, _ = effective_policies( - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - _, total_calls = estimate_metric_clusters_llm_calls( - evaluation, - metrics, - completed_pairs, - policies_for_estimate, - max_llm_calls=llm_budget, - ) - row_ids_for_task = list(selected_evaluation_row_ids) - - prior_raw = ( - evaluation.metric_clusters - if isinstance(evaluation.metric_clusters, dict) - else {} - ) - policy_blob: Dict[str, Any] = {} - if failure_policies: - policy_blob = failure_policies_to_db(failure_policies, source="user") - - evaluation.metric_clusters = { - "status": "running", - "groups": prior_raw.get("groups", []) if isinstance(prior_raw, dict) else [], - "discovered_problems": ( - prior_raw.get("discovered_problems", []) - if isinstance(prior_raw, dict) - else [] - ), - "generated_at": datetime.now(timezone.utc).isoformat(), - "generated_at_completed_rows": evaluation.completed_rows, - "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, - "provider": provider, - "model": model, - "max_llm_calls": llm_budget, - "llm_calls_used": 0, - "error_message": None, - "selected_evaluation_row_ids": selected_evaluation_row_ids or [], - **policy_blob, - } - if db is not None: - flag_modified(evaluation, "metric_clusters") - db.commit() - - from app.workers.tasks.generate_evaluation_metric_clusters import ( - generate_evaluation_metric_clusters_task, - ) - - async_result = generate_evaluation_metric_clusters_task.apply_async( - kwargs={ - "evaluation_id": str(evaluation.id), - "provider": provider, - "model": model, - "credential_id": str(credential_id) if credential_id else None, - "max_llm_calls": llm_budget, - "evaluation_row_ids": row_ids_for_task, - }, - queue="imports", - ) - if db is not None and isinstance(evaluation.metric_clusters, dict): - evaluation.metric_clusters["celery_task_id"] = async_result.id - flag_modified(evaluation, "metric_clusters") - db.commit() - - -def _revoke_metric_clusters_task(evaluation: CallImportEvaluation) -> None: - """Best-effort SIGTERM revoke of the in-flight clustering Celery task.""" - raw = evaluation.metric_clusters - if not isinstance(raw, dict): - return - task_id = str(raw.get("celery_task_id") or "").strip() - if not task_id: - return - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(task_id, terminate=True, signal="SIGTERM") - logger.info( - "Revoked metric-clusters task {} for evaluation {}", - task_id, - evaluation.id, - ) - except Exception as exc: # noqa: BLE001 - logger.warning( - "Failed to revoke metric-clusters task {} for evaluation {}: {}", - task_id, - evaluation.id, - exc, - ) - - -def _apply_metric_clusters_cancel(evaluation: CallImportEvaluation) -> bool: - """Mark clustering as cancelled and revoke the worker task. - - Returns True if a running job was cancelled, False if already terminal. - """ - raw = evaluation.metric_clusters - if not isinstance(raw, dict): - return False - if (raw.get("status") or "").lower() != "running": - return False - - _revoke_metric_clusters_task(evaluation) - progress = raw.get("progress") if isinstance(raw.get("progress"), dict) else {} - evaluation.metric_clusters = { - **raw, - "status": "cancelled", - "error_message": METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, - "progress": progress, - "celery_task_id": None, - } - return True - - -@router.get( - "/{eval_id}/metric-clusters/failure-policies", - response_model=MetricFailurePoliciesResponse, - operation_id="getCallImportEvaluationMetricClusterFailurePolicies", -) -async def get_call_import_evaluation_metric_cluster_failure_policies( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> MetricFailurePoliciesResponse: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - metrics, aggregates, policies, source, child_names_by_parent = _clustering_context( - db, evaluation, eval_rows - ) - previews = build_failure_policy_previews( - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - effective=policies, - ) - updated_at = None - raw_mc = evaluation.metric_clusters - if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): - try: - updated_at = datetime.fromisoformat( - str(raw_mc["failure_policies_updated_at"]) - ) - except ValueError: - updated_at = None - return MetricFailurePoliciesResponse( - previews=previews, - policies=policies, - source=source, - updated_at=updated_at, - ) - - -@router.put( - "/{eval_id}/metric-clusters/failure-policies", - response_model=MetricFailurePoliciesResponse, - operation_id="saveCallImportEvaluationMetricClusterFailurePolicies", -) -async def save_call_import_evaluation_metric_cluster_failure_policies( - call_import_id: UUID, - eval_id: UUID, - body: MetricFailurePoliciesSaveRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> MetricFailurePoliciesResponse: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - metrics, aggregates, _existing, _source, child_names_by_parent = _clustering_context( - db, evaluation, eval_rows - ) - try: - validate_failure_policies_for_metrics(body.policies, metrics) - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - - prior = ( - evaluation.metric_clusters - if isinstance(evaluation.metric_clusters, dict) - else {} - ) - evaluation.metric_clusters = merge_failure_policies_into_raw( - prior, - body.policies, - source="user", - ) - flag_modified(evaluation, "metric_clusters") - db.commit() - db.refresh(evaluation) - - policies, source = policies_from_evaluation_raw(evaluation.metric_clusters) - if source != "user": - source = "user" - previews = build_failure_policy_previews( - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - effective=policies, - ) - updated_at = None - raw_mc = evaluation.metric_clusters - if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): - try: - updated_at = datetime.fromisoformat( - str(raw_mc["failure_policies_updated_at"]) - ) - except ValueError: - updated_at = None - return MetricFailurePoliciesResponse( - previews=previews, - policies=policies, - source="user", - updated_at=updated_at, - ) - - -@router.get( - "/{eval_id}/metric-clusters/eligible-rows", - response_model=MetricClusterEligibleRowsResponse, - operation_id="listCallImportEvaluationMetricClusterEligibleRows", -) -async def list_call_import_evaluation_metric_cluster_eligible_rows( - call_import_id: UUID, - eval_id: UUID, - limit: Optional[int] = Query(default=None, ge=1), - count_only: bool = Query(default=False), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> MetricClusterEligibleRowsResponse: - """Completed rows that have at least one flagged quality metric.""" - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - completed_pairs = _completed_row_pairs_for_evaluation(db, eval_id) - metrics, _aggregates, policies, _source, _child_map = _clustering_context( - db, evaluation, eval_rows - ) - all_eligible = list_eligible_cluster_rows( - evaluation, completed_pairs, metrics, policies - ) - total = len(all_eligible) - if count_only: - return MetricClusterEligibleRowsResponse(items=[], total=total) - raw_items = all_eligible if limit is None else all_eligible[:limit] - items = [MetricClusterEligibleRow.model_validate(item) for item in raw_items] - return MetricClusterEligibleRowsResponse(items=items, total=total) - - -@router.get( - "/{eval_id}/metric-clusters", - response_model=Optional[EvaluationMetricClustersState], - operation_id="getCallImportEvaluationMetricClusters", -) -async def get_call_import_evaluation_metric_clusters( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationMetricClustersState]: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _metric_clusters_payload(evaluation) - - -@router.post( - "/{eval_id}/metric-clusters", - response_model=EvaluationMetricClustersState, - operation_id="generateCallImportEvaluationMetricClusters", -) -async def generate_call_import_evaluation_metric_clusters( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationMetricClustersRequest = Body( - default_factory=EvaluationMetricClustersRequest - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> EvaluationMetricClustersState: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not body.regenerate and not body.force: - cached = _metric_clusters_payload(evaluation) - if cached is not None and cached.status in {"running", "completed"}: - return cached - - eval_rows = _load_eval_rows(db, eval_id) - if not any(row.status == "completed" for row in eval_rows): - raise HTTPException( - status_code=400, - detail=( - "No completed rows yet. Wait for at least one row to " - "finish scoring before generating metric clusters." - ), - ) - - if body.evaluation_row_ids and body.row_limit is not None: - raise HTTPException( - status_code=400, - detail="Specify either evaluation_row_ids or row_limit, not both.", - ) - - if body.evaluation_row_ids: - completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) - completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} - requested = {str(rid) for rid in body.evaluation_row_ids} - unknown = sorted(requested - completed_id_set) - if unknown: - raise HTTPException( - status_code=400, - detail=( - "One or more evaluation_row_ids are missing or not completed: " - + ", ".join(unknown[:5]) - + ("…" if len(unknown) > 5 else "") - ), - ) - - from app.services.ai.llm_resolver import get_llm_provider_and_model - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, body.provider, body.model, body.credential_id - ) - - metrics, aggregates, _inferred, _source, child_names_by_parent = _clustering_context( - db, evaluation, eval_rows - ) - merged_policies = merge_clustering_policies( - body.failure_policies, - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - try: - validate_failure_policies_for_metrics( - body.failure_policies or merged_policies, metrics - ) - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - - if not has_clusterable_metrics(metrics, merged_policies, eval_rows): - raise HTTPException( - status_code=400, - detail=( - "No calls match any failure policy. Select failure values on " - "metrics that have matching rows, or leave metrics with no " - "failures unchecked — they are skipped automatically." - ), - ) - - filtered_pairs, selected_row_ids = _resolve_metric_cluster_row_selection( - db, - evaluation, - eval_rows, - body.evaluation_row_ids, - row_limit=body.row_limit, - policies=merged_policies, - ) - if not selected_row_ids: - raise HTTPException( - status_code=400, - detail=( - "No eligible rows to cluster. Select completed calls that match " - "at least one configured failure policy." - ), - ) - if not filtered_pairs: - raise HTTPException( - status_code=400, - detail="No completed rows match the selected evaluation_row_ids.", - ) - - _enqueue_metric_clusters_job( - evaluation, - provider=provider_enum.value, - model=model_str, - credential_id=body.credential_id, - force=body.force or body.regenerate, - max_llm_calls=body.max_llm_calls, - evaluation_row_ids=body.evaluation_row_ids, - selected_evaluation_row_ids=selected_row_ids, - failure_policies=merged_policies, - db=db, - ) - - db.refresh(evaluation) - return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( - status="running" - ) - - -@router.post( - "/{eval_id}/metric-clusters/cancel", - response_model=EvaluationMetricClustersState, - operation_id="cancelCallImportEvaluationMetricClusters", -) -async def cancel_call_import_evaluation_metric_clusters( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> EvaluationMetricClustersState: - """Abort in-flight failure-diagnostics clustering. - - Idempotent: if clustering is not ``running``, returns the current state - unchanged. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - _apply_metric_clusters_cancel(evaluation) - flag_modified(evaluation, "metric_clusters") - db.commit() - db.refresh(evaluation) - - return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( - status="idle" - ) - - -@router.get( - "/{eval_id}/prompt-improvements", - response_model=Optional[EvaluationPromptImprovementsState], - operation_id="getCallImportEvaluationPromptImprovements", -) -async def get_call_import_evaluation_prompt_improvements( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationPromptImprovementsState]: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _prompt_improvements_payload(evaluation) - - -@router.post( - "/{eval_id}/prompt-improvements", - response_model=EvaluationPromptImprovementsState, - operation_id="generateCallImportEvaluationPromptImprovements", -) -async def generate_call_import_evaluation_prompt_improvements( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationPromptImprovementsRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> EvaluationPromptImprovementsState: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - clusters = _metric_clusters_payload(evaluation) - if clusters is None or clusters.status != "completed": - raise HTTPException( - status_code=400, - detail=( - "Metric clusters must be completed before generating prompt " - "improvements. Run failure diagnostics first." - ), - ) - - from app.services.call_import_prompt_improvements import is_imported_agent - from app.services.ai.llm_resolver import get_llm_provider_and_model - - imported_agent = ( - db.query(PromptPartial) - .filter( - PromptPartial.id == body.imported_agent_id, - PromptPartial.organization_id == organization_id, - PromptPartial.workspace_id == workspace_id, - ) - .first() - ) - if imported_agent is None or not is_imported_agent(imported_agent): - raise HTTPException( - status_code=404, - detail="Imported agent not found in the active workspace", - ) - - if not body.regenerate and not body.force: - cached = _prompt_improvements_payload(evaluation) - if ( - cached is not None - and cached.status in {"running", "completed"} - and cached.imported_agent_id == str(body.imported_agent_id) - ): - return cached - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, body.provider, body.model, body.credential_id - ) - - _enqueue_prompt_improvements_job( - evaluation, - imported_agent_id=body.imported_agent_id, - imported_agent_name=imported_agent.name, - provider=provider_enum.value, - model=model_str, - credential_id=body.credential_id, - force=body.force or body.regenerate, - db=db, - ) - - db.refresh(evaluation) - return _prompt_improvements_payload(evaluation) or EvaluationPromptImprovementsState( - status="running", - imported_agent_id=str(body.imported_agent_id), - imported_agent_name=imported_agent.name, - ) - - -# --------------------------------------------------------------------------- -# Flow chart: turns per-row LLM-inferred ``sequence`` arrays into a -# directed graph of (label -> label) transitions across the whole run. -# Powers the aggregate Sankey-style React Flow chart on the evaluation -# overview; per-call flow charts are built client-side from the same -# ``sequence`` field on a single row's metric_scores entry. -# --------------------------------------------------------------------------- - - -_FLOW_TERMINAL_THRESHOLD = 0.2 # Mark as terminal when >=20% of sequences end here. -_FLOW_START_NODE_ID = "__START__" -_DISCOVERED_NODE_PREFIX = "disc:" - - -def _slug_label(value: Any) -> str: - """Lowercase + whitespace-collapse + underscore-join. - - Used everywhere we need a stable key for a metric/label name — - matching the same convention the worker uses when emitting - ``sequence`` entries and discovered keys. - """ - if value is None: - return "" - return "_".join(str(value).strip().lower().split()) - - -def _resolve_alias(alias_map: Dict[str, str], key: str) -> str: - """Walk the alias map until we hit a slug that doesn't redirect. - - The merge endpoint stores ``from_slug -> to_slug`` pairs. The delete - endpoint stores ``from_slug -> ""`` (empty string sentinel) to mark - a slug as tombstoned. Chains can accumulate when the user merges - A→B and later merges B→C; this helper collapses them so callers - always land on the final canonical slug. - - Returns: - * the canonical slug if it still resolves to a real label, - * an empty string if the slug has been tombstoned (callers MUST - treat an empty result as "drop this entry entirely"), - * the input ``key`` if it isn't aliased. - - Cycles are guarded by a hard step limit since the alias map is - user-driven. - """ - if not key: - return "" - if not alias_map: - return key - current = key - seen: set[str] = set() - for _ in range(16): - if current in seen: - return current - seen.add(current) - if current not in alias_map: - return current - nxt = alias_map[current] - if nxt == current: - return current - if nxt == "": - # Deletion sentinel — the user has explicitly retired this - # slug. Propagate the empty string up so callers drop it. - return "" - current = nxt - return current - - -# Reserved JSON key under which the worker stores top-level metric -# discoveries on each row's ``metric_scores`` dict. Mirrors the constant -# in ``app/workers/tasks/helpers/llm_evaluation.py`` — kept local here to -# avoid a worker import cycle from the routes module. -DISCOVERED_METRICS_KEY = "__discovered_metrics__" - -# Allowed values for an LLM-suggested top-level metric type. Kept in -# sync with ``DiscoveredMetricSuggestedType`` in -# ``app/models/schemas.py``. -_DISCOVERED_METRIC_TYPES = ("boolean", "rating", "category") - - -def normalize_scores_with_aliases( - metric_scores: Dict[str, Any], - evaluation: CallImportEvaluation, - db: Session, - organization_id: UUID, -) -> Dict[str, Any]: - """Rewrite per-row ``metric_scores`` to honor merges + promotions. - - Called by the worker right after ``evaluate_with_llm`` returns so - every row that finishes AFTER a user has merged or promoted a - discovered label persists data already reflecting that decision. - Without this hook, a worker holding a stale prompt could re-emit a - ``from_key`` slug long after the user merged it away. - - For every parent entry (``selection_mode != null`` and a - ``discovered_labels`` / ``sequence`` field) we: - - * resolve discovered slugs through the evaluation's - ``discovered_label_aliases`` map (transitively), - * drop any discovered_labels entry whose canonical slug now - matches a real promoted child of the parent (merging them out - of the panel for free), and - * collapse adjacent duplicate sequence entries that result. - - Returns ``metric_scores`` (mutated in place) for chaining. - """ - if not isinstance(metric_scores, dict): - return metric_scores - - aliases_top = ( - evaluation.discovered_label_aliases - if isinstance(evaluation.discovered_label_aliases, dict) - else {} - ) - - # Identify the parent entries inside metric_scores. They're the - # dicts that carry a ``selection_mode`` key (set by the LLM - # hierarchy parser) and either a ``sequence`` or a - # ``discovered_labels`` list. - for key, entry in list(metric_scores.items()): - if not isinstance(entry, dict): - continue - if entry.get("type") != "category" and not entry.get("selection_mode"): - continue - try: - parent_uuid = UUID(str(key)) - except (TypeError, ValueError): - continue - - alias_map = {} - sub = aliases_top.get(str(parent_uuid)) - if isinstance(sub, dict): - alias_map = { - str(k): str(v) - for k, v in sub.items() - if isinstance(k, str) and isinstance(v, str) - } - promoted = _promoted_child_slugs(db, parent_uuid, organization_id) - - # Rewrite discovered_labels: alias-resolve keys, drop duplicates - # post-resolution, and drop entries that have been promoted. - discovered = entry.get("discovered_labels") - if isinstance(discovered, list): - kept_disc: List[Dict[str, Any]] = [] - seen: set[str] = set() - for d in discovered: - if not isinstance(d, dict): - continue - slug = _slug_label(d.get("key") or d.get("name")) - slug = _resolve_alias(alias_map, slug) - if not slug or slug in promoted or slug in seen: - continue - seen.add(slug) - new_entry = dict(d) - new_entry["key"] = slug - kept_disc.append(new_entry) - entry["discovered_labels"] = kept_disc - - # Rewrite sequence: alias-resolve every entry; collapse adjacent - # duplicates that result. We DON'T drop slugs that match - # promoted children — the promoted child slug is still a valid - # sequence entry; the flow chart will resolve it to the real - # child node. - seq = entry.get("sequence") - if isinstance(seq, list): - new_seq: List[str] = [] - last: Optional[str] = None - for item in seq: - if not isinstance(item, str): - continue - slug = _resolve_alias(alias_map, _slug_label(item)) - if not slug or slug == last: - continue - new_seq.append(slug) - last = slug - entry["sequence"] = new_seq - - # Top-level metric discoveries live alongside the parent entries - # under the reserved ``DISCOVERED_METRICS_KEY`` slot. Apply the - # flat evaluation-level alias/tombstone map + suppress slugs that - # already correspond to a real top-level Metric so workers that - # finish AFTER the user has merged / deleted / promoted can't - # resurrect a retired candidate. - discovered_metrics_payload = metric_scores.get(DISCOVERED_METRICS_KEY) - if isinstance(discovered_metrics_payload, list): - flat_alias_map = ( - evaluation.discovered_metric_aliases - if isinstance(evaluation.discovered_metric_aliases, dict) - else {} - ) - promoted_metric_slugs = _promoted_top_level_metric_slugs( - db, organization_id - ) - kept_metrics: List[Dict[str, Any]] = [] - seen_metrics: set[str] = set() - for d in discovered_metrics_payload: - if not isinstance(d, dict): - continue - slug = _slug_label(d.get("key") or d.get("name")) - slug = _resolve_alias(flat_alias_map, slug) - if ( - not slug - or slug in promoted_metric_slugs - or slug in seen_metrics - ): - continue - seen_metrics.add(slug) - new_entry = dict(d) - new_entry["key"] = slug - kept_metrics.append(new_entry) - if kept_metrics: - metric_scores[DISCOVERED_METRICS_KEY] = kept_metrics - else: - # No survivors — drop the empty array so empty-discovery rows - # keep their pre-feature payload shape. - metric_scores.pop(DISCOVERED_METRICS_KEY, None) - - return metric_scores - - -def _alias_map_for_parent( - evaluation: CallImportEvaluation, parent_metric_id: UUID -) -> Dict[str, str]: - """Pull ``{from_slug: to_slug}`` for one parent out of the eval's blob. - - Stored shape on the evaluation row is - ``{parent_id_str: {from_slug: to_slug, ...}}``. Returns an empty - dict for parents that have never had a merge applied. - """ - raw = getattr(evaluation, "discovered_label_aliases", None) - if not isinstance(raw, dict): - return {} - submap = raw.get(str(parent_metric_id)) - if not isinstance(submap, dict): - return {} - return { - str(k): str(v) - for k, v in submap.items() - if isinstance(k, str) and isinstance(v, str) - } - - -def _promoted_child_slugs( - db: Session, parent_metric_id: UUID, organization_id: UUID -) -> set[str]: - """Slugs of every real child currently sitting under the parent. - - The Discovered Labels panel hides any candidate whose slug already - matches a real child — that covers both freshly-promoted candidates - and legacy children the LLM happened to re-discover. We pull from - the live ``metrics`` table rather than the eval's - ``selected_metric_groups`` snapshot so newly-promoted children take - effect immediately, even on evaluations that ran before the - promotion. - """ - children = ( - db.query(Metric.name) - .filter( - Metric.parent_metric_id == parent_metric_id, - Metric.organization_id == organization_id, - ) - .all() - ) - out: set[str] = set() - for (name,) in children: - slug = _slug_label(name) - if slug: - out.add(slug) - return out - - -def _promoted_top_level_metric_slugs( - db: Session, organization_id: UUID -) -> set[str]: - """Slugs of every top-level (non-child) Metric in the organization. - - Used to suppress discovered-metric candidates whose slug already - matches a real standalone metric. We intentionally include both - standalone metrics AND parent category metrics — a top-level - discovery that collides with either name is a duplicate by - definition. - """ - rows = ( - db.query(Metric.name) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id.is_(None), - ) - .all() - ) - out: set[str] = set() - for (name,) in rows: - slug = _slug_label(name) - if slug: - out.add(slug) - return out - - -def _get_running_discovered_labels( - db: Session, - eval_id: UUID, - parent_metric_id: UUID, - organization_id: Optional[UUID] = None, - alias_map: Optional[Dict[str, str]] = None, -) -> List[Dict[str, Any]]: - """Slug-deduped view of every discovered label seen in this eval so far. - - Walks each ``call_import_evaluation_rows`` row's - ``metric_scores[parent_id]["discovered_labels"]`` and folds entries - that share the same slug. Returns a list ordered by descending - count and stable on label key, shaped like:: - - [{"key": "customer_on_hold", "name": "Customer put on hold", - "description": "...", "sample_rationale": "...", "count": 12}] - - Powers two callers: - * The worker prompt builder ("REUSE the existing key if it fits") - — invoked just before each row's LLM call to feed the model the - running list of previously-discovered labels in this evaluation. - * The ``/discovered-labels`` API surface used by the frontend - Discovered Labels panel to render candidates with counts + - sample rationales. - - Non-completed rows are skipped: an in-flight row's discoveries are - not yet reliable (the row could fail and never produce final - metric_scores). We accept the tradeoff that rows running - concurrently won't see each other's labels — slug-collision dedup - catches identical re-inventions, and near-paraphrases surface in - the UI panel where the user can manually merge. - """ - - parent_id_str = str(parent_metric_id) - from app.db_sharding.eval_rows import load_evaluation_rows_for_run - - eval_rows = load_evaluation_rows_for_run(db, eval_id) - rows = [ - (row.metric_scores,) - for row in eval_rows - if row.status == CallImportRowStatus.COMPLETED.value - ] - - # Suppress slugs that have either: - # * been promoted to a real child of the parent (so the panel doesn't - # keep nagging the user about a candidate they've already - # accepted), or - # * been merged INTO another slug (the "from" side of a merge) — - # those occurrences fold into the canonical target instead. - promoted_slugs: set[str] = set() - if organization_id is not None: - promoted_slugs = _promoted_child_slugs( - db, parent_metric_id, organization_id - ) - aliases = alias_map or {} - - by_key: Dict[str, Dict[str, Any]] = {} - for (scores,) in rows: - if not isinstance(scores, dict): - continue - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - continue - discovered = parent_entry.get("discovered_labels") - if not isinstance(discovered, list): - continue - for entry in discovered: - if not isinstance(entry, dict): - continue - raw_key = entry.get("key") or entry.get("name") - key = _slug_label(raw_key) - if not key: - continue - # Apply user merges + deletions first, THEN drop anything - # that ended up on a real child slug. Order matters: a - # candidate that was merged into a slug which has since - # been promoted should disappear, not show up at the - # canonical slug. An empty resolved key means the slug was - # tombstoned via the delete endpoint. - key = _resolve_alias(aliases, key) - if not key or key in promoted_slugs: - continue - name = (entry.get("name") or "").strip() or key.replace("_", " ") - description = (entry.get("description") or "").strip() or None - sample = (entry.get("rationale") or "").strip() or None - - existing = by_key.get(key) - if existing is None: - # Track up to N=3 distinct rationales per candidate so - # the Promote-to-child flow can pre-fill the new - # sub-metric's rubric with concrete LLM examples - # without the user copy-pasting from the row table. - # ``sample_rationale`` is preserved for back-compat - # with older clients; ``examples`` is the new field. - examples = [sample] if sample else [] - by_key[key] = { - "key": key, - "name": name, - "description": description, - "sample_rationale": sample, - "examples": examples, - "count": 1, - } - continue - - existing["count"] += 1 - if not existing["description"] and description: - existing["description"] = description - if not existing["sample_rationale"] and sample: - existing["sample_rationale"] = sample - # Append distinct rationales (case-insensitive trim) up - # to a small cap. Headroom is intentionally one above - # what the UI surfaces (2) so we have a backup when the - # first rationale is unhelpful. - if sample: - ex_list: List[str] = existing.setdefault("examples", []) - if len(ex_list) < 3 and not any( - s.strip().lower() == sample.strip().lower() for s in ex_list - ): - ex_list.append(sample) - - return sorted( - by_key.values(), - key=lambda item: (-item["count"], item["key"]), - ) - - -def _get_running_discovered_metrics( - db: Session, - eval_id: UUID, - organization_id: Optional[UUID] = None, - alias_map: Optional[Dict[str, str]] = None, -) -> List[Dict[str, Any]]: - """Slug-deduped view of every discovered top-level metric in this eval. - - Mirrors :func:`_get_running_discovered_labels` but is keyed at the - evaluation level (no ``parent_metric_id``). Walks each completed - row's ``metric_scores[DISCOVERED_METRICS_KEY]`` list, folds entries - that share the same slug (post-alias resolution), and suppresses - slugs that already correspond to a real top-level :class:`Metric` - in the organization. - - Each returned entry is shaped:: - - {"key": "customer_satisfaction", - "name": "Customer Satisfaction", - "description": "...", - "suggested_type": "boolean" | "rating" | "category", - "sample_rationale": "...", - "examples": ["..."], - "count": 12} - """ - - from app.db_sharding.eval_rows import load_evaluation_rows_for_run - - eval_rows = load_evaluation_rows_for_run(db, eval_id) - rows = [ - (row.metric_scores,) - for row in eval_rows - if row.status == CallImportRowStatus.COMPLETED.value - ] - - promoted_slugs: set[str] = set() - if organization_id is not None: - promoted_slugs = _promoted_top_level_metric_slugs( - db, organization_id - ) - aliases = alias_map or {} - - by_key: Dict[str, Dict[str, Any]] = {} - for (scores,) in rows: - if not isinstance(scores, dict): - continue - discovered = scores.get(DISCOVERED_METRICS_KEY) - if not isinstance(discovered, list): - continue - for entry in discovered: - if not isinstance(entry, dict): - continue - raw_key = entry.get("key") or entry.get("name") - key = _slug_label(raw_key) - if not key: - continue - # Apply user merges + deletions first, THEN drop anything - # that ended up on an already-existing top-level metric - # slug. Empty resolved key = tombstoned. - key = _resolve_alias(aliases, key) - if not key or key in promoted_slugs: - continue - name = (entry.get("name") or "").strip() or key.replace( - "_", " " - ) - description = (entry.get("description") or "").strip() or None - sample = (entry.get("rationale") or "").strip() or None - raw_type = str(entry.get("suggested_type") or "").strip().lower() - if raw_type not in _DISCOVERED_METRIC_TYPES: - raw_type = "boolean" - - existing = by_key.get(key) - if existing is None: - examples = [sample] if sample else [] - by_key[key] = { - "key": key, - "name": name, - "description": description, - "suggested_type": raw_type, - "sample_rationale": sample, - "examples": examples, - "count": 1, - } - continue - - existing["count"] += 1 - if not existing["description"] and description: - existing["description"] = description - if not existing["sample_rationale"] and sample: - existing["sample_rationale"] = sample - # Keep the most-frequently-suggested type. We don't track - # per-type frequency yet; defer to the first non-default - # type encountered when the existing entry has the default. - if existing.get("suggested_type") == "boolean" and raw_type != "boolean": - existing["suggested_type"] = raw_type - if sample: - ex_list: List[str] = existing.setdefault("examples", []) - if len(ex_list) < 3 and not any( - s.strip().lower() == sample.strip().lower() for s in ex_list - ): - ex_list.append(sample) - - return sorted( - by_key.values(), - key=lambda item: (-item["count"], item["key"]), - ) - - -def _build_flow_graph( - eval_rows: List[CallImportEvaluationRow], - parent_metric: Metric, - children: List[Metric], - alias_map: Optional[Dict[str, str]] = None, - extra_children: Optional[List[Metric]] = None, -) -> MetricFlowResponse: - """Walk per-row ``sequence`` arrays and produce aggregate nodes/edges. - - A synthetic ``START`` node is prepended to every sequence so the - diagram has a single origin. Children that never appear in any - sequence are still emitted as nodes (count=0) so the UI can render - them in the legend. - - ``alias_map`` lets callers fold merged-out discovered slugs into - their canonical target before building the graph; ``extra_children`` - are children of the parent that aren't in the legend list (e.g. - children promoted *after* the evaluation was created and therefore - missing from ``selected_metric_groups``) but should still resolve in - sequences so the slug doesn't get redrawn as a discovered candidate. - """ - parent_id_str = str(parent_metric.id) - aliases = alias_map or {} - # Build a fast lookup keyed by both the lower_snake child key (what the - # LLM emits in ``sequence``) and the child UUID (what some clients may - # store) so legacy / drifted payloads still resolve. - child_lookup: Dict[str, Metric] = {} - for child in children: - slug = _slug_label(child.name) - child_lookup[slug] = child - child_lookup[str(child.id)] = child - # ``extra_children`` are resolved-only — they shouldn't add legend - # nodes (those come from the explicit ``children`` argument), but - # they need to be in ``child_lookup`` so a sequence step that - # matches a freshly-promoted child resolves to the real child UUID - # instead of falling through to ``discovered_lookup`` and rendering - # as a "discovered" node. - if extra_children: - for child in extra_children: - slug = _slug_label(child.name) - if slug and slug not in child_lookup: - child_lookup[slug] = child - cid = str(child.id) - child_lookup.setdefault(cid, child) - - # Discovered labels: walk every row's discovered_labels first so we - # know which discovered slugs are valid before resolving sequences. - # Discovered nodes get a ``disc:`` prefixed id so they can't collide - # with real child UUIDs in the node/edge graph. We apply - # ``alias_map`` first so merged-out source slugs fold into their - # canonical target — preserving the user's "merge" intent on still- - # in-flight rows whose JSON wasn't rewritten by the merge endpoint. - discovered_lookup: Dict[str, Dict[str, Any]] = {} - for row in eval_rows: - scores = ( - row.metric_scores if isinstance(row.metric_scores, dict) else {} - ) - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - continue - raw_discovered = parent_entry.get("discovered_labels") - if not isinstance(raw_discovered, list): - continue - for entry in raw_discovered: - if not isinstance(entry, dict): - continue - slug = _slug_label(entry.get("key") or entry.get("name")) - slug = _resolve_alias(aliases, slug) - if not slug or slug in child_lookup: - continue - name = (entry.get("name") or "").strip() or slug.replace("_", " ") - existing = discovered_lookup.get(slug) - if existing is None: - discovered_lookup[slug] = { - "id": f"{_DISCOVERED_NODE_PREFIX}{slug}", - "name": name, - } - - node_counts: Dict[str, int] = {} - edge_counts: Dict[tuple[str, str], int] = {} - terminal_counts: Dict[str, int] = {} - - total_rows = len(eval_rows) - rows_with_sequence = 0 - - for row in eval_rows: - scores = ( - row.metric_scores if isinstance(row.metric_scores, dict) else {} - ) - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - continue - raw_sequence = parent_entry.get("sequence") - if not isinstance(raw_sequence, list): - continue - - resolved_ids: List[str] = [] - last_resolved: Optional[str] = None - for item in raw_sequence: - if not isinstance(item, str): - continue - normalized = _resolve_alias(aliases, _slug_label(item)) - child = child_lookup.get(normalized) or child_lookup.get(item) - if child is not None: - cid = str(child.id) - # Adjacent dedupe AFTER alias resolution so two - # different raw slugs that fold to the same target - # don't draw a self-edge through the chart. - if cid == last_resolved: - continue - resolved_ids.append(cid) - last_resolved = cid - continue - disc = discovered_lookup.get(normalized) - if disc is not None: - if disc["id"] == last_resolved: - continue - resolved_ids.append(disc["id"]) - last_resolved = disc["id"] - - if not resolved_ids: - continue - - rows_with_sequence += 1 - for nid in resolved_ids: - node_counts[nid] = node_counts.get(nid, 0) + 1 - - edge_counts[(_FLOW_START_NODE_ID, resolved_ids[0])] = ( - edge_counts.get((_FLOW_START_NODE_ID, resolved_ids[0]), 0) + 1 - ) - for src, tgt in zip(resolved_ids, resolved_ids[1:]): - if src == tgt: - continue - edge_counts[(src, tgt)] = edge_counts.get((src, tgt), 0) + 1 - - terminal_id = resolved_ids[-1] - terminal_counts[terminal_id] = terminal_counts.get(terminal_id, 0) + 1 - - nodes: List[MetricFlowNode] = [] - # Always include a START node so the UI has a stable entry point. - nodes.append( - MetricFlowNode( - id=_FLOW_START_NODE_ID, - label="Start", - count=rows_with_sequence, - is_terminal=False, - ) - ) - - def _emit_child_node(child: Metric) -> None: - cid = str(child.id) - count = node_counts.get(cid, 0) - terminal_count = terminal_counts.get(cid, 0) - is_terminal = False - if rows_with_sequence > 0: - is_terminal = ( - terminal_count / rows_with_sequence - ) >= _FLOW_TERMINAL_THRESHOLD - nodes.append( - MetricFlowNode( - id=cid, - label=child.name, - count=count, - is_terminal=is_terminal, - ) - ) - - emitted_child_ids: set[str] = set() - for child in children: - cid = str(child.id) - if cid in emitted_child_ids: - continue - emitted_child_ids.add(cid) - _emit_child_node(child) - # Extra children (promoted after the eval was created) only get - # legend nodes if they actually appear in the data — otherwise we'd - # pollute the diagram with every standalone promotion the user has - # ever made under this parent. - if extra_children: - for child in extra_children: - cid = str(child.id) - if cid in emitted_child_ids: - continue - if node_counts.get(cid, 0) == 0: - continue - emitted_child_ids.add(cid) - _emit_child_node(child) - # Append discovered nodes after the real children so legend ordering - # keeps user-defined labels first. - for slug, info in discovered_lookup.items(): - nid = info["id"] - count = node_counts.get(nid, 0) - terminal_count = terminal_counts.get(nid, 0) - is_terminal = False - if rows_with_sequence > 0: - is_terminal = ( - terminal_count / rows_with_sequence - ) >= _FLOW_TERMINAL_THRESHOLD - nodes.append( - MetricFlowNode( - id=nid, - label=info["name"], - count=count, - is_terminal=is_terminal, - is_discovered=True, - ) - ) - - edges: List[MetricFlowEdge] = [ - MetricFlowEdge(source=src, target=tgt, count=count) - for (src, tgt), count in sorted( - edge_counts.items(), key=lambda kv: kv[1], reverse=True - ) - ] - - return MetricFlowResponse( - parent_metric_id=parent_id_str, - parent_metric_name=parent_metric.name, - selection_mode=parent_metric.selection_mode, - nodes=nodes, - edges=edges, - total_rows=total_rows, - rows_with_sequence=rows_with_sequence, - ) - - -@router.get( - "/{eval_id}/flow", - response_model=MetricFlowResponse, - operation_id="getCallImportEvaluationFlow", -) -async def get_call_import_evaluation_flow( - call_import_id: UUID, - eval_id: UUID, - parent_metric_id: UUID = Query( - ..., - description=( - "Parent (category) metric whose children's sequences should be " - "aggregated into a flow graph." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> MetricFlowResponse: - """Aggregate the LLM-inferred per-row sequences into one flow graph. - - Returns ``nodes`` (one per child of the parent metric, plus a - synthetic ``START`` node) and ``edges`` (counts of consecutive - label transitions across every row that produced a sequence). The - frontend feeds this directly into a React Flow / xyflow canvas; - edge thickness should scale with ``count / total_rows`` and - ``is_terminal`` nodes should be styled as outcomes. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - if not parent.selection_mode: - raise HTTPException( - status_code=400, - detail=( - "Flow charts are only meaningful for parent metrics " - "(selection_mode set). This metric is standalone." - ), - ) - - # Children are taken from selected_metric_groups when present so the - # flow chart reflects exactly the subset that ran in this - # evaluation; otherwise fall back to every enabled child of the - # parent. - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - parent_id_str = str(parent.id) - children: List[Metric] = [] - if parent_id_str in groups_raw and isinstance( - groups_raw[parent_id_str], list - ): - child_ids: List[UUID] = [] - for c in groups_raw[parent_id_str]: - try: - child_ids.append(UUID(str(c))) - except (TypeError, ValueError): - continue - if child_ids: - children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.id.in_(child_ids), - ) - .order_by(Metric.created_at.asc()) - .all() - ) - if not children: - children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id == parent.id, - ) - .order_by(Metric.created_at.asc()) - .all() - ) - - # Children promoted AFTER this evaluation was created aren't in - # ``selected_metric_groups`` but their slugs still appear in already- - # scored rows' sequences. Pass them as ``extra_children`` so those - # sequence entries resolve against the real (now promoted) child - # instead of being redrawn as discovered candidates. - extra_children: List[Metric] = [] - if children: - existing_ids = {child.id for child in children} - all_children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id == parent.id, - ) - .all() - ) - extra_children = [c for c in all_children if c.id not in existing_ids] - - eval_rows = _load_eval_rows(db, eval_id) - - alias_map = _alias_map_for_parent(evaluation, parent.id) - return _build_flow_graph( - eval_rows, - parent, - children, - alias_map=alias_map, - extra_children=extra_children, - ) - - -@router.get( - "/{eval_id}/discovered-labels", - response_model=DiscoveredLabelsResponse, - operation_id="getCallImportEvaluationDiscoveredLabels", -) -async def get_call_import_evaluation_discovered_labels( - call_import_id: UUID, - eval_id: UUID, - parent_metric_id: UUID = Query( - ..., - description=( - "Parent (category) metric whose LLM-discovered candidate " - "sub-labels should be aggregated across rows." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredLabelsResponse: - """Aggregate candidate sub-labels the LLM discovered during this eval. - - Only meaningful for parents with ``allow_discovery=true``; for other - parents we just return an empty ``items`` list rather than 400-ing - so the frontend can call the endpoint unconditionally for every - parent on the Flow tab without branching. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - - alias_map = _alias_map_for_parent(evaluation, parent_metric_id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - parent_metric_id, - organization_id=organization_id, - alias_map=alias_map, - ) - items = [DiscoveredLabelItem(**item) for item in items_raw] - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), items=items - ) - - -@router.post( - "/{eval_id}/discovered-labels/merge", - response_model=DiscoveredLabelsResponse, - operation_id="mergeCallImportEvaluationDiscoveredLabels", -) -async def merge_call_import_evaluation_discovered_labels( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredLabelMergeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredLabelsResponse: - """Rewrite every row's ``discovered_labels`` entry from from_key -> to_key. - - Idempotent — re-merging the same pair is a no-op. Discovered slugs - inside per-row ``sequence`` arrays are also rewritten so the flow - chart stays consistent with the panel. When a row already has - ``to_key`` and we're merging ``from_key`` into it, we drop the - ``from_key`` entry instead of producing two entries with the same - slug. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == body.parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - - from_key = _slug_label(body.from_key) - to_key = _slug_label(body.to_key) - if not from_key or not to_key: - raise HTTPException( - status_code=400, - detail="from_key and to_key must be non-empty slugs.", - ) - if from_key == to_key: - # No-op; just return the current aggregate so the client can - # refresh its view. - alias_map_existing = _alias_map_for_parent(evaluation, parent.id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - body.parent_metric_id, - organization_id=organization_id, - alias_map=alias_map_existing, - ) - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), - items=[DiscoveredLabelItem(**item) for item in items_raw], - ) - - parent_id_str = str(parent.id) - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _merge_discovered_label_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - return False - - mutated = False - discovered = parent_entry.get("discovered_labels") - if isinstance(discovered, list): - kept: List[Dict[str, Any]] = [] - existing_to = next( - ( - e - for e in discovered - if isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) == to_key - ), - None, - ) - for entry in discovered: - if not isinstance(entry, dict): - kept.append(entry) - continue - key = _slug_label(entry.get("key") or entry.get("name")) - if key == from_key: - if existing_to is not None: - mutated = True - continue - new_entry = dict(entry) - new_entry["key"] = to_key - kept.append(new_entry) - mutated = True - else: - kept.append(entry) - if mutated: - parent_entry["discovered_labels"] = kept - - seq = parent_entry.get("sequence") - if isinstance(seq, list): - new_seq: List[str] = [] - seq_changed = False - last_added: Optional[str] = None - for item in seq: - if isinstance(item, str) and _slug_label(item) == from_key: - seq_changed = True - if last_added == to_key: - continue - new_seq.append(to_key) - last_added = to_key - else: - new_seq.append(item) - last_added = ( - _slug_label(item) if isinstance(item, str) else None - ) - if seq_changed: - parent_entry["sequence"] = new_seq - mutated = True - - if mutated: - row.metric_scores = dict(scores) - return mutated - - foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_label_row) - - # Persist the merge at the evaluation level too. This is what makes - # the merge survive future scoring: rows that finish AFTER this - # call (e.g. retries, in-flight workers) will go through the - # alias map in the API surface even if the per-row JSON they - # write still mentions ``from_key``. We chain through any existing - # alias so merging A→B and then B→C resolves A→C in the panel. - raw_aliases = ( - evaluation.discovered_label_aliases - if isinstance(evaluation.discovered_label_aliases, dict) - else {} - ) - aliases_top = dict(raw_aliases) - parent_aliases = dict(aliases_top.get(parent_id_str) or {}) - # Resolve transitively: if to_key itself was previously merged into - # something else, point from_key at the canonical end-of-chain. - canonical_to = _resolve_alias(parent_aliases, to_key) - parent_aliases[from_key] = canonical_to - # Re-target any earlier aliases that pointed AT from_key — without - # this, A→B and then B→C would leave A still pointing to B (now a - # broken pointer because B is gone). Rewriting them keeps the - # alias map self-consistent. - for k, v in list(parent_aliases.items()): - if v == from_key: - parent_aliases[k] = canonical_to - aliases_top[parent_id_str] = parent_aliases - evaluation.discovered_label_aliases = aliases_top - - db.commit() - - alias_map_after = _alias_map_for_parent(evaluation, parent.id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - body.parent_metric_id, - organization_id=organization_id, - alias_map=alias_map_after, - ) - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), - items=[DiscoveredLabelItem(**item) for item in items_raw], - ) - - -@router.post( - "/{eval_id}/discovered-labels/delete", - response_model=DiscoveredLabelsResponse, - operation_id="deleteCallImportEvaluationDiscoveredLabel", -) -async def delete_call_import_evaluation_discovered_label( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredLabelDeleteRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredLabelsResponse: - """Tombstone a single LLM-discovered candidate for this evaluation. - - Symmetric with the merge endpoint, but instead of redirecting the - slug at another candidate we mark it as deleted. After this call: - - * the slug is stripped from every row's - ``metric_scores[parent].discovered_labels`` list, and from - every row's ``sequence`` array (so the flow chart no longer - draws a node for it); - * the slug is recorded in - ``evaluation.discovered_label_aliases[parent][slug] = ""`` - so any worker that finishes a row AFTER this call (e.g. a row - still in flight when the user clicked Delete) silently drops - the slug instead of resurrecting it. - - Idempotent: deleting an already-deleted slug is a no-op. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == body.parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - - target_key = _slug_label(body.key) - if not target_key: - raise HTTPException( - status_code=400, - detail="key must be a non-empty slug.", - ) - - parent_id_str = str(parent.id) - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _delete_discovered_label_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - return False - - mutated = False - discovered = parent_entry.get("discovered_labels") - if isinstance(discovered, list): - kept = [ - e - for e in discovered - if not ( - isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) - == target_key - ) - ] - if len(kept) != len(discovered): - parent_entry["discovered_labels"] = kept - mutated = True - - seq = parent_entry.get("sequence") - if isinstance(seq, list): - new_seq: List[str] = [] - seq_changed = False - last_added: Optional[str] = None - for item in seq: - if isinstance(item, str) and _slug_label(item) == target_key: - seq_changed = True - continue - if isinstance(item, str): - norm = _slug_label(item) - if norm == last_added: - seq_changed = True - continue - last_added = norm - new_seq.append(item) - if seq_changed: - parent_entry["sequence"] = new_seq - mutated = True - - if mutated: - row.metric_scores = dict(scores) - return mutated - - foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_label_row) - - # 3. Persist the tombstone on the evaluation so workers that finish - # later don't re-surface the deleted slug. We also retarget any - # existing aliases whose ``to_key`` was the deleted slug — without - # this, a previous merge that pointed at this slug would leave a - # dangling pointer. - raw_aliases = ( - evaluation.discovered_label_aliases - if isinstance(evaluation.discovered_label_aliases, dict) - else {} - ) - aliases_top = dict(raw_aliases) - parent_aliases = dict(aliases_top.get(parent_id_str) or {}) - parent_aliases[target_key] = "" # deletion sentinel - for k, v in list(parent_aliases.items()): - if v == target_key: - parent_aliases[k] = "" - aliases_top[parent_id_str] = parent_aliases - evaluation.discovered_label_aliases = aliases_top - - db.commit() - - alias_map_after = _alias_map_for_parent(evaluation, parent.id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - body.parent_metric_id, - organization_id=organization_id, - alias_map=alias_map_after, - ) - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), - items=[DiscoveredLabelItem(**item) for item in items_raw], - ) - - -# --------------------------------------------------------------------------- -# Discovered TOP-LEVEL METRICS (per-evaluation discovery) -# -# These endpoints are the parallel of the discovered-labels trio above but -# scoped to the evaluation as a whole instead of to a parent metric. They -# all live under ``/{eval_id}/discovered-metrics`` and operate on the -# reserved ``DISCOVERED_METRICS_KEY`` slot of each per-row -# ``metric_scores`` plus the flat ``CallImportEvaluation.discovered_metric_aliases`` -# map (no parent-id nesting). -# --------------------------------------------------------------------------- - - -def _flat_metric_aliases( - evaluation: CallImportEvaluation, -) -> Dict[str, str]: - """Pull the flat ``{from_slug: to_slug}`` map for an evaluation.""" - raw = getattr(evaluation, "discovered_metric_aliases", None) - if not isinstance(raw, dict): - return {} - return { - str(k): str(v) - for k, v in raw.items() - if isinstance(k, str) and isinstance(v, str) - } - - -@router.get( - "/{eval_id}/discovered-metrics", - response_model=DiscoveredMetricsResponse, - operation_id="getCallImportEvaluationDiscoveredMetrics", -) -async def get_call_import_evaluation_discovered_metrics( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredMetricsResponse: - """Aggregate top-level metric candidates the LLM discovered during this eval. - - Returns an empty ``items`` list when the evaluation did not opt - into top-level metric discovery; this keeps the frontend able to - call the endpoint unconditionally without branching on the - evaluation's ``discover_new_metrics`` flag. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not bool(getattr(evaluation, "discover_new_metrics", False)): - return DiscoveredMetricsResponse(evaluation_id=evaluation.id, items=[]) - - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - -@router.post( - "/{eval_id}/discovered-metrics/merge", - response_model=DiscoveredMetricsResponse, - operation_id="mergeCallImportEvaluationDiscoveredMetrics", -) -async def merge_call_import_evaluation_discovered_metrics( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredMetricMergeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredMetricsResponse: - """Rewrite every row's ``__discovered_metrics__`` entry from→to. - - Mirrors the discovered-labels merge endpoint but operates on the - flat top-level metric list. Idempotent — re-merging is a no-op. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - from_key = _slug_label(body.from_key) - to_key = _slug_label(body.to_key) - if not from_key or not to_key: - raise HTTPException( - status_code=400, - detail="from_key and to_key must be non-empty slugs.", - ) - if from_key == to_key: - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _merge_discovered_metric_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - discovered = scores.get(DISCOVERED_METRICS_KEY) - if not isinstance(discovered, list): - return False - - kept: List[Dict[str, Any]] = [] - mutated = False - existing_to = next( - ( - e - for e in discovered - if isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) == to_key - ), - None, - ) - for entry in discovered: - if not isinstance(entry, dict): - kept.append(entry) - continue - key = _slug_label(entry.get("key") or entry.get("name")) - if key == from_key: - if existing_to is not None: - mutated = True - continue - new_entry = dict(entry) - new_entry["key"] = to_key - kept.append(new_entry) - mutated = True - else: - kept.append(entry) - if mutated: - scores[DISCOVERED_METRICS_KEY] = kept - row.metric_scores = dict(scores) - return mutated - - foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_metric_row) - - raw_aliases = ( - evaluation.discovered_metric_aliases - if isinstance(evaluation.discovered_metric_aliases, dict) - else {} - ) - aliases = dict(raw_aliases) - canonical_to = _resolve_alias(aliases, to_key) - aliases[from_key] = canonical_to - for k, v in list(aliases.items()): - if v == from_key: - aliases[k] = canonical_to - evaluation.discovered_metric_aliases = aliases - - db.commit() - - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - -@router.post( - "/{eval_id}/discovered-metrics/delete", - response_model=DiscoveredMetricsResponse, - operation_id="deleteCallImportEvaluationDiscoveredMetric", -) -async def delete_call_import_evaluation_discovered_metric( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredMetricDeleteRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredMetricsResponse: - """Tombstone a single LLM-discovered top-level metric candidate.""" - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - target_key = _slug_label(body.key) - if not target_key: - raise HTTPException( - status_code=400, - detail="key must be a non-empty slug.", - ) - - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _delete_discovered_metric_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - discovered = scores.get(DISCOVERED_METRICS_KEY) - if not isinstance(discovered, list): - return False - kept = [ - e - for e in discovered - if not ( - isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) - == target_key - ) - ] - if len(kept) == len(discovered): - return False - if kept: - scores[DISCOVERED_METRICS_KEY] = kept - else: - scores.pop(DISCOVERED_METRICS_KEY, None) - row.metric_scores = dict(scores) - return True - - foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_metric_row) - - raw_aliases = ( - evaluation.discovered_metric_aliases - if isinstance(evaluation.discovered_metric_aliases, dict) - else {} - ) - aliases = dict(raw_aliases) - aliases[target_key] = "" # tombstone - for k, v in list(aliases.items()): - if v == target_key: - aliases[k] = "" - evaluation.discovered_metric_aliases = aliases - - db.commit() - - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - -@router.delete( - "/{eval_id}/rows/{eval_row_id}", - status_code=status.HTTP_204_NO_CONTENT, - operation_id="deleteCallImportEvaluationRow", -) -async def delete_call_import_evaluation_row( - call_import_id: UUID, - eval_id: UUID, - eval_row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Response: - """Delete a single per-row scoring entry within an evaluation run. - - Useful when the user wants to drop a noisy row before re-exporting - the CSV — e.g. a row whose audio was corrupt and skewed the - aggregate. Counters on the parent are recomputed so the rolled-up - status stays accurate. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - from app.db_sharding.eval_rows import delete_evaluation_row_on_shards - - if not delete_evaluation_row_on_shards(eval_row_id, eval_id): - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - _rollup_evaluation_status(evaluation, db) - db.commit() - return Response(status_code=status.HTTP_204_NO_CONTENT) - - eval_row = ( - db.query(CallImportEvaluationRow) - .filter( - CallImportEvaluationRow.id == eval_row_id, - CallImportEvaluationRow.evaluation_id == eval_id, - ) - .first() - ) - if not eval_row: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - - # If the row was still in flight, best-effort revoke the worker task - # so it doesn't try to write into a deleted DB row mid-execution. - if eval_row.celery_task_id and eval_row.status in {"pending", "running"}: - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(eval_row.celery_task_id, terminate=False) - except Exception: - pass - - db.delete(eval_row) - db.flush() - _rollup_evaluation_status(evaluation, db) - db.commit() - return Response(status_code=status.HTTP_204_NO_CONTENT) - - -# --------------------------------------------------------------------------- -# Retry endpoints -# --------------------------------------------------------------------------- -# -# The create endpoint enqueues every row of a fresh run; these endpoints -# let the user re-enqueue a *subset* of rows in an existing run — most -# commonly the ones that failed. We keep the worker contract identical -# (``evaluate_call_import_row_task(eval_row_id)``), so the retry path -# only has to reset row state and re-fan-out. When a row is missing its -# diarised transcript and the run was configured for diarised -# transcripts, we chain through ``transcribe_call_import_row_task`` the -# same way the create endpoint does — that's what makes "retry" feel -# like "just fix it" instead of "fail again immediately". - - -def _prepare_source_row_for_retry( - source_row: CallImportRow, - *, - transcribe_overwrite: bool, -) -> None: - """Clear stale diarisation markers so retry dispatch can re-run the pipeline.""" - source_row.celery_task_id = None - - # Re-fetch recordings when a prior import failed or stalled without S3 audio. - # Mirrors retry_failed_call_import_rows so eval retry can re-enqueue imports. - if ( - source_row.status - in (CallImportRowStatus.FAILED, CallImportRowStatus.PROCESSING) - and not (source_row.recording_s3_key or "").strip() - ): - source_row.status = CallImportRowStatus.PENDING - source_row.error_message = None - - if transcribe_overwrite and (source_row.diarised_transcript or "").strip(): - source_row.diarised_transcript = None - - has_dia = bool((source_row.diarised_transcript or "").strip()) - dia_status = (source_row.diarised_transcript_status or "").strip().lower() - - if has_dia and not transcribe_overwrite: - source_row.diarised_transcript_status = "completed" - source_row.diarised_transcript_error = None - return - - if dia_status in {"failed", "pending", "running", "idle"}: - source_row.diarised_transcript_status = "idle" - source_row.diarised_transcript_error = None - - -def _reset_eval_row_for_retry( - eval_row: CallImportEvaluationRow, - *, - metric_ids: Optional[List[UUID]] = None, - skip_revoke: bool = False, -) -> None: - """Wipe per-row state so the worker can re-run it cleanly. - - Mirrors the initial state used by ``create_call_import_evaluation`` - when it first inserts a row, with the addition of revoking any - lingering Celery task id. - - When ``metric_ids`` is provided, this is a **metric-subset retry**: - only the scores for those metrics are removed from - ``metric_scores`` (other metrics' previously-computed values are - preserved so the worker's partial-merge write keeps them intact). - Otherwise the entire ``metric_scores`` dict is reset, matching the - legacy behaviour. - """ - if ( - not skip_revoke - and eval_row.celery_task_id - and eval_row.status in {"pending", "running"} - ): - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(eval_row.celery_task_id, terminate=False) - except Exception: # noqa: BLE001 — revoke is best-effort - pass - eval_row.status = "pending" - eval_row.error_message = None - if metric_ids: - # Strip ONLY the targeted metric keys. Both string and UUID - # forms can appear in ``metric_scores`` depending on which - # code path wrote the dict, so we normalise to lower-case - # strings for the comparison. - existing = ( - eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} - ) - target_keys = {str(mid).lower() for mid in metric_ids} - eval_row.metric_scores = { - key: value - for key, value in existing.items() - if str(key).lower() not in target_keys - } - else: - eval_row.metric_scores = {} - eval_row.started_at = None - eval_row.finished_at = None - eval_row.celery_task_id = None - - -def _enqueue_eval_rows_with_optional_transcribe( - db: Session, - evaluation: CallImportEvaluation, - eval_rows_with_source: List[ - Tuple[CallImportEvaluationRow, CallImportRow] - ], - *, - transcribe_overwrite: bool = False, - restricted_metric_ids: Optional[List[UUID]] = None, -) -> Tuple[int, int]: - """Schedule throttled evaluation dispatch for pending eval rows. - - Returns ``(evaluate_only_count, transcribe_then_evaluate_count)`` for - logging/UI compatibility. Actual Celery fan-out is handled by - :func:`dispatch_evaluation_rows_task` under Redis fair-share limits. - """ - from app.workers.concurrency.eval_dispatch import _needs_transcribe_for_eval - from app.workers.concurrency.fair_dispatch import ( - schedule_fair_dispatch, - store_evaluation_transcribe_overwrite, - store_row_restricted_metrics, - ) - - eval_only_count = 0 - transcribe_count = 0 - if eval_rows_with_source: - for eval_row, source_row in eval_rows_with_source: - if _needs_transcribe_for_eval( - evaluation, - source_row, - transcribe_overwrite=transcribe_overwrite, - ): - transcribe_count += 1 - else: - eval_only_count += 1 - - restricted_metric_ids_str: Optional[List[str]] = ( - [str(mid) for mid in restricted_metric_ids] - if restricted_metric_ids - else None - ) - if restricted_metric_ids_str: - for eval_row, _ in eval_rows_with_source: - store_row_restricted_metrics(eval_row.id, restricted_metric_ids_str) - else: - restricted_metric_ids_str = ( - [str(mid) for mid in restricted_metric_ids] if restricted_metric_ids else None - ) - store_evaluation_transcribe_overwrite( - evaluation.id, - overwrite=transcribe_overwrite, - ) - schedule_fair_dispatch(max_workspace_turns=999) - return eval_only_count, transcribe_count - - -def _apply_telephony_retry_overrides( - db: Session, - *, - call_import: CallImport, - organization_id: UUID, - payload: CallImportEvaluationRetryRequest, -) -> None: - """Pin or clear telephony credentials on the batch for this retry pass.""" - fields_set = payload.model_fields_set - if ( - "provider" not in fields_set - and "telephony_integration_id" not in fields_set - ): - return - - from app.api.v1.routes.call_imports import _resolve_telephony_integration - - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - else: - call_import.provider = None - call_import.telephony_integration_id = None - db.flush() - - -def _apply_retry_overrides( - db: Session, - evaluation: CallImportEvaluation, - organization_id: UUID, - payload: CallImportEvaluationRetryRequest, -) -> None: - """Validate + persist the LLM/STT override fields on the run. - - Mirrors the validation in ``create_call_import_evaluation`` but - only touches the fields the caller actually sent — leaving any - field ``None`` preserves the run's existing value. Raises - ``HTTPException(400)`` on bad input so the route handler can let - FastAPI turn it into a clean 400 response. - """ - # --- LLM provider + model (must be sent together) --- - if payload.llm_provider is not None or payload.llm_model is not None: - if not (payload.llm_provider and payload.llm_model): - raise HTTPException( - status_code=400, - detail=( - "Both llm_provider and llm_model are required when " - "overriding the run LLM on retry." - ), - ) - try: - evaluation.llm_provider = ModelProvider( - payload.llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Unknown LLM provider '{payload.llm_provider}'. " - "Valid keys are documented in ModelProvider." - ), - ) - new_model = payload.llm_model.strip() or None - if not new_model: - raise HTTPException( - status_code=400, detail="llm_model cannot be empty." - ) - evaluation.llm_model = new_model - - # --- LLM credential pin --- - if payload.llm_credential_id is not None: - cred = ( - db.query(AIProvider) - .filter( - AIProvider.id == payload.llm_credential_id, - AIProvider.organization_id == organization_id, - ) - .first() - ) - if not cred: - raise HTTPException( - status_code=400, - detail=( - "The provided llm_credential_id does not exist in " - "this organization." - ), - ) - evaluation.llm_credential_id = payload.llm_credential_id - - if payload.llm_config is not None: - evaluation.llm_config = payload.llm_config - - # --- Per-metric LLM overrides --- - # We accept the same dict shape as the create endpoint but - # constrain keys to leaf metrics that are actually in this run. - # Passing an empty dict explicitly clears existing overrides. - if payload.metric_llm_overrides is not None: - valid_leaf_ids = { - str(mid) for mid in (evaluation.selected_metric_ids or []) - } - overrides_payload: Dict[str, Dict[str, Any]] = {} - for metric_id, override in payload.metric_llm_overrides.items(): - if metric_id not in valid_leaf_ids: - raise HTTPException( - status_code=400, - detail=( - "metric_llm_overrides references metric " - f"{metric_id} which is not a leaf metric in " - "this run." - ), - ) - override_dict: Dict[str, Any] = {} - if override.provider is not None: - if not override.model: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a " - "provider but no model." - ), - ) - try: - override_dict["provider"] = ModelProvider( - override.provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} uses " - f"unknown provider '{override.provider}'." - ), - ) - override_dict["model"] = override.model.strip() - elif override.model: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a model " - "but no provider." - ), - ) - if override.credential_id is not None: - override_dict["credential_id"] = str(override.credential_id) - if override.llm_config is not None: - override_dict["llm_config"] = override.llm_config - if override_dict: - overrides_payload[metric_id] = override_dict - evaluation.metric_llm_overrides = overrides_payload or None - - # --- STT provider + model (must be sent together) --- - if payload.stt_provider is not None or payload.stt_model is not None: - if not (payload.stt_provider and payload.stt_model): - raise HTTPException( - status_code=400, - detail=( - "Both stt_provider and stt_model are required " - "when overriding the run STT on retry." - ), - ) - try: - evaluation.stt_provider = ModelProvider( - payload.stt_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=f"Unknown STT provider '{payload.stt_provider}'.", - ) - new_stt_model = payload.stt_model.strip() or None - if not new_stt_model: - raise HTTPException( - status_code=400, detail="stt_model cannot be empty." - ) - evaluation.stt_model = new_stt_model - - # --- STT credential pin --- - if payload.stt_credential_id is not None: - evaluation.stt_credential_id = payload.stt_credential_id - - # --- LLM diariser provider + model (must be sent together) --- - if ( - payload.diarization_llm_provider is not None - or payload.diarization_llm_model is not None - ): - if not ( - payload.diarization_llm_provider - and payload.diarization_llm_model - ): - raise HTTPException( - status_code=400, - detail=( - "Both diarization_llm_provider and " - "diarization_llm_model are required when overriding " - "the run diariser on retry." - ), - ) - try: - evaluation.diarisation_llm_provider = ModelProvider( - payload.diarization_llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - "Unknown diarisation LLM provider " - f"'{payload.diarization_llm_provider}'." - ), - ) - new_diariser_model = ( - payload.diarization_llm_model.strip() or None - ) - if not new_diariser_model: - raise HTTPException( - status_code=400, - detail="diarization_llm_model cannot be empty.", - ) - evaluation.diarisation_llm_model = new_diariser_model - - if payload.diarization_llm_credential_id is not None: - evaluation.diarisation_llm_credential_id = ( - payload.diarization_llm_credential_id - ) - - # ``diarization_prompt`` semantics: None = leave untouched; - # empty string = clear (fall back to the canonical default at - # worker time); anything else = persist verbatim. - if payload.diarization_prompt is not None: - cleaned = payload.diarization_prompt.strip() - evaluation.diarisation_prompt = cleaned or None - - if payload.transcribe_mode is not None: - mode = payload.transcribe_mode.strip().lower() - if mode not in {"stt_llm", "llm_only"}: - raise HTTPException( - status_code=400, - detail=( - f"Unknown transcribe_mode '{payload.transcribe_mode}'. " - "Valid values are 'stt_llm' and 'llm_only'." - ), - ) - evaluation.transcribe_mode = mode - - -def _gather_retry_targets( - db: Session, - evaluation: CallImportEvaluation, - requested_ids: Optional[List[UUID]], - *, - include_completed: bool = False, -) -> Tuple[ - List[Tuple[CallImportEvaluationRow, CallImportRow]], - List[CallImportEvaluationRetrySkippedItem], -]: - """Resolve which rows to retry + reasons for any we refuse. - - When ``requested_ids`` is None we retry every row whose status is - ``failed`` (or every row when ``include_completed`` is also set — - used by the metric-subset retry path which legitimately wants to - recompute a metric on already-successful rows). When the caller - passes ids explicitly we still filter out rows that are currently - in flight; ``include_completed`` controls whether previously- - successful rows are eligible. - """ - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - from app.db_sharding.eval_rows import gather_retry_targets_sharded - - return gather_retry_targets_sharded( - db, - evaluation, - requested_ids, - include_completed=include_completed, - ) - - eval_rows_query = db.query(CallImportEvaluationRow).filter( - CallImportEvaluationRow.evaluation_id == evaluation.id - ) - - targets: List[Tuple[CallImportEvaluationRow, CallImportRow]] = [] - skipped: List[CallImportEvaluationRetrySkippedItem] = [] - - if requested_ids is None: - if include_completed: - # "Retry everything" path used by the metric-subset re-run - # UI. Still skip in-flight rows below so we don't trample - # work the worker is actively doing. - candidate_rows = eval_rows_query.filter( - CallImportEvaluationRow.status.in_(["failed", "completed"]) - ).all() - else: - candidate_rows = eval_rows_query.filter( - CallImportEvaluationRow.status == "failed" - ).all() - else: - requested_set = set(requested_ids) - candidate_rows = eval_rows_query.filter( - CallImportEvaluationRow.id.in_(requested_set) - ).all() - found_ids = {row.id for row in candidate_rows} - for missing in requested_set - found_ids: - skipped.append( - CallImportEvaluationRetrySkippedItem( - eval_row_id=missing, - reason="unknown", - ) - ) - - if not candidate_rows: - return targets, skipped - - source_row_ids = [row.call_import_row_id for row in candidate_rows] - source_rows = ( - db.query(CallImportRow) - .filter(CallImportRow.id.in_(source_row_ids)) - .all() - ) - source_by_id = {row.id: row for row in source_rows} - - for eval_row in candidate_rows: - if eval_row.status in {"pending", "running"}: - skipped.append( - CallImportEvaluationRetrySkippedItem( - eval_row_id=eval_row.id, - reason="in_progress", - ) - ) - continue - if eval_row.status == "completed" and not include_completed: - skipped.append( - CallImportEvaluationRetrySkippedItem( - eval_row_id=eval_row.id, - reason="completed", - ) - ) - continue - source_row = source_by_id.get(eval_row.call_import_row_id) - if source_row is None: - skipped.append( - CallImportEvaluationRetrySkippedItem( - eval_row_id=eval_row.id, - reason="source_row_missing", - ) - ) - continue - targets.append((eval_row, source_row)) - - return targets, skipped - - -@router.post( - "/{eval_id}/retry", - response_model=CallImportEvaluationRetryResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="retryCallImportEvaluation", -) -async def retry_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - payload: Optional[CallImportEvaluationRetryRequest] = Body(default=None), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationRetryResponse: - """Re-enqueue failed rows in an evaluation run. - - Default behavior (no body) is "retry every row that failed". Pass - ``eval_row_ids`` to scope the retry to a specific subset (e.g. the - single row a user clicked in the UI). Rows that are still - in-flight or already completed are returned in ``skipped`` rather - than re-enqueued, so this endpoint is always safe to call. - - When ``metric_ids`` is set in the payload, this is a **metric- - subset retry**: only the listed metrics are recomputed (and merged - into the row's existing ``metric_scores`` — other metrics' values - are preserved). The route auto-flips ``include_completed=True`` in - that case so previously-successful rows are eligible for re- - scoring; without it the call would no-op because every row would - be skipped as ``completed``. - - The worker contract is the same as the create endpoint: - ``evaluate_call_import_row_task(eval_row_id, [restricted_metric_ids])``. - When the run is configured for diarised transcripts and the row's - diarised transcript is missing, we chain through - ``transcribe_call_import_row_task`` first — matching the - auto-transcribe behavior of POST ``/evaluations``. - """ - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - requested_ids = payload.eval_row_ids if payload else None - # Metric-subset retry: validate that every metric is something this - # run actually scored. Empty list is rejected too — callers that - # want a full re-run should omit the field entirely. - # - # ``selected_metric_ids`` holds the LEAVES only (children for - # hierarchical / category metrics, standalone metrics otherwise) — - # see ``leaf_metric_ids`` in :func:`create_call_import_evaluation`. - # Parent IDs for hierarchical metrics live separately in - # ``selected_metric_groups`` (``{parent_id: [child_ids]}``) so the - # UI can reconstruct the tree without round-tripping through the - # metric table. - # - # The Re-run-metrics modal surfaces PARENTS for hierarchical - # metrics (it suppresses individual children via - # ``childrenInGroups`` in ``CallImportEvaluationDetail.tsx``), so a - # naive ``metric_ids ⊆ selected_metric_ids`` check rejects every - # parent-ID request with a misleading "unknown ids" 400. We accept - # both shapes here and then EXPAND any parent IDs into - # ``{parent_id, *child_ids}`` so the downstream helpers see the - # full set of keys that need clearing + the full set of leaves - # that need re-scoring. - metric_ids: Optional[List[UUID]] = ( - payload.metric_ids if payload else None - ) - if metric_ids is not None: - if not metric_ids: - raise HTTPException( - status_code=400, - detail=( - "metric_ids must be a non-empty list. Omit the " - "field to re-run all metrics." - ), - ) - - leaf_set: Set[str] = { - str(item).lower() - for item in (evaluation.selected_metric_ids or []) - } - # ``selected_metric_groups`` is a dict ``{parent_id_str: - # [child_id_str, ...]}`` (see line ~487 in - # ``create_call_import_evaluation``). We tolerate stale data - # (string / UUID / non-dict) without crashing the retry path — - # if it's malformed we just treat it as "no parents" and fall - # back to the leaf-only check. - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - parent_to_children_str: Dict[str, List[str]] = {} - for parent_key, children_raw in groups_raw.items(): - if not isinstance(children_raw, (list, tuple)): - continue - children_norm = [ - str(c).lower() for c in children_raw if c is not None - ] - parent_to_children_str[str(parent_key).lower()] = children_norm - parent_set = set(parent_to_children_str.keys()) - - unknown = [ - mid for mid in metric_ids - if str(mid).lower() not in leaf_set - and str(mid).lower() not in parent_set - ] - if unknown: - raise HTTPException( - status_code=400, - detail=( - "metric_ids must be a subset of this evaluation's " - f"selected metrics; unknown ids: {[str(u) for u in unknown]}." - ), - ) - - # Expand parent IDs into ``{parent, *children}`` so: - # * ``_reset_eval_row_for_retry`` strips BOTH the parent - # entry (with ``chosen_child_id`` / rationale) AND every - # per-child boolean entry that the LLM evaluator wrote - # under each child's ID (see - # ``app/workers/tasks/helpers/llm_evaluation.py`` lines - # 1584 and 1649). - # * ``_enqueue_eval_rows_with_optional_transcribe`` → - # ``evaluate_call_import_row_task`` filters the work-list - # off ``selected_metric_ids`` (leaves), so we MUST hand it - # the child IDs for the parent to actually get re-scored. - # Leaves pass through unchanged. - expanded: List[UUID] = [] - seen: Set[str] = set() - for mid in metric_ids: - mid_norm = str(mid).lower() - children_str = parent_to_children_str.get(mid_norm) - if children_str is not None: - # Parent: include the parent ID itself (so the parent - # entry in ``metric_scores`` is also cleared) and all - # of its children. - candidates = [mid_norm, *children_str] - else: - candidates = [mid_norm] - for candidate in candidates: - if candidate in seen: - continue - try: - expanded.append(UUID(candidate)) - except (TypeError, ValueError): - # Defensive: skip junk values rather than 500. - continue - seen.add(candidate) - metric_ids = expanded - - # ``include_completed`` is auto-enabled when the caller asked for a - # metric subset (otherwise the metric-subset retry would always - # no-op on a green run, which is the whole reason this feature - # exists). The explicit payload flag wins for full-row retries. - include_completed = bool( - (payload.include_completed if payload else False) - or (metric_ids is not None) - ) - - transcribe_overwrite = bool( - payload.transcribe_overwrite if payload else False - ) - - skipped: List[CallImportEvaluationRetrySkippedItem] = [] - if requested_ids is None: - from app.db_sharding.eval_rows import count_evaluation_rows_for_run - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - statuses = ( - ["failed", "completed"] if include_completed else ["failed"] - ) - target_count = count_evaluation_rows_for_run( - db, eval_id, statuses=statuses - ) - else: - from sqlalchemy import func - - count_query = db.query(func.count(CallImportEvaluationRow.id)).filter( - CallImportEvaluationRow.evaluation_id == eval_id - ) - if include_completed: - count_query = count_query.filter( - CallImportEvaluationRow.status.in_(["failed", "completed"]) - ) - else: - count_query = count_query.filter( - CallImportEvaluationRow.status == "failed" - ) - target_count = int(count_query.scalar() or 0) - if target_count == 0: - return CallImportEvaluationRetryResponse( - requeued=0, - transcribe_requeued=0, - skipped=skipped, - ) - else: - targets, skipped = _gather_retry_targets( - db, - evaluation, - requested_ids, - include_completed=include_completed, - ) - if not targets: - return CallImportEvaluationRetryResponse( - requeued=0, - transcribe_requeued=0, - skipped=skipped, - ) - target_count = len(targets) - - # Apply LLM / STT overrides BEFORE enqueueing so the persisted run - # config is correct by the time the worker reads it. - if payload is not None: - _apply_retry_overrides(db, evaluation, organization_id, payload) - _apply_telephony_retry_overrides( - db, - call_import=call_import, - organization_id=organization_id, - payload=payload, - ) - - evaluation.error_message = None - evaluation.finished_at = None - evaluation.status = "running" - if not evaluation.started_at: - from datetime import datetime, timezone - - evaluation.started_at = datetime.now(timezone.utc) - - _claim_evaluation_bulk_operation(eval_id, "retry") - db.commit() - - from app.workers.tasks.call_import_bulk_ops import ( - retry_call_import_evaluation_task, - ) - - retry_call_import_evaluation_task.delay( - str(eval_id), - { - "eval_row_ids": [str(rid) for rid in requested_ids] - if requested_ids - else None, - "metric_ids": [str(mid) for mid in metric_ids] if metric_ids else None, - "include_completed": include_completed, - "transcribe_overwrite": transcribe_overwrite, - }, - ) - - return CallImportEvaluationRetryResponse( - requeued=target_count, - transcribe_requeued=0, - skipped=skipped, - ) - - -@router.post( - "/{eval_id}/rows/{eval_row_id}/retry", - response_model=CallImportEvaluationRowResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="retryCallImportEvaluationRow", -) -async def retry_call_import_evaluation_row( - call_import_id: UUID, - eval_id: UUID, - eval_row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationRowResponse: - """Re-enqueue a single failed evaluation row. - - Convenience wrapper around ``retry_call_import_evaluation`` for the - "Retry this row" affordance in the row table. Returns the - refreshed row so the UI can update its badge immediately, without - waiting for the next polling tick. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - _require_no_evaluation_bulk_operation(eval_id) - - from app.db_sharding.eval_rows import ( - evaluation_row_session, - find_evaluation_row_in_run, - ) - from app.db_sharding.sessions import is_sharding_enabled - - eval_row, _source_stub = find_evaluation_row_in_run(db, eval_id, eval_row_id) - if eval_row is None: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - - if eval_row.status in {"pending", "running"}: - raise HTTPException( - status_code=409, - detail=( - "This row is still in progress — wait for it to finish " - "before retrying." - ), - ) - - targets, _ = _gather_retry_targets(db, evaluation, [eval_row.id]) - if not targets: - raise HTTPException( - status_code=409, - detail=( - "This row cannot be retried in its current state " - f"(status={eval_row.status})." - ), - ) - - if is_sharding_enabled(): - with evaluation_row_session(eval_row_id) as ( - row_db, - _catalog_db, - eval_row, - source_row, - _shard_id, - ): - _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) - _reset_eval_row_for_retry(eval_row) - row_db.commit() - targets = [(eval_row, source_row)] - else: - for er, source_row in targets: - _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) - _reset_eval_row_for_retry(er) - - evaluation.error_message = None - evaluation.finished_at = None - evaluation.status = "running" - if not evaluation.started_at: - from datetime import datetime, timezone - - evaluation.started_at = datetime.now(timezone.utc) - db.flush() - _rollup_evaluation_status(evaluation, db) - db.commit() - - try: - _enqueue_eval_rows_with_optional_transcribe(db, evaluation, targets) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to re-enqueue retry for evaluation row {}", eval_row_id - ) - if is_sharding_enabled(): - with evaluation_row_session(eval_row_id) as ( - row_db, - _catalog_db, - eval_row, - _source_row, - _shard_id, - ): - eval_row.status = "failed" - eval_row.error_message = f"Failed to re-enqueue retry: {exc}" - row_db.commit() - else: - eval_row.status = "failed" - eval_row.error_message = f"Failed to re-enqueue retry: {exc}" - _rollup_evaluation_status(evaluation, db) - db.commit() - raise HTTPException( - status_code=500, - detail=f"Failed to re-enqueue retry: {exc}", - ) - - if is_sharding_enabled(): - with evaluation_row_session(eval_row_id) as ( - _row_db, - _catalog_db, - eval_row, - source_row, - _shard_id, - ): - return _to_evaluation_row_response(eval_row, source_row, evaluation) - - db.refresh(eval_row) - source_row = targets[0][1] - return _to_evaluation_row_response(eval_row, source_row, evaluation) - - -from app.core.auth.capabilities import EVALS_RUN, EVALS_VIEW, REPORTS_GENERATE -from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities - -apply_workspace_route_capabilities( - router, - view_capability=EVALS_VIEW, - manage_capability=EVALS_RUN, - run_capability=EVALS_RUN, - report_capability=REPORTS_GENERATE, -) +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +import asyncio +import csv +import base64 +import io +import json +import math +import re +import statistics +from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple +from uuid import UUID, uuid4 + +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import desc, func, or_, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.call_imports.audit import ( + actor_emails_for_evaluation, + emails_for_user_ids, + stamp_call_import_actor, + stamp_evaluation_actor, + user_ids_from_evaluations, +) +from app.services.workspace_rbac import resolve_workspace_capabilities +from app.models.database import ( + AIProvider, + CallImport, + CallImportEvaluation, + CallImportEvaluationReportSnapshot, + CallImportEvaluationPdfReport, + CallImportEvaluationRow, + CallImportRow, + Metric, + PromptPartial, + Workspace, +) +from app.models.enums import CallImportRowStatus, ModelProvider +from app.models.schemas import ( + CallImportEvaluationAggregateResponse, + CallImportEvaluationBulkDelete, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationCreate, + CallImportEvaluationListResponse, + CallImportEvaluationResponse, + CallImportEvaluationRetryRequest, + CallImportEvaluationRetryResponse, + CallImportEvaluationRetrySkippedItem, + CallImportEvaluationRowListResponse, + CallImportEvaluationRowResponse, + CallImportEvaluationUpdate, + CallImportMetricAggregate, + CallImportMetricHistogramBucket, + CallImportMetricLabelPair, + CallImportMetricSummary, + CallImportMetricValueCount, + DiscoveredLabelDeleteRequest, + DiscoveredLabelItem, + DiscoveredLabelMergeRequest, + DiscoveredLabelsResponse, + DiscoveredMetricDeleteRequest, + DiscoveredMetricItem, + DiscoveredMetricMergeRequest, + DiscoveredMetricsResponse, + EvaluationInsightsRequest, + EvaluationTldrSummary, + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + EvaluationPromptImprovementsRequest, + EvaluationPromptImprovementsState, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, + MetricFailurePolicy, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + EvaluationUserInsightsRequest, + EvaluationUserInsightsState, + MetricFlowEdge, + MetricPeriodDelta, + MetricFlowNode, + MetricFlowResponse, +) +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + estimate_metric_clusters_llm_calls, + filter_completed_row_pairs, + list_eligible_cluster_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.metric_failure_policy import ( + aggregate_primary_percent, + build_failure_policy_previews, + effective_policies, + failure_rate_percent_from_rows, + failure_policies_to_db, + has_clusterable_metrics, + merge_clustering_policies, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) +from app.services.call_import_user_insights import ( + normalize_max_llm_calls, + total_llm_calls_for_rows, + user_insights_state_from_raw, +) + +router = APIRouter( + prefix="/call-imports/{call_import_id}/evaluations", + tags=["Call Import Evaluations"], + dependencies=[Depends(require_enterprise_feature("call_imports"))], +) + + +class CallImportEvaluationPdfReportRequest(BaseModel): + vendor_name: str = Field(..., min_length=1, max_length=120) + report_type: Literal["external", "internal"] = "external" + include_weekly_delta: bool = False + include_period_delta: bool = False + baseline_evaluation_id: Optional[str] = None + period_label: Optional[str] = Field(default=None, max_length=64) + use_case: Optional[str] = Field(default=None, max_length=120) + internal_brand_image_id: Optional[str] = None + external_brand_image_id: Optional[str] = None + report_config: Dict[str, Any] = Field(default_factory=dict) + platform_base_url: Optional[str] = Field( + default=None, + max_length=512, + description="Frontend origin for deep links to example calls in internal PDFs.", + ) + + @field_validator("vendor_name") + @classmethod + def _clean_vendor_name(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("Vendor name is required.") + return cleaned + + +class CallImportEvaluationPdfReportResponse(BaseModel): + id: str + filename: str + preview_url: Optional[str] = None + download_url: Optional[str] = None + created_at: datetime + created_by: Optional[str] = None + report_type: str + vendor_name: str + config_summary: Optional[str] = None + storage_available: bool = True + cache_hit: bool = False + + +class CallImportEvaluationPdfReportListItem(BaseModel): + id: str + filename: Optional[str] = None + vendor_name: str + report_type: str + created_by: Optional[str] = None + created_at: datetime + config_summary: Optional[str] = None + cache_fingerprint: Optional[str] = None + + +class CallImportEvaluationPdfReportListResponse(BaseModel): + items: List[CallImportEvaluationPdfReportListItem] + + +class CallImportEvaluationBaselineCandidate(BaseModel): + evaluation_id: str + name: str + dataset: str + period_label: Optional[str] = None + period_start: Optional[date] = None + period_end: Optional[date] = None + period_display: str + completed_rows: int + created_at: datetime + is_default: bool = False + + +class CallImportEvaluationBaselineCandidatesResponse(BaseModel): + items: List[CallImportEvaluationBaselineCandidate] + default_evaluation_id: Optional[str] = None + + +def _require_import( + db: Session, + call_import_id: UUID, + organization_id: UUID, +) -> CallImport: + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException(status_code=404, detail="Call import not found") + return call_import + + +def require_call_import_capability(capability: str): + """Ensure the caller has *capability* in the call import's workspace (not just the header).""" + + def _dep( + call_import_id: UUID, + principal: Principal = Depends(get_principal), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), + ) -> CallImport: + call_import = _require_import(db, call_import_id, organization_id) + caps, _, role = resolve_workspace_capabilities( + db, + principal=principal, + workspace_id=call_import.workspace_id, + organization_id=organization_id, + ) + if capability not in caps: + raise HTTPException( + status_code=403, + detail=capability_denied_message( + capability, + role_name=role.name if role else None, + workspace_label="the active workspace", + ), + ) + return call_import + + return _dep + + +def _flatten_transcript(text: Optional[str]) -> str: + """Collapse a multi-line transcript onto a single line for spreadsheet export. + + The diarised transcript is stored as ``: `` lines joined + by ``\\n`` because the in-app ``TranscriptView`` parses those line + breaks to render chat bubbles. In Excel / Google Sheets that same + newline-per-turn formatting causes each cell to balloon vertically, + which the user reads as "lots of empty space on top of the cell". + Flattening at export time keeps the DB shape intact while giving the + spreadsheet a single-line cell per row. + """ + if not text: + return "" + parts = [ + segment.strip() + for segment in text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + ] + return " ".join(p for p in parts if p) + + +def _evaluated_transcript_source_label( + evaluation: CallImportEvaluation, + source_row: CallImportRow, +) -> str: + """Label which transcript source this row was scored against.""" + source = (evaluation.transcript_source or "diarised").strip().lower() + if source == "production": + if not (source_row.transcript or "").strip(): + return "" + return "Production" + if not (source_row.diarised_transcript or "").strip(): + return "" + return "Diarised" + + +def _pick_evaluation_row_transcript( + source_row: Optional[CallImportRow], + evaluation: Optional[CallImportEvaluation] = None, +) -> Optional[str]: + """Transcript shown in evaluation row detail for the run's source.""" + if source_row is None: + return None + source = ( + (evaluation.transcript_source or "diarised").strip().lower() + if evaluation is not None + else "diarised" + ) + if source == "production": + raw = (source_row.transcript or "").strip() + return raw or None + diarised = (source_row.diarised_transcript or "").strip() + if diarised: + return diarised + raw = (source_row.transcript or "").strip() + return raw or None + + +def _to_evaluation_row_response( + eval_row_obj: CallImportEvaluationRow, + source_row: Optional[CallImportRow], + evaluation: Optional[CallImportEvaluation] = None, +) -> CallImportEvaluationRowResponse: + """Serialize one evaluation row plus joined source-row metadata.""" + return CallImportEvaluationRowResponse( + id=eval_row_obj.id, + evaluation_id=eval_row_obj.evaluation_id, + call_import_row_id=eval_row_obj.call_import_row_id, + row_index=source_row.row_index if source_row else None, + conversation_id=source_row.conversation_id if source_row else None, + transcript=_pick_evaluation_row_transcript(source_row, evaluation), + raw_columns=source_row.raw_columns if source_row else None, + recording_url=source_row.recording_url if source_row else None, + recording_date=source_row.recording_date if source_row else None, + recording_s3_key=source_row.recording_s3_key if source_row else None, + diarised_transcript_status=( + source_row.diarised_transcript_status if source_row else None + ), + diarised_transcript_error=( + source_row.diarised_transcript_error if source_row else None + ), + status=eval_row_obj.status, + metric_scores=eval_row_obj.metric_scores or {}, + error_message=eval_row_obj.error_message, + started_at=eval_row_obj.started_at, + finished_at=eval_row_obj.finished_at, + created_at=eval_row_obj.created_at, + updated_at=eval_row_obj.updated_at, + ) + + +def _serialize_selected_metric_ids(value) -> List[UUID]: + result: List[UUID] = [] + if not isinstance(value, list): + return result + for item in value: + try: + result.append(UUID(str(item))) + except (TypeError, ValueError): + continue + return result + + +def _metrics_for_ids(db: Session, org_id: UUID, ids: List[UUID]) -> List[Metric]: + if not ids: + return [] + rows = ( + db.query(Metric) + .filter( + Metric.organization_id == org_id, + Metric.id.in_(ids), + ) + .all() + ) + by_id = {row.id: row for row in rows} + return [by_id[mid] for mid in ids if mid in by_id] + + +def _expand_metric_selection( + db: Session, + org_id: UUID, + selected_ids: List[UUID], +) -> Tuple[List[Metric], Dict[UUID, List[Metric]]]: + """Resolve user-supplied metric ids into actual leaves + parent grouping. + + Rules: + * If a parent id is in ``selected_ids`` and no specific children of + that parent are also listed, include EVERY enabled child of that + parent. + * If a parent id AND some of its children are listed, include only + the listed children (treat the parent selection as the + "container" so users can deselect labels). + * Standalone metrics (no parent, no children) pass through + unchanged. + * Disabled metrics are filtered out at this layer so the caller + doesn't have to repeat the check. + + Returns: + (effective_metrics, parent_to_children) + + ``effective_metrics`` is the deduplicated list of metrics the + worker will actually score (children + standalone). Order is + preserved from ``selected_ids`` for display stability. + + ``parent_to_children`` maps each parent metric id (UUID) to the + list of its selected children. Useful for grouping in the LLM + prompt builder. + """ + if not selected_ids: + return [], {} + + requested = list(selected_ids) + initial_rows = ( + db.query(Metric) + .filter( + Metric.organization_id == org_id, + Metric.id.in_(requested), + ) + .all() + ) + initial_by_id = {row.id: row for row in initial_rows} + + parent_ids_requested = { + m.id for m in initial_rows if m.selection_mode and not m.parent_metric_id + } + # Map parent id -> children explicitly requested by the user. + explicit_children_by_parent: Dict[UUID, List[Metric]] = {} + for m in initial_rows: + if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: + explicit_children_by_parent.setdefault( + m.parent_metric_id, [] + ).append(m) + + # For parents without explicit children, hydrate every enabled child. + parents_needing_full_expansion = [ + pid + for pid in parent_ids_requested + if pid not in explicit_children_by_parent + ] + auto_expanded_children: Dict[UUID, List[Metric]] = {} + if parents_needing_full_expansion: + for pid in parents_needing_full_expansion: + child_rows = ( + db.query(Metric) + .filter( + Metric.organization_id == org_id, + Metric.parent_metric_id == pid, + Metric.enabled.is_(True), + ) + .order_by(Metric.created_at.asc()) + .all() + ) + auto_expanded_children[pid] = child_rows + + parent_to_children: Dict[UUID, List[Metric]] = {} + for pid in parent_ids_requested: + children = explicit_children_by_parent.get( + pid + ) or auto_expanded_children.get(pid, []) + # Drop disabled children so the worker doesn't waste a slot on + # them. Empty parents (no enabled children) are still tracked + # because the UI may want to show "0 of 0" rather than swallow + # them silently. + parent_to_children[pid] = [c for c in children if c.enabled] + + effective: List[Metric] = [] + seen: set[UUID] = set() + for mid in requested: + m = initial_by_id.get(mid) + if m is None: + continue + if m.selection_mode and not m.parent_metric_id: + # Parent row itself is not scored — only its children. + for child in parent_to_children.get(m.id, []): + if child.id in seen or not child.enabled: + continue + seen.add(child.id) + effective.append(child) + continue + if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: + # Already accounted for via the parent expansion above. + continue + if not m.enabled: + continue + if m.id in seen: + continue + seen.add(m.id) + effective.append(m) + + return effective, parent_to_children + + +def _evaluation_bulk_operation_for_response( + evaluation_id: UUID, +) -> Optional[str]: + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + ) + + return get_evaluation_bulk_operation(evaluation_id) + + +def _serialize_eval( + db: Session, + row: CallImportEvaluation, + *, + sibling_evaluation_ids: Optional[List[UUID]] = None, + user_emails: Optional[Dict[UUID, str]] = None, +) -> CallImportEvaluationResponse: + selected_ids = _serialize_selected_metric_ids(row.selected_metric_ids) + + # Pull every metric referenced anywhere in the run's grouping (leaves, + # standalone, AND parents from selected_metric_groups) so the UI can + # render parent labels even when only children were materialized into + # selected_metric_ids. + groups_raw: Dict[str, List[str]] = {} + if isinstance(row.selected_metric_groups, dict): + for parent_str, children in row.selected_metric_groups.items(): + if not isinstance(children, list): + continue + cleaned: List[str] = [] + for c in children: + try: + UUID(str(c)) + cleaned.append(str(c)) + except (TypeError, ValueError): + continue + try: + UUID(parent_str) + groups_raw[parent_str] = cleaned + except (TypeError, ValueError): + continue + + metric_ids_for_lookup: List[UUID] = list(selected_ids) + for parent_str in groups_raw.keys(): + try: + pid = UUID(parent_str) + if pid not in metric_ids_for_lookup: + metric_ids_for_lookup.append(pid) + except (TypeError, ValueError): + continue + + metrics = _metrics_for_ids( + db, row.organization_id, metric_ids_for_lookup + ) + + from app.services.call_imports.progress_counters import merge_eval_counters_for_ui + + ui_completed_raw, ui_failed_raw = merge_eval_counters_for_ui(row) + total = int(row.total_rows or 0) + ui_completed = ( + min(ui_completed_raw, total) if total else ui_completed_raw + ) + ui_failed = min(ui_failed_raw, total) if total else ui_failed_raw + + if user_emails is None: + user_emails = emails_for_user_ids(db, user_ids_from_evaluations([row])) + created_email, updated_email = actor_emails_for_evaluation(row, user_emails) + + return CallImportEvaluationResponse( + id=row.id, + call_import_id=row.call_import_id, + organization_id=row.organization_id, + name=row.name, + selected_metric_ids=selected_ids, + selected_metric_groups=groups_raw or None, + metrics=[ + CallImportMetricSummary( + id=metric.id, + name=metric.name, + metric_type=metric.metric_type, + description=metric.description, + parent_metric_id=metric.parent_metric_id, + selection_mode=metric.selection_mode, + # Required by the Flow tab to know whether a parent + # opted into discovery; without it the + # DiscoveredLabelsPanel stays hidden even when the + # worker is actively producing discovered_labels. + allow_discovery=bool( + getattr(metric, "allow_discovery", False) + ), + ) + for metric in metrics + ], + status=row.status, + total_rows=row.total_rows, + completed_rows=ui_completed, + failed_rows=ui_failed, + error_message=row.error_message, + llm_provider=row.llm_provider, + llm_model=row.llm_model, + llm_credential_id=row.llm_credential_id, + llm_config=( + row.llm_config if isinstance(getattr(row, "llm_config", None), dict) else None + ), + metric_llm_overrides=( + row.metric_llm_overrides + if isinstance(row.metric_llm_overrides, dict) + else None + ), + stt_provider=row.stt_provider, + stt_model=row.stt_model, + stt_credential_id=row.stt_credential_id, + diarisation_llm_provider=getattr(row, "diarisation_llm_provider", None), + diarisation_llm_model=getattr(row, "diarisation_llm_model", None), + diarisation_llm_credential_id=getattr( + row, "diarisation_llm_credential_id", None + ), + diarisation_prompt=getattr(row, "diarisation_prompt", None), + transcribe_mode=( + (getattr(row, "transcribe_mode", None) or "stt_llm") + ), + transcript_source=(row.transcript_source or "diarised"), + sibling_evaluation_ids=list(sibling_evaluation_ids or []), + started_at=row.started_at, + finished_at=row.finished_at, + created_at=row.created_at, + updated_at=row.updated_at, + created_by_email=created_email, + last_updated_by_email=updated_email, + tldr_summary=_tldr_summary_payload(row), + user_insights=_user_insights_payload(row), + metric_clusters=_metric_clusters_payload(row), + discover_new_metrics=bool( + getattr(row, "discover_new_metrics", False) + ), + bulk_operation=_evaluation_bulk_operation_for_response(row.id), + ) + + +def _normalize_name(value: Optional[str]) -> Optional[str]: + """Trim user-supplied name; empty string becomes ``NULL``.""" + if value is None: + return None + trimmed = value.strip() + return trimmed or None + + +def _rollup_evaluation_status(evaluation: CallImportEvaluation, db: Session) -> None: + """Recompute counters + terminal status after rows are added/removed. + + Uses a single aggregate query instead of loading every row status. + """ + from app.workers.tasks.evaluate_call_import_row_core import ( + _apply_parent_status_from_counters, + reconcile_evaluation_counters, + ) + + reconcile_evaluation_counters(db, evaluation) + _apply_parent_status_from_counters(evaluation) + db.flush() + + if evaluation.status in {"completed", "failed", "partial"}: + from app.models.database import CallImport + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + + call_import = ( + db.query(CallImport) + .filter(CallImport.id == evaluation.call_import_id) + .first() + ) + if call_import is not None: + rollup_call_import_batch_status(db, call_import) + + +@router.post( + "", + response_model=CallImportEvaluationResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="createCallImportEvaluation", +) +async def create_call_import_evaluation( + call_import_id: UUID, + payload: CallImportEvaluationCreate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationResponse: + del api_key + call_import = _require_import(db, call_import_id, organization_id) + + metric_ids = payload.metric_ids + if not metric_ids: + raise HTTPException( + status_code=400, + detail="Select at least one metric to run the evaluation against.", + ) + + org_metrics = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.id.in_(metric_ids), + ) + .all() + ) + by_id = {metric.id: metric for metric in org_metrics} + unknown_ids = [mid for mid in metric_ids if mid not in by_id] + if unknown_ids: + raise HTTPException( + status_code=400, + detail=( + "These metric ids do not exist in your organization: " + f"{', '.join(str(mid) for mid in unknown_ids)}. " + "Refresh the metrics list and try again." + ), + ) + # Parents themselves are containers, not scored rows, so a disabled + # parent shouldn't block the run as long as it has enabled children. + # We only reject disabled rows that the worker will actually try to + # evaluate (children + standalone leaves). + disabled_leaves = [ + metric + for metric in org_metrics + if not metric.enabled + and not (metric.selection_mode and not metric.parent_metric_id) + ] + if disabled_leaves: + names = ", ".join(metric.name for metric in disabled_leaves) + raise HTTPException( + status_code=400, + detail=( + f"These metrics are disabled and cannot be evaluated: {names}. " + "Enable them on the Metrics page (or pick different ones) and " + "try again." + ), + ) + + # Expand hierarchical selection: parents auto-include their enabled + # children, mixed parent+child selections respect the user's subset. + effective_metrics, parent_to_children = _expand_metric_selection( + db, organization_id, metric_ids + ) + if not effective_metrics: + raise HTTPException( + status_code=400, + detail=( + "None of the selected metrics yielded an enabled leaf to " + "evaluate. Check that parent categories have enabled " + "children, then try again." + ), + ) + + # The effective list (children + standalone leaves) is what gets + # persisted to ``selected_metric_ids`` and scored by the worker. + # The original parents are preserved in ``selected_metric_groups`` + # so the UI can rebuild the tree later. + leaf_metric_ids: List[UUID] = [m.id for m in effective_metrics] + selected_metric_groups: Dict[str, List[str]] = { + str(pid): [str(c.id) for c in children] + for pid, children in parent_to_children.items() + } + metric_rows = effective_metrics + valid_metric_id_strs = {str(m.id) for m in metric_rows} + + # ----- Validate run-level + per-metric LLM config ----- + llm_provider_norm: Optional[str] = None + llm_model_norm: Optional[str] = None + if payload.llm_provider or payload.llm_model: + if not (payload.llm_provider and payload.llm_model): + raise HTTPException( + status_code=400, + detail="Both llm_provider and llm_model are required when overriding the run LLM.", + ) + try: + llm_provider_norm = ModelProvider( + payload.llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Unknown LLM provider '{payload.llm_provider}'. " + "Valid keys are documented in ModelProvider." + ), + ) + llm_model_norm = payload.llm_model.strip() or None + if not llm_model_norm: + raise HTTPException( + status_code=400, detail="llm_model cannot be empty." + ) + + if payload.llm_credential_id is not None: + cred = ( + db.query(AIProvider) + .filter( + AIProvider.id == payload.llm_credential_id, + AIProvider.organization_id == organization_id, + ) + .first() + ) + if not cred: + raise HTTPException( + status_code=400, + detail=( + "The provided llm_credential_id does not exist in this " + "organization." + ), + ) + + # Per-metric overrides: keys can be either a leaf metric id (applies + # to that metric only) or a parent metric id (applies to every + # child of that parent). Parent keys are expanded to their + # children so the worker only sees concrete leaf ids. + metric_overrides_payload: Optional[Dict[str, Dict[str, Any]]] = None + if payload.metric_llm_overrides: + metric_overrides_payload = {} + for metric_id, override in payload.metric_llm_overrides.items(): + target_leaf_ids: List[str] = [] + if metric_id in valid_metric_id_strs: + target_leaf_ids = [metric_id] + else: + # Maybe it's a parent id — expand to the children that + # are part of THIS run. + try: + parent_uuid = UUID(metric_id) + except (TypeError, ValueError): + raise HTTPException( + status_code=400, + detail=( + "metric_llm_overrides references metric " + f"{metric_id} which is not a valid UUID." + ), + ) + children_for_parent = parent_to_children.get(parent_uuid) + if not children_for_parent: + raise HTTPException( + status_code=400, + detail=( + "metric_llm_overrides references metric " + f"{metric_id} which is not in metric_ids." + ), + ) + target_leaf_ids = [str(c.id) for c in children_for_parent] + + override_dict: Dict[str, Any] = {} + if override.provider is not None: + if not override.model: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a provider " + "but no model." + ), + ) + try: + override_dict["provider"] = ModelProvider( + override.provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} uses unknown " + f"provider '{override.provider}'." + ), + ) + override_dict["model"] = override.model.strip() + elif override.model: + # Model without provider doesn't make sense — treat as 400 + # so the UI can fix it instead of silently falling back. + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a model but " + "no provider." + ), + ) + if override.credential_id is not None: + override_dict["credential_id"] = str(override.credential_id) + if override.llm_config is not None: + override_dict["llm_config"] = override.llm_config + if override_dict: + for leaf_id in target_leaf_ids: + metric_overrides_payload[leaf_id] = override_dict + + # ----- Validate auto-transcribe settings ----- + # Diarised runs auto-diarise rows missing a diarised transcript and + # require STT + diariser LLM config. Production runs score the CSV + # transcript directly and skip diarisation entirely. + use_diarised = payload.transcript_sources[0] == "diarised" + auto_transcribe = use_diarised + + transcribe_mode_norm: Optional[str] = None + stt_provider_norm: Optional[str] = None + stt_model_norm: Optional[str] = None + diarisation_llm_provider_norm: Optional[str] = None + diarisation_llm_model_norm: Optional[str] = None + diarisation_prompt_norm: Optional[str] = None + + if use_diarised: + transcribe_mode_norm = (payload.transcribe_mode or "stt_llm").strip().lower() + if transcribe_mode_norm not in {"stt_llm", "llm_only"}: + raise HTTPException( + status_code=400, + detail=( + f"Unknown transcribe_mode '{payload.transcribe_mode}'. " + "Expected 'stt_llm' or 'llm_only'." + ), + ) + + if transcribe_mode_norm == "stt_llm": + if not payload.stt_provider: + raise HTTPException( + status_code=400, + detail=( + "stt_provider is required when " + "transcribe_mode='stt_llm': every evaluation run " + "auto-diarises rows that are missing a diarised " + "transcript." + ), + ) + if not payload.stt_model: + raise HTTPException( + status_code=400, + detail=( + "stt_model is required when transcribe_mode='stt_llm'." + ), + ) + try: + stt_provider_norm = ModelProvider( + payload.stt_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unknown STT provider '{payload.stt_provider}'.", + ) + stt_model_norm = payload.stt_model.strip() or None + if not stt_model_norm: + raise HTTPException( + status_code=400, detail="stt_model cannot be empty." + ) + else: + # llm_only — explicitly reject lingering STT inputs so the + # contract is unambiguous (the worker would ignore them but + # silent acceptance hides accidental misconfiguration). + if (payload.stt_provider or "").strip() or ( + payload.stt_model or "" + ).strip(): + raise HTTPException( + status_code=400, + detail=( + "stt_provider / stt_model must be omitted when " + "transcribe_mode='llm_only'; the LLM consumes the " + "audio directly." + ), + ) + + # --- Validate LLM diariser settings ----- + if not payload.diarization_llm_provider: + raise HTTPException( + status_code=400, + detail=( + "diarization_llm_provider is required: every evaluation " + "run diarises STT output with an LLM." + ), + ) + if not payload.diarization_llm_model: + raise HTTPException( + status_code=400, + detail=( + "diarization_llm_model is required: every evaluation " + "run diarises STT output with an LLM." + ), + ) + try: + diarisation_llm_provider_norm = ModelProvider( + payload.diarization_llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Unknown diarisation LLM provider " + f"'{payload.diarization_llm_provider}'." + ), + ) + diarisation_llm_model_norm = ( + payload.diarization_llm_model.strip() or None + ) + if not diarisation_llm_model_norm: + raise HTTPException( + status_code=400, + detail="diarization_llm_model cannot be empty.", + ) + diarisation_prompt_norm = ( + payload.diarization_prompt.strip() + if isinstance(payload.diarization_prompt, str) + else None + ) or None + + from app.models.enums import CallImportParameterType, CallImportStatus + from app.services.call_imports.bulk_ops import ( + count_all_source_rows, + count_completed_source_rows, + count_source_rows_with_production_transcript, + ) + + starting_from_mapped = False + if call_import.status == CallImportStatus.MAPPED: + if not call_import.source_s3_key or not call_import.source_format: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This batch has no staged source file. Upload and map " + "a CSV/Excel file before running evaluation." + ), + ) + if not call_import.schema_id: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Cannot run evaluation without a mapped schema.", + ) + from app.api.v1.routes.call_imports import ( + _ensure_blob_storage_enabled, + _resolve_schema, + _resolve_telephony_integration, + _validate_direct_url_import_ready, + ) + + workspace_id = call_import.workspace_id + schema = _resolve_schema( + db, organization_id, workspace_id, call_import.schema_id + ) + parameters = list(schema.parameters) + if not use_diarised: + transcript_mapped = any( + param.type == CallImportParameterType.TRANSCRIPT + and (call_import.parameter_mapping or {}).get(param.name) + for param in parameters + ) + if not transcript_mapped: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "No transcript column is mapped in this batch. " + "Map a schema transcript parameter to a CSV column, " + "or choose 'Diarize then evaluate'." + ), + ) + if payload.telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + else: + _validate_direct_url_import_ready( + parameters, dict(call_import.parameter_mapping or {}) + ) + integration = None + + _ensure_blob_storage_enabled() + + if integration is not None: + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + else: + call_import.provider = None + call_import.telephony_integration_id = None + + call_import.total_rows = 0 + call_import.completed_rows = 0 + call_import.failed_rows = 0 + call_import.error_message = None + call_import.status = CallImportStatus.PROCESSING + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + starting_from_mapped = True + + if use_diarised: + total_row_count = count_completed_source_rows(db, call_import.id) + else: + # Production runs score CSV text — rows need not wait for + # recording fetch to finish before they are evaluable. + total_row_count = count_source_rows_with_production_transcript( + db, call_import.id + ) + + requested_sources: List[str] = list(payload.transcript_sources) + + if ( + not use_diarised + and not starting_from_mapped + and count_all_source_rows(db, call_import.id) > 0 + and total_row_count == 0 + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "No rows have a production transcript. " + "Choose 'Diarize then evaluate' or import rows with " + "a transcript column." + ), + ) + + base_name = _normalize_name(payload.name) + + def _name_for_source(source: str) -> Optional[str]: + # Single-source runs preserve the user's chosen name verbatim. + del source + return base_name + + created_evaluations: List[CallImportEvaluation] = [] + + for source in requested_sources: + evaluation = CallImportEvaluation( + call_import_id=call_import.id, + organization_id=organization_id, + # Mirror the parent CallImport's workspace so listings can + # filter on workspace_id directly without joining. + workspace_id=call_import.workspace_id, + name=_name_for_source(source), + selected_metric_ids=[ + str(metric_id) for metric_id in leaf_metric_ids + ], + selected_metric_groups=selected_metric_groups or None, + status="pending", + total_rows=total_row_count, + completed_rows=0, + failed_rows=0, + llm_provider=llm_provider_norm, + llm_model=llm_model_norm, + llm_credential_id=payload.llm_credential_id, + llm_config=payload.llm_config, + metric_llm_overrides=metric_overrides_payload, + stt_provider=stt_provider_norm, + stt_model=stt_model_norm, + stt_credential_id=( + payload.stt_credential_id if auto_transcribe else None + ), + diarisation_llm_provider=diarisation_llm_provider_norm, + diarisation_llm_model=diarisation_llm_model_norm, + diarisation_llm_credential_id=( + payload.diarization_llm_credential_id if auto_transcribe else None + ), + diarisation_prompt=diarisation_prompt_norm, + transcribe_mode=transcribe_mode_norm, + transcript_source=source, + discover_new_metrics=bool( + getattr(payload, "discover_new_metrics", False) + ), + ) + stamp_evaluation_actor(evaluation, principal, creating=True) + db.add(evaluation) + db.flush() + created_evaluations.append(evaluation) + + db.commit() + for evaluation in created_evaluations: + db.refresh(evaluation) + + primary_evaluation = created_evaluations[0] + sibling_ids = [e.id for e in created_evaluations[1:]] + + if not total_row_count and not starting_from_mapped: + for evaluation in created_evaluations: + evaluation.status = "completed" + db.commit() + for evaluation in created_evaluations: + db.refresh(evaluation) + return _serialize_eval( + db, primary_evaluation, sibling_evaluation_ids=sibling_ids + ) + + if starting_from_mapped: + from app.workers.tasks.call_import_bulk_ops import ( + materialize_mapped_call_import_evaluation_task, + ) + + for evaluation in created_evaluations: + materialize_mapped_call_import_evaluation_task.delay( + str(call_import.id), + str(organization_id), + str(call_import.workspace_id), + str(evaluation.id), + transcribe_overwrite=payload.transcribe_overwrite, + ) + else: + from app.workers.tasks.call_import_bulk_ops import ( + materialize_call_import_evaluation_task, + ) + + for evaluation in created_evaluations: + materialize_call_import_evaluation_task.delay( + str(evaluation.id), + transcribe_overwrite=payload.transcribe_overwrite, + ) + + for evaluation in created_evaluations: + db.refresh(evaluation) + + return _serialize_eval( + db, primary_evaluation, sibling_evaluation_ids=sibling_ids + ) + + +@router.get( + "", + response_model=CallImportEvaluationListResponse, + operation_id="listCallImportEvaluations", +) +async def list_call_import_evaluations( + call_import_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationListResponse: + del api_key + _require_import(db, call_import_id, organization_id) + rows = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .order_by(desc(CallImportEvaluation.created_at)) + .all() + ) + email_map = emails_for_user_ids(db, user_ids_from_evaluations(rows)) + return CallImportEvaluationListResponse( + items=[_serialize_eval(db, row, user_emails=email_map) for row in rows], + total=len(rows), + ) + + +@router.get( + "/{eval_id}", + response_model=CallImportEvaluationResponse, + operation_id="getCallImportEvaluation", +) +async def get_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationResponse: + del api_key + _require_import(db, call_import_id, organization_id) + row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + return _serialize_eval(db, row) + + +@router.get( + "/{eval_id}/rows", + response_model=CallImportEvaluationRowListResponse, + operation_id="listCallImportEvaluationRows", +) +async def list_call_import_evaluation_rows( + call_import_id: UUID, + eval_id: UUID, + page: int = Query(1, ge=1), + page_size: int = Query(100, ge=1, le=500), + q: Optional[str] = Query( + None, + description=( + "Free-text search across conversation_id and transcript " + "(case-insensitive substring match)." + ), + ), + metric_id: Optional[UUID] = Query( + None, + description=( + "If set, only return rows whose ``metric_scores[metric_id].value`` " + "exactly matches ``metric_value`` (string-compared). " + "Use together with ``metric_value``." + ), + ), + metric_value: Optional[str] = Query( + None, + description="Value to match against metric_id (string compare).", + ), + status_filter: Optional[str] = Query( + None, + alias="status", + description="Restrict to rows with this evaluation row status.", + ), + flow_parent_id: Optional[UUID] = Query( + None, + description=( + "Parent (category) metric whose ``sequence`` array should be " + "checked against ``flow_node`` and ``flow_edge_target``. Used " + "to drill into the calls behind a flow-chart node or edge." + ), + ), + flow_node: Optional[str] = Query( + None, + description=( + "If set together with ``flow_parent_id``, only return rows " + "whose sequence under that parent contains this step. Accepts " + "either a child metric UUID (resolved to slug(name)), a " + "``disc:`` discovered-label id, or a raw slug." + ), + ), + flow_edge_target: Optional[str] = Query( + None, + description=( + "Optional companion to ``flow_node``: when set, restrict to " + "rows whose sequence contains the directed transition " + "``flow_node -> flow_edge_target`` (immediately adjacent). " + "Same id format as ``flow_node``." + ), + ), + discovered_parent_id: Optional[UUID] = Query( + None, + description=( + "Parent (category) metric that defines the discovery scope " + "for ``discovered_label_key`` / ``has_discovered``." + ), + ), + discovered_label_key: Optional[str] = Query( + None, + description=( + "If set together with ``discovered_parent_id``, only return " + "rows whose ``metric_scores[parent].discovered_labels`` " + "list contains an entry with this slug (after applying " + "evaluation-level merge aliases)." + ), + ), + has_discovered: Optional[bool] = Query( + None, + description=( + "If true together with ``discovered_parent_id``, only return " + "rows that have at least one LLM-discovered label for the " + "parent. Useful to triage which calls produced novel labels." + ), + ), + sort_by: Optional[str] = Query( + None, + description=( + "Column to sort by. Accepted values: ``row_index`` (default " + "when omitted), ``conversation_id``, ``status`` (the " + "evaluation-row status), or ``metric:`` to sort " + "by ``metric_scores[].value``. Metric sorts compare " + "the extracted JSON text — adequate for booleans, enum " + "labels, and 0-1 ratings; large integer values may sort " + "lexicographically (10 before 2)." + ), + ), + sort_dir: Optional[str] = Query( + "asc", + description="Sort direction: ``asc`` (default) or ``desc``.", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationRowListResponse: + del api_key + _require_import(db, call_import_id, organization_id) + + eval_row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not eval_row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + query = ( + db.query(CallImportEvaluationRow, CallImportRow) + .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) + .filter(CallImportEvaluationRow.evaluation_id == eval_id) + ) + + # --- Filters ---------------------------------------------------------- + if q and q.strip(): + needle = f"%{q.strip()}%" + # Search across both transcript columns so a hit in either the + # production or the diarised version surfaces the row, + # independent of which source the evaluation actually scored. + query = query.filter( + or_( + CallImportRow.conversation_id.ilike(needle), + CallImportRow.transcript.ilike(needle), + CallImportRow.diarised_transcript.ilike(needle), + ) + ) + + if status_filter: + # The CallImportEvaluationRow.status column is a string in PG so a + # plain == filter works; we lowercase to match the stored values. + query = query.filter( + CallImportEvaluationRow.status == status_filter.strip().lower() + ) + + if metric_id is not None and metric_value is not None: + # ``metric_scores`` is a JSONB column shaped like + # ``{"": {"value": , "type": "boolean", ...}}``. We + # extract the nested ``value`` as text and compare to the user + # input as a string — that handles bool/int/enum without needing + # per-type casts. ``metric_value`` is matched case-insensitively + # so chart clicks on labels like "True" survive any casing drift + # between worker output and the chart label. + path_value = func.json_extract_path_text( + CallImportEvaluationRow.metric_scores, + str(metric_id), + "value", + ) + query = query.filter(func.lower(path_value) == metric_value.strip().lower()) + + # --- Flow chart drilldown filter ------------------------------------- + # Translates a clicked node (or edge) on the flow chart into a + # SQL filter against ``metric_scores[].sequence``. The + # frontend sends either a child UUID, a ``disc:`` discovered + # node id, or a raw slug — we normalize all three to the slug that + # actually appears in stored ``sequence`` arrays. + if flow_parent_id is not None and flow_node and flow_node.strip(): + parent_id_str_local = str(flow_parent_id) + alias_map_flow = _alias_map_for_parent(eval_row, flow_parent_id) + + def _flow_node_to_slug(raw: str) -> Optional[str]: + raw_clean = raw.strip() + if not raw_clean: + return None + if raw_clean == _FLOW_START_NODE_ID: + # The synthetic START node isn't a real sequence entry; + # filtering on it is meaningless so we skip silently. + return None + if raw_clean.startswith(_DISCOVERED_NODE_PREFIX): + return _resolve_alias( + alias_map_flow, + _slug_label(raw_clean[len(_DISCOVERED_NODE_PREFIX) :]), + ) + # Try to interpret as a child metric UUID first; fall back + # to treating it as a slug. + try: + child_uuid = UUID(raw_clean) + except (TypeError, ValueError): + return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) + child = ( + db.query(Metric.name) + .filter( + Metric.id == child_uuid, + Metric.organization_id == organization_id, + ) + .first() + ) + if child and child[0]: + return _resolve_alias(alias_map_flow, _slug_label(child[0])) + return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) + + from_slug = _flow_node_to_slug(flow_node) + target_slug: Optional[str] = None + if flow_edge_target and flow_edge_target.strip(): + target_slug = _flow_node_to_slug(flow_edge_target) + + if from_slug: + # The ``metric_scores`` column is declared as ``Column(JSON)`` + # in the model so on databases where the table was created + # from the model (rather than the migration) the physical + # type is ``json``, not ``jsonb``. The JSONB-only operators + # below (``jsonb_exists``, ``jsonb_array_elements_text``, + # ``@>``) require a JSONB input — we cast once up front so + # the same SQL works regardless of which path created the + # table. + scores_jsonb = ( + "(call_import_evaluation_rows.metric_scores)::jsonb" + ) + if target_slug: + # Edge filter: rows whose sequence under this parent + # contains ``from_slug`` immediately followed by + # ``target_slug``. Implemented as a correlated EXISTS + # over ``jsonb_array_elements_text`` with ORDINALITY, + # which is the portable way to express "next array + # index" against a JSONB array in Postgres. + edge_filter_sql = text( + f""" + EXISTS ( + SELECT 1 + FROM jsonb_array_elements_text( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ) + ) WITH ORDINALITY AS s1(elem, ord) + JOIN jsonb_array_elements_text( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ) + ) WITH ORDINALITY AS s2(elem, ord) + ON s2.ord = s1.ord + 1 + WHERE s1.elem = :from_slug + AND s2.elem = :to_slug + ) + """ + ).bindparams( + p_id=parent_id_str_local, + from_slug=from_slug, + to_slug=target_slug, + ) + query = query.filter(edge_filter_sql) + else: + # Node filter: rows whose ``metric_scores -> parent -> + # 'sequence'`` array contains ``from_slug``. We use the + # function form ``jsonb_exists`` rather than the ``?`` + # operator to avoid psycopg2 mistaking the question + # mark for a parameter placeholder. + node_filter_sql = text( + f""" + jsonb_exists( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ), + :slug + ) + """ + ).bindparams(p_id=parent_id_str_local, slug=from_slug) + query = query.filter(node_filter_sql) + + # --- Discovered label filters --------------------------------------- + # Surfaces "which calls produced THIS LLM-discovered label" and the + # broader "which calls produced ANY LLM-discovered label". Both + # operate on ``metric_scores[].discovered_labels`` (a list + # of dicts) plus the same ``sequence`` array — covering both legacy + # rows where the slug only made it into ``sequence`` and newer + # rows where it landed in both. + if discovered_parent_id is not None and ( + discovered_label_key or has_discovered + ): + d_parent_str = str(discovered_parent_id) + alias_map_disc = _alias_map_for_parent(eval_row, discovered_parent_id) + # See note above: cast once so the JSONB operators don't reject + # the column when it's typed as ``json`` in the database. + scores_jsonb = "(call_import_evaluation_rows.metric_scores)::jsonb" + if discovered_label_key and discovered_label_key.strip(): + target = _resolve_alias( + alias_map_disc, _slug_label(discovered_label_key) + ) + if target: + # Match rows whose discovered_labels list has an entry + # ``{"key": }`` OR whose sequence array still + # contains the slug. The latter covers older rows that + # were rewritten by a merge in the discovered_labels + # blob but whose sequence may have lagged. + contains_json = json.dumps( + {d_parent_str: {"discovered_labels": [{"key": target}]}} + ) + disc_filter_sql = text( + f""" + ( + {scores_jsonb} @> CAST(:contains AS JSONB) + OR + jsonb_exists( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ), + :slug + ) + ) + """ + ).bindparams( + contains=contains_json, + p_id=d_parent_str, + slug=target, + ) + query = query.filter(disc_filter_sql) + elif has_discovered: + # No specific slug — just rows that surfaced any candidate + # under this parent. We coalesce missing paths to ``[]`` so + # ``jsonb_array_length`` always sees an array (it raises on + # non-array inputs, but our shape guarantees a list when + # the key is present). + has_disc_sql = text( + f""" + jsonb_array_length( + COALESCE( + {scores_jsonb} -> :p_id -> 'discovered_labels', + '[]'::jsonb + ) + ) > 0 + """ + ).bindparams(p_id=d_parent_str) + query = query.filter(has_disc_sql) + + # --- Sorting ---------------------------------------------------------- + # Column-click sorting from the UI. Falls back to ``row_index`` so + # paging stays stable when the user clears the sort. We always add a + # secondary ``row_index`` tiebreaker so duplicate sort keys (e.g. + # many rows with ``status = 'completed'``) keep a deterministic + # order across page boundaries — without this, pagination can + # double-show or skip rows when Postgres picks a different physical + # order on each query. + direction_desc = (sort_dir or "asc").strip().lower() == "desc" + + def _apply_direction(column_expr): + return column_expr.desc() if direction_desc else column_expr.asc() + + # Whether the caller's ``sort_by`` resolved to a known column. We + # use this flag to decide whether ``sort_dir`` is honoured on the + # fallback path: unrecognized columns (typos, stale UI state) fall + # back to the implicit ``row_index ASC`` default and intentionally + # ignore ``sort_dir`` so users don't get a surprise reverse order + # from a typo'd column name. + sort_recognized = False + sort_by_clean = (sort_by or "").strip() + primary_sort = None + metric_uuid: Optional[UUID] = None + if sort_by_clean == "row_index": + sort_recognized = True + # Falls through to the default ``order_by`` below with + # ``primary_sort`` still None — but ``sort_recognized=True`` + # tells the fallback branch to apply the requested direction. + elif sort_by_clean == "conversation_id": + sort_recognized = True + primary_sort = _apply_direction(CallImportRow.conversation_id) + elif sort_by_clean == "status": + sort_recognized = True + primary_sort = _apply_direction(CallImportEvaluationRow.status) + elif sort_by_clean.startswith("metric:"): + raw_metric_id = sort_by_clean.split(":", 1)[1].strip() + try: + metric_uuid = UUID(raw_metric_id) + except (TypeError, ValueError): + metric_uuid = None + if metric_uuid is not None: + sort_recognized = True + # ``metric_scores`` is JSON-typed but the helper functions + # for path extraction differ between Postgres (production) + # and SQLite (default test backend). Branch on the active + # dialect so we can use the right primitive: + # * Postgres → ``json_extract_path_text(col, key, "value")`` + # which returns the value as TEXT for both ``json`` and + # ``jsonb`` columns. + # * SQLite → ``json_extract(col, '$."".value')`` + # using JSONPath syntax. ``metric_uuid`` is already + # validated above (``UUID(raw_metric_id)``), so the + # interpolated path is safe from injection. + # NULL values (rows where the metric wasn't scored) sort + # to the END regardless of direction so un-scored rows + # don't crowd the top of an ascending sort. + dialect_name = ( + db.bind.dialect.name if db.bind is not None else "postgresql" + ) + if dialect_name == "sqlite": + json_path = f'$."{metric_uuid}".value' + path_value = func.json_extract( + CallImportEvaluationRow.metric_scores, + json_path, + ) + else: + path_value = func.json_extract_path_text( + CallImportEvaluationRow.metric_scores, + str(metric_uuid), + "value", + ) + primary_sort = ( + path_value.desc().nullslast() + if direction_desc + else path_value.asc().nullslast() + ) + + if primary_sort is not None: + query = query.order_by(primary_sort, CallImportRow.row_index.asc()) + elif sort_recognized: + # Explicit ``sort_by=row_index`` request — honour direction. + query = query.order_by(_apply_direction(CallImportRow.row_index)) + else: + # No sort requested OR unrecognized column — safe default of + # ``row_index ASC``. We deliberately ignore ``sort_dir`` here + # so a typo'd / stale ``sort_by`` doesn't quietly invert the + # default order. + query = query.order_by(CallImportRow.row_index.asc()) + from app.db_sharding.eval_rows import fetch_evaluation_row_pairs_page + from app.db_sharding.sessions import is_sharding_enabled + + def _pair_row_index( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> int: + return int(pair[1].row_index or 0) + + def _directed_string(value: Optional[str], desc: bool) -> Tuple[int, ...]: + text = value or "" + if not desc: + return (0, *text.encode("utf-8")) + return (1, *(-byte for byte in text.encode("utf-8"))) + + if sort_by_clean == "conversation_id": + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[Any, ...]: + return ( + _directed_string(pair[1].conversation_id, direction_desc), + _pair_row_index(pair), + ) + elif sort_by_clean == "status": + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[Any, ...]: + return ( + _directed_string(pair[0].status, direction_desc), + _pair_row_index(pair), + ) + elif sort_by_clean.startswith("metric:") and metric_uuid is not None: + metric_id_str = str(metric_uuid) + + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[Any, ...]: + scores = pair[0].metric_scores or {} + entry = scores.get(metric_id_str, {}) + raw_value = entry.get("value") if isinstance(entry, dict) else None + null_rank = 1 if raw_value is None else 0 + return ( + null_rank, + _directed_string( + str(raw_value) if raw_value is not None else None, + direction_desc, + ), + _pair_row_index(pair), + ) + elif sort_recognized and sort_by_clean == "row_index": + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[int, ...]: + idx = _pair_row_index(pair) + return (-idx,) if direction_desc else (idx,) + else: + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[int, ...]: + return (_pair_row_index(pair),) + + if is_sharding_enabled(): + def _build_query(session: Session): + return query.with_session(session) + + total, rows = fetch_evaluation_row_pairs_page( + db, + _build_query, + page=page, + page_size=page_size, + sort_key=_pair_sort_key, + bounded_shard_fetch=( + not sort_recognized or sort_by_clean == "row_index" + ), + ) + else: + total = query.count() + rows = query.offset((page - 1) * page_size).limit(page_size).all() + + # Row detail shows the transcript for this run's chosen source. + items: List[CallImportEvaluationRowResponse] = [ + _to_evaluation_row_response(eval_row_obj, source_row, eval_row) + for eval_row_obj, source_row in rows + ] + + return CallImportEvaluationRowListResponse( + items=items, + total=total, + page=page, + page_size=page_size, + ) + + +@router.get( + "/{eval_id}/export", + operation_id="exportCallImportEvaluationCsv", +) +async def export_call_import_evaluation_csv( + call_import_id: UUID, + eval_id: UUID, + format: Literal["csv", "xlsx"] = Query( + "csv", + description=( + "Output format. ``csv`` returns a UTF-8 BOM CSV; ``xlsx`` " + "returns a native Excel workbook (single sheet)." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> StreamingResponse: + del api_key + call_import = _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + selected_metric_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) + # Include parent metric ids referenced in selected_metric_groups so + # the export shows a parent "Chosen Label" column next to its + # children's true/false columns. + lookup_ids: List[UUID] = list(selected_metric_ids) + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + for parent_str in groups_raw.keys(): + try: + pid = UUID(parent_str) + if pid not in lookup_ids: + lookup_ids.append(pid) + except (TypeError, ValueError): + continue + metrics = _metrics_for_ids(db, organization_id, lookup_ids) + metric_names = {str(metric.id): metric.name for metric in metrics} + metrics_by_id = {str(metric.id): metric for metric in metrics} + + # Two export-time modes depending on how the batch was uploaded: + # + # * Schema-driven (new): ``call_imports.schema_id`` is set, + # ``parameter_mapping`` records which CSV header fed each + # parameter, and ``raw_columns`` on each row is keyed by + # parameter NAME. Export headers are the parameter names. + # * Legacy (pre-schema): ``column_mapping`` / ``extra_columns`` / + # ``custom_column_mapping`` drive the columns and + # ``raw_columns`` is keyed by the original CSV header. + # + # We bucket entries into ``standard_export_headers`` (raw_columns + # key == export header) and ``custom_export`` (export header + # differs from the raw_columns key) so the row-projection loop + # below stays mode-agnostic. + standard_export_headers: List[str] = [] + custom_export: List[tuple[str, str]] = [] # [(export_header, raw_columns_key)] + + if call_import.schema_id is not None: + # Use the live schema parameter list for column ordering. Falls + # back to whatever's in ``parameter_mapping`` if the schema was + # deleted (defensive - the FK is ON DELETE RESTRICT, but tests + # / future cascades may still hit this branch). + from app.models.database import CallImportSchema as _ImportSchema + + schema_obj = ( + db.query(_ImportSchema) + .filter(_ImportSchema.id == call_import.schema_id) + .first() + ) + if schema_obj is not None: + params_sorted = sorted( + schema_obj.parameters, key=lambda p: p.ordering or 0 + ) + for param in params_sorted: + if param.name and param.name not in standard_export_headers: + standard_export_headers.append(param.name) + else: + for param_name in (call_import.parameter_mapping or {}).keys(): + if param_name and param_name not in standard_export_headers: + standard_export_headers.append(param_name) + else: + mapping = call_import.column_mapping or {} + mapped_headers = [ + mapping.get("external_call_id"), + mapping.get("transcript"), + mapping.get("recording_url"), + ] + for header in [*mapped_headers, *(call_import.extra_columns or [])]: + if ( + isinstance(header, str) + and header + and header not in standard_export_headers + ): + standard_export_headers.append(header) + + custom_mapping = call_import.custom_column_mapping or {} + if isinstance(custom_mapping, dict): + for name, csv_header in custom_mapping.items(): + if not isinstance(name, str) or not isinstance(csv_header, str): + continue + if not name or not csv_header: + continue + if name in standard_export_headers: + continue # would clobber a real column + custom_export.append((name, csv_header)) + + if ( + call_import.source_format == "audio" + and "conversation_id" not in standard_export_headers + ): + standard_export_headers.insert(0, "conversation_id") + + # Build the metric columns: each parent (if any) gets a value column + # and (when capture_rationale=true) a " - LLM Rationale" + # column. The per-child boolean columns are intentionally suppressed + # — categorization metrics now collapse to exactly two columns in + # the export, mirroring the in-app table. + child_ids_in_groups: set[str] = set() + for parent_str, child_strs in groups_raw.items(): + for child_str in child_strs: + if isinstance(child_str, str): + child_ids_in_groups.add(child_str) + + metric_headers: List[str] = [] + rationale_headers: Dict[str, str] = {} # metric_id_str -> rationale column name + seen_metric_ids: set[str] = set() + + def _add_metric_column(metric: Metric) -> None: + mid_str = str(metric.id) + if mid_str in seen_metric_ids: + return + # Skip any child whose parent is part of this run — the parent + # column above already shows the chosen child name as its + # value. + if mid_str in child_ids_in_groups: + return + seen_metric_ids.add(mid_str) + header = metric_names[mid_str] + metric_headers.append(header) + if bool(getattr(metric, "capture_rationale", False)): + rationale_header = f"{header} - LLM Rationale" + metric_headers.append(rationale_header) + rationale_headers[mid_str] = rationale_header + + for parent_str in groups_raw.keys(): + parent = metrics_by_id.get(parent_str) + if parent: + _add_metric_column(parent) + # Children of an in-run parent are deliberately not emitted — + # the ``child_ids_in_groups`` guard inside ``_add_metric_column`` + # is what enforces this. We still iterate the keys above (not + # ``.items()``) so the parent-only emission is explicit. + # Append anything left over (standalone metrics not in any group, or + # legacy runs without ``selected_metric_groups``). + for metric in metrics: + if metric.selection_mode and not metric.parent_metric_id: + continue # already handled above + if str(metric.id) in seen_metric_ids: + continue + _add_metric_column(metric) + + # Three new fixed columns surface the two transcript fields and the + # evaluation's transcript_source as live values pulled from the + # ``CallImportRow`` (not from the frozen ``raw_columns`` snapshot). + # The user can now compare "what was in the CSV" vs "what the + # diarisation worker produced" without round-tripping through the + # UI, and downstream tools can verify which transcript the metrics + # were computed against. + PRODUCTION_TRANSCRIPT_HEADER = "Production Transcript" + DIARISED_TRANSCRIPT_HEADER = "Diarised Transcript" + EVAL_SOURCE_HEADER = "Evaluated Transcript Source" + + fieldnames = [ + *standard_export_headers, + *[h for h, _ in custom_export], + PRODUCTION_TRANSCRIPT_HEADER, + DIARISED_TRANSCRIPT_HEADER, + EVAL_SOURCE_HEADER, + *metric_headers, + ] + + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + rows = sorted( + load_evaluation_row_pairs(db, eval_id), + key=lambda pair: int(pair[1].row_index or 0), + ) + else: + rows = ( + db.query(CallImportEvaluationRow, CallImportRow) + .join( + CallImportRow, + CallImportRow.id == CallImportEvaluationRow.call_import_row_id, + ) + .filter(CallImportEvaluationRow.evaluation_id == eval_id) + .order_by(CallImportRow.row_index.asc()) + .all() + ) + + def _project_rows() -> Iterator[Dict[str, str]]: + for eval_row, source_row in rows: + row_out: Dict[str, str] = {} + raw = ( + source_row.raw_columns + if isinstance(source_row.raw_columns, dict) + else {} + ) + for header in standard_export_headers: + value = raw.get(header) + if value is None and header == "conversation_id": + value = source_row.conversation_id + row_out[header] = "" if value is None else str(value) + for export_header, csv_header in custom_export: + value = raw.get(csv_header) + row_out[export_header] = "" if value is None else str(value) + + # Live transcripts pulled from the row, NOT from raw_columns, + # so re-diarised values are always reflected in the export. + # Both transcript columns are flattened to a single line so the + # spreadsheet cell doesn't balloon vertically — the in-app + # ``TranscriptView`` still has the DB copy with line breaks + # intact for chat-bubble rendering. + row_out[PRODUCTION_TRANSCRIPT_HEADER] = _flatten_transcript( + source_row.transcript + ) + row_out[DIARISED_TRANSCRIPT_HEADER] = _flatten_transcript( + source_row.diarised_transcript + ) + row_out[EVAL_SOURCE_HEADER] = _evaluated_transcript_source_label( + evaluation, + source_row, + ) + + scores = ( + eval_row.metric_scores + if isinstance(eval_row.metric_scores, dict) + else {} + ) + for metric in metrics: + metric_score = ( + scores.get(str(metric.id)) + if isinstance(scores, dict) + else None + ) + value = ( + metric_score.get("value") + if isinstance(metric_score, dict) + else None + ) + # Parent metrics (selection_mode set) render the chosen + # child name for single_choice or the ";"-joined list of + # true child names for multi_label. + if ( + metric.selection_mode + and not metric.parent_metric_id + and isinstance(metric_score, dict) + ): + if metric.selection_mode == "multi_label": + selected = metric_score.get("selected_child_names") + if isinstance(selected, list): + value = ";".join(str(s) for s in selected) + else: + value = ( + metric_score.get("chosen_child_name") + or metric_score.get("value") + ) + row_out[metric.name] = "" if value is None else str(value) + rationale_header = rationale_headers.get(str(metric.id)) + if rationale_header is not None: + rationale = ( + metric_score.get("rationale") + if isinstance(metric_score, dict) + else None + ) + row_out[rationale_header] = ( + "" if rationale is None else str(rationale) + ) + yield row_out + + base_filename = f"call-import-{call_import_id}-evaluation-{eval_id}" + + if format == "xlsx": + # xlsx is unicode-native (Hindi/Devanagari, emoji, etc.) so the + # UTF-8-BOM dance isn't needed here. ``write_only`` mode keeps + # peak memory bounded for large evaluations because openpyxl + # only buffers the current row. + try: + from openpyxl import Workbook # type: ignore + from openpyxl.cell import WriteOnlyCell # type: ignore + from openpyxl.styles import Font # type: ignore + except ImportError as exc: # pragma: no cover - exercised by pyproject lock + raise HTTPException( + status_code=500, + detail=( + "Excel export requires the 'openpyxl' package which is " + "not installed." + ), + ) from exc + + workbook = Workbook(write_only=True) + worksheet = workbook.create_sheet(title="Evaluation") + + bold_font = Font(bold=True) + header_cells = [] + for header in fieldnames: + cell = WriteOnlyCell(worksheet, value=header) + cell.font = bold_font + header_cells.append(cell) + worksheet.append(header_cells) + + for row_dict in _project_rows(): + worksheet.append([row_dict.get(h, "") for h in fieldnames]) + + buffer = io.BytesIO() + workbook.save(buffer) + xlsx_bytes = buffer.getvalue() + filename = f"{base_filename}.xlsx" + return StreamingResponse( + iter([xlsx_bytes]), + media_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + for row_dict in _project_rows(): + writer.writerow(row_dict) + + # Excel on Windows defaults to the system ANSI codepage (Windows-1252) + # when a CSV has no encoding marker, which turns UTF-8 Hindi/Devanagari + # / any non-ASCII text into mojibake (e.g. ``ठीक`` → ``ठीक``). + # A UTF-8 BOM tells Excel to switch to UTF-8 decoding and is silently + # skipped by every other UTF-8-aware reader (pandas, LibreOffice, + # Google Sheets, etc.), so the data round-trips correctly everywhere. + csv_text = output.getvalue() + # ``utf-8-sig`` adds the UTF-8 BOM so Excel on Windows decodes the file + # as UTF-8 instead of the system codepage. We also declare the same + # codec in the Content-Type header so well-behaved HTTP clients (incl. + # ``httpx`` / ``requests`` in our tests) strip the BOM during decode. + csv_bytes = csv_text.encode("utf-8-sig") + filename = f"{base_filename}.csv" + return StreamingResponse( + iter([csv_bytes]), + media_type="text/csv; charset=utf-8-sig", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +def _report_filename_slug(value: str) -> str: + slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.strip().lower()).strip("-") + return slug or "client" + + +def _pdf_report_actor(principal: Principal) -> tuple[Optional[str], Optional[UUID]]: + created_by = principal.email + if not created_by and principal.user_id: + created_by = str(principal.user_id) + return created_by, principal.user_id + + +def _pdf_report_response_from_row( + row: CallImportEvaluationPdfReport, + *, + cache_hit: bool = False, +) -> CallImportEvaluationPdfReportResponse: + filename = row.filename or "report.pdf" + preview_url, download_url = presigned_urls_for_pdf_report( + row.s3_key or "", + filename, + ) + return CallImportEvaluationPdfReportResponse( + id=str(row.id), + filename=filename, + preview_url=preview_url, + download_url=download_url, + created_at=row.created_at or datetime.now(timezone.utc), + created_by=row.created_by, + report_type=row.report_type, + vendor_name=row.vendor_name, + config_summary=config_summary_from_report_config( + row.report_config if isinstance(row.report_config, dict) else {} + ), + storage_available=bool(row.s3_key), + cache_hit=cache_hit, + ) + + +def _pdf_report_list_item_from_row( + row: CallImportEvaluationPdfReport, +) -> CallImportEvaluationPdfReportListItem: + return CallImportEvaluationPdfReportListItem( + id=str(row.id), + filename=row.filename, + vendor_name=row.vendor_name, + report_type=row.report_type, + created_by=row.created_by, + created_at=row.created_at or datetime.now(timezone.utc), + config_summary=config_summary_from_report_config( + row.report_config if isinstance(row.report_config, dict) else {} + ), + cache_fingerprint=row.cache_fingerprint, + ) + + +def _report_branding_for_import_workspace( + db: Session, + organization_id: UUID, + workspace_id: UUID, + *, + internal_brand_image_id: Optional[str] = None, + external_brand_image_id: Optional[str] = None, +) -> tuple[dict[str, str] | list[str], Optional[str]]: + workspace = ( + db.query(Workspace) + .filter( + Workspace.id == workspace_id, + Workspace.organization_id == organization_id, + ) + .first() + ) + raw = workspace.report_branding if workspace and isinstance(workspace.report_branding, dict) else {} + images = raw.get("images") if isinstance(raw.get("images"), list) else [] + loaded_images: list[dict[str, str]] = [] + for item in images: + if not isinstance(item, dict) or not item.get("s3_key"): + continue + content_type = str(item.get("content_type") or "image/png") + try: + from app.services.storage.s3_service import s3_service + + image_bytes = s3_service.download_file_by_key(str(item["s3_key"])) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Unable to load report branding image for workspace {}: {}", + workspace_id, + exc, + ) + continue + encoded = base64.b64encode(image_bytes).decode("ascii") + role = str(item.get("role") or "generic") + if role not in {"internal", "external", "generic"}: + role = "generic" + loaded_images.append( + { + "id": str(item.get("id") or ""), + "role": role, + "data_uri": f"data:{content_type};base64,{encoded}", + } + ) + + def _pick(role: str, selected_id: Optional[str]) -> Optional[str]: + if selected_id: + for loaded in loaded_images: + if loaded["id"] == selected_id: + return loaded["data_uri"] + for loaded in loaded_images: + if loaded["role"] == role: + return loaded["data_uri"] + return None + + logo_data_uris: dict[str, str] = {} + internal_uri = _pick("internal", internal_brand_image_id) + external_uri = _pick("external", external_brand_image_id) + if internal_uri: + logo_data_uris["internal"] = internal_uri + if external_uri: + logo_data_uris["external"] = external_uri + if ( + not logo_data_uris + and not internal_brand_image_id + and not external_brand_image_id + ): + # Backward compatibility for workspaces that only had a generic logo + # library before the two-slot report header existed. + generic_uris = [ + loaded["data_uri"] + for loaded in loaded_images + if loaded.get("data_uri") + ] + if generic_uris: + heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None + return generic_uris[:4], heading + heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None + return logo_data_uris, heading + + +def _display_metrics_for_pdf_report( + db: Session, + organization_id: UUID, + evaluation: CallImportEvaluation, +) -> list[Metric]: + selected_metric_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) + lookup_ids: List[UUID] = list(selected_metric_ids) + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + for parent_str in groups_raw.keys(): + try: + parent_id = UUID(parent_str) + except (TypeError, ValueError): + continue + if parent_id not in lookup_ids: + lookup_ids.append(parent_id) + + metrics = _metrics_for_ids(db, organization_id, lookup_ids) + child_ids_in_groups: set[str] = set() + for child_strs in groups_raw.values(): + if not isinstance(child_strs, list): + continue + child_ids_in_groups.update(str(child_id) for child_id in child_strs) + + metrics_by_id = {str(metric.id): metric for metric in metrics} + display: list[Metric] = [] + seen: set[str] = set() + + for parent_str in groups_raw.keys(): + parent = metrics_by_id.get(str(parent_str)) + if parent and str(parent.id) not in seen: + display.append(parent) + seen.add(str(parent.id)) + + for metric in metrics: + metric_id = str(metric.id) + if metric_id in seen or metric_id in child_ids_in_groups: + continue + if metric.selection_mode and not metric.parent_metric_id: + continue + display.append(metric) + seen.add(metric_id) + + return display + + +def _metrics_for_clustering( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], +) -> List[Metric]: + """All enabled quality metrics scored in this run, normalized for clustering. + + Hierarchical children are collapsed to their parent metric so cluster + groups render at the category level (e.g. ``AI reveal``) instead of the + child label level (e.g. ``Yes`` / ``No``). + """ + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + aggregate_metric_ids: List[UUID] = [] + for agg in aggregates: + if (agg.metric_category or "quality") == "user_insight": + continue + try: + aggregate_metric_ids.append(UUID(agg.metric_id)) + except (TypeError, ValueError): + continue + if not aggregate_metric_ids: + return [] + + aggregate_metrics = _metrics_for_ids( + db, evaluation.organization_id, aggregate_metric_ids + ) + by_id = {metric.id: metric for metric in aggregate_metrics} + + normalized_ids: List[UUID] = [] + seen: set[UUID] = set() + for metric_id in aggregate_metric_ids: + metric = by_id.get(metric_id) + target_id = ( + metric.parent_metric_id + if metric is not None and metric.parent_metric_id + else metric_id + ) + if target_id in seen: + continue + seen.add(target_id) + normalized_ids.append(target_id) + + metrics = _metrics_for_ids(db, evaluation.organization_id, normalized_ids) + return [ + metric + for metric in metrics + if getattr(metric, "enabled", True) and not _metric_is_user_insight(metric) + ] + + +def _metric_is_user_insight(metric: Metric) -> bool: + if (getattr(metric, "metric_category", "quality") or "quality") == "user_insight": + return True + text_value = " ".join( + str(part or "").lower() + for part in (getattr(metric, "name", ""), getattr(metric, "description", "")) + ) + normalized = text_value.replace("-", " ").replace("_", " ") + phrases = ( + "call context", + "caller context", + "product identification", + "out of scope", + "identity match", + "user identity", + "caller identity", + "frustration trigger", + "video call offer", + "video call reception", + ) + return any(phrase in normalized for phrase in phrases) + + +def _evaluation_rows_for_period( + db: Session, + evaluation_id: UUID, +) -> list[tuple[CallImportEvaluationRow, CallImportRow]]: + return ( + db.query(CallImportEvaluationRow, CallImportRow) + .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) + .filter(CallImportEvaluationRow.evaluation_id == evaluation_id) + .order_by(CallImportRow.row_index.asc()) + .all() + ) + + +def _baseline_candidate_evaluations( + db: Session, + organization_id: UUID, + workspace_id: UUID, + current_evaluation: CallImportEvaluation, + current_period_start: Optional[date], + *, + limit: int = 20, +) -> list[dict[str, Any]]: + candidates = ( + db.query(CallImportEvaluation, CallImport) + .join(CallImport, CallImport.id == CallImportEvaluation.call_import_id) + .filter( + CallImportEvaluation.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + CallImportEvaluation.id != current_evaluation.id, + CallImportEvaluation.status == "completed", + CallImportEvaluation.completed_rows > 0, + ) + .order_by(desc(CallImportEvaluation.created_at)) + .limit(limit * 3) + .all() + ) + items: list[dict[str, Any]] = [] + for candidate_eval, candidate_import in candidates: + rows = _evaluation_rows_for_period(db, candidate_eval.id) + period_start, period_end, period_label, period_display = _report_period_from_rows(rows) + if current_period_start and period_start and period_start >= current_period_start: + continue + dataset = ( + (candidate_import.dataset or "").strip() + or (candidate_import.original_filename or candidate_import.filename or "").strip() + or "Unknown dataset" + ) + evaluation_name = ( + (candidate_eval.name or "").strip() + or str(candidate_eval.id)[:8] + ) + items.append( + { + "evaluation_id": str(candidate_eval.id), + "name": evaluation_name, + "dataset": dataset, + "period_label": period_label, + "period_start": period_start, + "period_end": period_end, + "period_display": period_display, + "completed_rows": int(candidate_eval.completed_rows or 0), + "created_at": candidate_eval.created_at, + "is_default": False, + } + ) + if len(items) >= limit: + break + items.sort( + key=lambda item: ( + item["period_start"] or date.min, + item["created_at"] or datetime.min.replace(tzinfo=timezone.utc), + ), + reverse=True, + ) + if items: + items[0]["is_default"] = True + return items + + +def _resolve_baseline_evaluation( + db: Session, + organization_id: UUID, + workspace_id: UUID, + current_evaluation: CallImportEvaluation, + current_period_start: Optional[date], + baseline_evaluation_id: Optional[str], +) -> Optional[CallImportEvaluation]: + candidates = _baseline_candidate_evaluations( + db, + organization_id, + workspace_id, + current_evaluation, + current_period_start, + ) + allowed_ids = {item["evaluation_id"] for item in candidates} + if baseline_evaluation_id: + baseline_id = str(baseline_evaluation_id).strip() + if baseline_id not in allowed_ids: + raise HTTPException( + status_code=400, + detail="Selected baseline evaluation is not a valid prior run for this report.", + ) + return ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == UUID(baseline_id), + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not candidates: + return None + default_id = candidates[0]["evaluation_id"] + return ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == UUID(default_id), + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + + +def _benchmark_context_for_evaluation( + db: Session, + baseline_evaluation: Optional[CallImportEvaluation], +) -> Optional[dict[str, str]]: + if baseline_evaluation is None: + return None + baseline_import = ( + db.query(CallImport) + .filter(CallImport.id == baseline_evaluation.call_import_id) + .first() + ) + rows = _evaluation_rows_for_period(db, baseline_evaluation.id) + period_start, _period_end, period_label, _period_display = _report_period_from_rows(rows) + dataset = ( + (baseline_import.dataset or "").strip() + if baseline_import and baseline_import.dataset + else None + ) + filename = ( + (baseline_import.original_filename or baseline_import.filename or "").strip() + if baseline_import + else None + ) + evaluation_label = ( + (baseline_evaluation.name or "").strip() + if baseline_evaluation.name + else str(baseline_evaluation.id)[:8] + ) + period = period_label or ( + period_start.isoformat() if period_start else "previous report" + ) + return { + "dataset": dataset or filename or "Unknown dataset", + "evaluation": evaluation_label, + "evaluation_id": str(baseline_evaluation.id), + "period": period, + } + + +def _period_deltas_from_evaluation( + db: Session, + baseline_evaluation: CallImportEvaluation, + current_metric_aggregates: list[dict[str, Any]], + current_evaluation: CallImportEvaluation, + current_eval_rows: List[CallImportEvaluationRow], +) -> dict[str, dict[str, str]]: + baseline_rows = _evaluation_rows_for_period(db, baseline_evaluation.id) + baseline_eval_rows = [eval_row for eval_row, _source_row in baseline_rows] + baseline_aggregate_models = _compute_metric_aggregates( + db, + baseline_evaluation, + baseline_eval_rows, + ) + baseline_metric_aggregates = [ + _aggregate_to_dict(aggregate) for aggregate in baseline_aggregate_models + ] + _metrics, _aggs, policies, _source, _child_map = _clustering_context( + db, current_evaluation, current_eval_rows + ) + metric_by_id = {str(m.id): m for m in _metrics} + current_by_id = { + str(item.get("metric_id")): item for item in current_metric_aggregates + } + previous_by_id = { + str(item.get("metric_id")): item + for item in baseline_metric_aggregates + if isinstance(item, dict) + } + deltas: dict[str, dict[str, str]] = {} + for metric_id, current in current_by_id.items(): + metric = metric_by_id.get(metric_id) + policy = policies.get(metric_id) + previous_raw = previous_by_id.get(metric_id) + if metric is None or policy is None: + deltas[metric_id] = { + "label": "No previous-week baseline", + "detail": "No comparable prior report snapshot was found.", + } + continue + current_pct = failure_rate_percent_from_rows( + current_eval_rows, metric, policy + ) + previous_pct = failure_rate_percent_from_rows( + baseline_eval_rows, metric, policy + ) + if current_pct is None or previous_pct is None: + current_pct = current_pct or _aggregate_primary_percent(current, policy) + previous_pct = ( + previous_pct or _aggregate_primary_percent(previous_raw, policy) + if previous_raw + else None + ) + if current_pct is None or previous_pct is None: + deltas[metric_id] = { + "label": "No previous-week baseline", + "detail": "No comparable prior report snapshot was found.", + } + continue + delta = current_pct - previous_pct + sign = "+" if delta >= 0 else "" + deltas[metric_id] = { + "label": f"{sign}{delta:.1f} pp", + "detail": ( + f"Current report {current_pct:.1f}% vs previous report " + f"{previous_pct:.1f}%" + ), + } + return deltas + + +_DELTA_EXPLANATION_SYSTEM_PROMPT = ( + "You are a senior conversation-analytics reviewer. You will receive " + "week-over-week metric failure-rate deltas plus reconciled failure " + "cluster context per metric.\n\n" + "Return STRICT JSON only:\n" + "{\n" + ' "explanations": {"": "<1-2 sentence explanation of why the delta likely occurred>"}\n' + "}\n\n" + "Constraints:\n" + "- Only include metrics supplied in the prompt.\n" + "- Cluster labels are generated independently each run and are NOT stable " + "IDs. Never compare an unmatched current label to 0% baseline.\n" + "- Use matched_theme_shifts for label-aligned comparisons, " + "gap_label_shifts for structural shifts, and new_themes_current_period " + "for themes that emerged without a baseline match.\n" + "- If reconciliation is uncertain, explain using the numeric delta and " + "gap_label_shifts only.\n" + "- Keep each explanation to 1-2 short sentences (~220 chars).\n" + "- Vendor-safe, factual language; no markdown." +) + + +def _period_delta_explanation_cache_key( + baseline_evaluation_id: UUID, + *, + completed_rows: int, + baseline_completed_rows: int, +) -> str: + return ( + f"{baseline_evaluation_id}:{completed_rows}:" + f"{baseline_completed_rows}:reconciled-v2" + ) + + +def _normalize_cluster_label(label: str) -> str: + return re.sub(r"[^a-z0-9]+", " ", (label or "").lower()).strip() + + +_CLUSTER_LABEL_STOPWORDS = frozenset( + { + "a", + "an", + "the", + "and", + "or", + "during", + "while", + "with", + "for", + "from", + "into", + "general", + "user", + "bot", + "agent", + } +) + + +def _cluster_label_tokens(label: str) -> set[str]: + return { + token + for token in _normalize_cluster_label(label).split() + if token and token not in _CLUSTER_LABEL_STOPWORDS and len(token) > 2 + } + + +def _cluster_label_similarity(left: str, right: str) -> float: + tokens_left = _cluster_label_tokens(left) + tokens_right = _cluster_label_tokens(right) + if not tokens_left or not tokens_right: + return 0.0 + intersection = tokens_left & tokens_right + if not intersection: + return 0.0 + union = tokens_left | tokens_right + jaccard = len(intersection) / len(union) + smaller = tokens_left if len(tokens_left) <= len(tokens_right) else tokens_right + overlap_ratio = len(intersection) / len(smaller) + return max(jaccard, overlap_ratio * 0.85) + + +def _group_clusters_by_gap_label( + clusters: list[dict[str, Any]], +) -> dict[str, list[dict[str, Any]]]: + grouped: dict[str, list[dict[str, Any]]] = {} + for cluster in clusters: + gap_label = str(cluster.get("gap_label") or "UNKNOWN") + grouped.setdefault(gap_label, []).append(cluster) + return grouped + + +def _append_matched_cluster_pair( + matched: list[dict[str, Any]], + current: dict[str, Any], + baseline: dict[str, Any], + *, + match_confidence: float, + match_method: str, +) -> None: + matched.append( + { + "current_label": current.get("label"), + "baseline_label": baseline.get("label"), + "gap_label": current.get("gap_label") or baseline.get("gap_label"), + "current_share_pct": current.get("share_pct"), + "baseline_share_pct": baseline.get("share_pct"), + "share_delta_pp": round( + float(current.get("share_pct") or 0.0) + - float(baseline.get("share_pct") or 0.0), + 1, + ), + "match_confidence": round(match_confidence, 2), + "match_method": match_method, + } + ) + + +def _aggregate_share_by_gap_label( + clusters: list[dict[str, Any]], +) -> dict[str, float]: + totals: dict[str, float] = {} + for cluster in clusters: + gap_label = str(cluster.get("gap_label") or "UNKNOWN") + totals[gap_label] = totals.get(gap_label, 0.0) + float( + cluster.get("share_pct") or 0.0 + ) + return {gap: round(share, 1) for gap, share in totals.items()} + + +def _reconcile_cluster_periods( + current_clusters: list[dict[str, Any]], + baseline_clusters: list[dict[str, Any]], + *, + similarity_threshold: float = 0.35, +) -> dict[str, Any]: + """Align independently-generated cluster labels before delta explanation.""" + matched: list[dict[str, Any]] = [] + current_unmatched = list(current_clusters) + remaining_baseline = list(baseline_clusters) + + current_by_gap = _group_clusters_by_gap_label(current_unmatched) + baseline_by_gap = _group_clusters_by_gap_label(remaining_baseline) + for gap_label in list(current_by_gap): + current_group = current_by_gap.get(gap_label) or [] + baseline_group = baseline_by_gap.get(gap_label) or [] + if len(current_group) != 1 or len(baseline_group) != 1: + continue + current = current_group[0] + baseline = baseline_group[0] + _append_matched_cluster_pair( + matched, + current, + baseline, + match_confidence=0.75, + match_method="single_cluster_per_gap_label", + ) + current_unmatched.remove(current) + remaining_baseline.remove(baseline) + current_by_gap[gap_label] = [] + baseline_by_gap[gap_label] = [] + + for current in list(current_unmatched): + best_idx: Optional[int] = None + best_score = 0.0 + for idx, baseline in enumerate(remaining_baseline): + score = _cluster_label_similarity( + str(current.get("label") or ""), + str(baseline.get("label") or ""), + ) + if current.get("gap_label") == baseline.get("gap_label"): + score += 0.1 + if score > best_score: + best_score = score + best_idx = idx + + if best_idx is not None and best_score >= similarity_threshold: + baseline = remaining_baseline.pop(best_idx) + _append_matched_cluster_pair( + matched, + current, + baseline, + match_confidence=best_score, + match_method="label_similarity", + ) + + matched_current_labels = { + str(item.get("current_label") or "") for item in matched + } + matched_baseline_labels = { + str(item.get("baseline_label") or "") for item in matched + } + current_unmatched = [ + cluster + for cluster in current_clusters + if str(cluster.get("label") or "") not in matched_current_labels + ] + remaining_baseline = [ + cluster + for cluster in baseline_clusters + if str(cluster.get("label") or "") not in matched_baseline_labels + ] + + new_themes = [ + { + "label": cluster.get("label"), + "gap_label": cluster.get("gap_label"), + "share_pct": cluster.get("share_pct"), + "note": "New theme in current period (no close baseline match).", + } + for cluster in current_unmatched + ] + + retired_themes = [ + { + "label": baseline.get("label"), + "gap_label": baseline.get("gap_label"), + "share_pct": baseline.get("share_pct"), + "note": "Theme present in baseline only (retired or renamed).", + } + for baseline in remaining_baseline + ] + + current_gap = _aggregate_share_by_gap_label(current_clusters) + baseline_gap = _aggregate_share_by_gap_label(baseline_clusters) + gap_label_shifts: dict[str, dict[str, float]] = {} + for gap_label in set(current_gap) | set(baseline_gap): + current_share = current_gap.get(gap_label, 0.0) + baseline_share = baseline_gap.get(gap_label, 0.0) + if abs(current_share - baseline_share) >= 0.5: + gap_label_shifts[gap_label] = { + "current_share_pct": current_share, + "baseline_share_pct": baseline_share, + "share_delta_pp": round(current_share - baseline_share, 1), + } + + return { + "matched_theme_shifts": matched, + "new_themes_current_period": new_themes, + "retired_themes_baseline_period": retired_themes, + "gap_label_shifts": gap_label_shifts, + "reconciliation_note": ( + "Cluster labels are generated independently each run and may " + "rename the same failure mode. Do not treat unmatched current " + "labels as 0% in the baseline period." + ), + } + + +def _load_period_delta_explanations_cache( + evaluation: CallImportEvaluation, + cache_key: str, +) -> Optional[dict[str, str]]: + raw = getattr(evaluation, "period_delta_explanations", None) + if not isinstance(raw, dict): + return None + entry = raw.get(cache_key) + if not isinstance(entry, dict): + return None + explanations_raw = entry.get("explanations") + if not isinstance(explanations_raw, dict): + return None + return { + str(metric_id): str(why).strip() + for metric_id, why in explanations_raw.items() + if str(metric_id).strip() and isinstance(why, str) and why.strip() + } + + +def _save_period_delta_explanations_cache( + db: Session, + evaluation: CallImportEvaluation, + cache_key: str, + explanations: dict[str, str], +) -> None: + raw = evaluation.period_delta_explanations + if not isinstance(raw, dict): + raw = {} + updated = dict(raw) + updated[cache_key] = { + "explanations": explanations, + "generated_at": datetime.now(timezone.utc).isoformat(), + } + evaluation.period_delta_explanations = updated + flag_modified(evaluation, "period_delta_explanations") + db.commit() + + +def _cluster_summary_for_metric( + state: Optional[EvaluationMetricClustersState], + metric_id: str, +) -> list[dict[str, Any]]: + if state is None or state.status != "completed": + return [] + for group in state.groups: + if str(group.metric_id) != metric_id: + continue + return [ + { + "label": cluster.label, + "gap_label": cluster.gap_label, + "share_pct": round(cluster.share_pct, 1), + "count": cluster.count, + } + for cluster in group.clusters[:5] + ] + return [] + + +def _merge_delta_why( + raw_deltas: dict[str, dict[str, str]], + explanations: dict[str, str], +) -> dict[str, dict[str, str]]: + if not explanations: + return raw_deltas + merged: dict[str, dict[str, str]] = {} + for metric_id, delta in raw_deltas.items(): + updated = dict(delta) + why = explanations.get(metric_id) + if why: + updated["why"] = why + merged[metric_id] = updated + return merged + + +def _explain_period_deltas( + db: Session, + organization_id: UUID, + evaluation: CallImportEvaluation, + baseline_evaluation: CallImportEvaluation, + raw_deltas: dict[str, dict[str, str]], + *, + min_delta_pp: float = 0.5, +) -> dict[str, dict[str, str]]: + """Attach ``why`` explanations to period deltas using cached LLM output.""" + if not raw_deltas: + return raw_deltas + + cache_key = _period_delta_explanation_cache_key( + baseline_evaluation.id, + completed_rows=evaluation.completed_rows, + baseline_completed_rows=baseline_evaluation.completed_rows, + ) + cached = _load_period_delta_explanations_cache(evaluation, cache_key) + if cached is not None: + return _merge_delta_why(raw_deltas, cached) + + current_clusters = _metric_clusters_payload(evaluation) + baseline_clusters = _metric_clusters_payload(baseline_evaluation) + metrics_for_prompt: list[dict[str, Any]] = [] + for metric_id, delta in raw_deltas.items(): + label = delta.get("label") or "" + if "No previous-week baseline" in label: + continue + match = re.search(r"([+-]?\d+(?:\.\d+)?)\s*pp", label) + if match and abs(float(match.group(1))) < min_delta_pp: + continue + current_summary = _cluster_summary_for_metric(current_clusters, metric_id) + baseline_summary = _cluster_summary_for_metric(baseline_clusters, metric_id) + if not current_summary and not baseline_summary: + continue + cluster_reconciliation = _reconcile_cluster_periods( + current_summary, + baseline_summary, + ) + metrics_for_prompt.append( + { + "metric_id": metric_id, + "delta_label": label, + "delta_detail": delta.get("detail") or "", + "cluster_reconciliation": cluster_reconciliation, + } + ) + + if not metrics_for_prompt: + return raw_deltas + + provider_hint: Optional[str] = None + model_hint: Optional[str] = None + tldr_raw = evaluation.tldr_summary + if isinstance(tldr_raw, dict): + if isinstance(tldr_raw.get("provider"), str): + provider_hint = tldr_raw["provider"] + if isinstance(tldr_raw.get("model"), str): + model_hint = tldr_raw["model"] + + from app.services.ai.llm_resolver import get_llm_provider_and_model + from app.services.call_import_user_insights import _call_llm, _parse_json_object + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, provider_hint, model_hint + ) + try: + text = _call_llm( + db, + organization_id, + provider_enum, + model_str, + [ + {"role": "system", "content": _DELTA_EXPLANATION_SYSTEM_PROMPT}, + { + "role": "user", + "content": json.dumps( + {"metrics": metrics_for_prompt}, + ensure_ascii=False, + default=str, + ), + }, + ], + temperature=0.3, + max_tokens=900, + ) + except Exception as exc: + logger.warning("[PeriodDeltaExplain] LLM call failed: {}", exc) + return raw_deltas + + parsed = _parse_json_object(text) + explanations_raw = parsed.get("explanations") + explanations: dict[str, str] = {} + if isinstance(explanations_raw, dict): + for metric_id, why in explanations_raw.items(): + if isinstance(why, str) and why.strip(): + explanations[str(metric_id)] = why.strip() + + if explanations: + _save_period_delta_explanations_cache( + db, evaluation, cache_key, explanations + ) + return _merge_delta_why(raw_deltas, explanations) + + +def _period_deltas_with_explanations( + db: Session, + organization_id: UUID, + evaluation: CallImportEvaluation, + baseline_evaluation: CallImportEvaluation, + raw_deltas: dict[str, dict[str, str]], +) -> dict[str, dict[str, str]]: + return _explain_period_deltas( + db, + organization_id, + evaluation, + baseline_evaluation, + raw_deltas, + ) + + +def _benchmark_context_for_snapshot( + db: Session, + previous_snapshot: Optional[CallImportEvaluationReportSnapshot], +) -> Optional[dict[str, str]]: + if previous_snapshot is None: + return None + previous_import = ( + db.query(CallImport) + .filter(CallImport.id == previous_snapshot.call_import_id) + .first() + ) + previous_eval = ( + db.query(CallImportEvaluation) + .filter(CallImportEvaluation.id == previous_snapshot.evaluation_id) + .first() + ) + dataset = ( + (previous_import.dataset or "").strip() + if previous_import and previous_import.dataset + else None + ) + filename = ( + (previous_import.original_filename or previous_import.filename or "").strip() + if previous_import + else None + ) + evaluation_label = ( + (previous_eval.name or "").strip() + if previous_eval and previous_eval.name + else str(previous_snapshot.evaluation_id)[:8] + ) + period = previous_snapshot.period_label or ( + previous_snapshot.period_start.isoformat() + if previous_snapshot.period_start + else "previous report" + ) + return { + "dataset": dataset or filename or "Unknown dataset", + "evaluation": evaluation_label, + "evaluation_id": str(previous_snapshot.evaluation_id), + "period": period, + } + + +def _clamp_prose_to_sentences( + text: str, + *, + max_sentences: int = 3, + max_chars: int = 300, +) -> str: + """Keep concise audit/TLDR prose within sentence and character limits.""" + cleaned = (text or "").strip() + if not cleaned: + return cleaned + cleaned = re.sub(r"\s*\n+\s*", " ", cleaned).strip() + sentences = [ + sentence.strip() + for sentence in re.split(r"(?<=[.!?])\s+", cleaned) + if sentence.strip() + ] + if sentences: + result = " ".join(sentences[:max_sentences]).strip() + else: + result = cleaned + if len(result) > max_chars: + trimmed = result[: max_chars - 3].rsplit(" ", 1)[0].rstrip(".,;:") + result = f"{trimmed}..." if trimmed else result[:max_chars] + return result + + +def _audit_summary_text_from_tldr( + summary: Optional[EvaluationTldrSummary], +) -> Optional[str]: + if summary is None: + return None + narrative = _clamp_prose_to_sentences(summary.narrative.strip()) + return narrative or None + + +def _metric_insights_from_tldr( + summary: Optional[EvaluationTldrSummary], +) -> dict[str, str]: + if summary is None: + return {} + return { + str(metric_id): insight.strip() + for metric_id, insight in summary.metric_insights.items() + if str(metric_id).strip() and insight.strip() + } + + +def _report_period_from_rows( + rows: list[tuple[CallImportEvaluationRow, CallImportRow]], +) -> tuple[Optional[date], Optional[date], Optional[str], str]: + dates = [ + source_row.recording_date + for eval_row, source_row in rows + if eval_row.status == "completed" and source_row.recording_date + ] + if not dates: + return None, None, None, "Not specified" + start = min(dates) + end = max(dates) + week_anchor = max(dates) + week_start = week_anchor - timedelta(days=week_anchor.weekday()) + week_end = week_start + timedelta(days=6) + iso_year, iso_week, _ = week_anchor.isocalendar() + label = f"{iso_year}-W{iso_week:02d}" + if week_start.year == week_end.year: + week_range = f"{week_start.strftime('%b %d')}–{week_end.strftime('%b %d, %Y')}" + else: + week_range = ( + f"{week_start.strftime('%b %d, %Y')}–{week_end.strftime('%b %d, %Y')}" + ) + display = f"W{iso_week:02d} · {week_range}" + return start, end, label, display + + +def _aggregate_to_dict(aggregate: CallImportMetricAggregate) -> dict[str, Any]: + if hasattr(aggregate, "model_dump"): + return aggregate.model_dump(mode="json") + return aggregate.dict() + + +def _aggregate_primary_percent( + raw: dict[str, Any], + policy: Optional[MetricFailurePolicy] = None, +) -> Optional[float]: + return aggregate_primary_percent(raw, policy) + + +def _child_names_by_parent( + db: Session, + organization_id: UUID, + parent_metric_ids: Sequence[UUID], +) -> Dict[str, List[str]]: + if not parent_metric_ids: + return {} + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id.in_(list(parent_metric_ids)), + ) + .all() + ) + out: Dict[str, List[str]] = {} + for child in children: + pid = str(child.parent_metric_id) + out.setdefault(pid, []).append(child.name) + return out + + +def _clustering_context( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], +) -> Tuple[ + List[Metric], + List[CallImportMetricAggregate], + Dict[str, MetricFailurePolicy], + Literal["inferred", "user"], + Dict[str, List[str]], +]: + metrics = _metrics_for_clustering(db, evaluation, eval_rows) + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) and not getattr(m, "parent_metric_id", None) + ] + child_names_by_parent = _child_names_by_parent( + db, evaluation.organization_id, parent_ids + ) + policies, source = effective_policies( + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + return metrics, aggregates, policies, source, child_names_by_parent + + +def _period_deltas_from_aggregates( + previous_metric_aggregates: list[dict[str, Any]], + current_metric_aggregates: list[dict[str, Any]], + policies: Optional[Dict[str, MetricFailurePolicy]] = None, +) -> dict[str, dict[str, str]]: + current_by_id = {str(item.get("metric_id")): item for item in current_metric_aggregates} + previous_by_id = { + str(item.get("metric_id")): item + for item in previous_metric_aggregates + if isinstance(item, dict) + } + deltas: dict[str, dict[str, str]] = {} + for metric_id, current in current_by_id.items(): + previous_raw = previous_by_id.get(metric_id) + policy = (policies or {}).get(metric_id) + current_pct = _aggregate_primary_percent(current, policy) + previous_pct = ( + _aggregate_primary_percent(previous_raw, policy) + if previous_raw + else None + ) + if current_pct is None or previous_pct is None: + deltas[metric_id] = { + "label": "No previous-week baseline", + "detail": "No comparable prior report snapshot was found.", + } + continue + delta = current_pct - previous_pct + sign = "+" if delta >= 0 else "" + deltas[metric_id] = { + "label": f"{sign}{delta:.1f} pp", + "detail": f"Current report {current_pct:.1f}% vs previous report {previous_pct:.1f}%", + } + return deltas + + +def _period_deltas_from_snapshot( + previous: Optional[CallImportEvaluationReportSnapshot], + current_metric_aggregates: list[dict[str, Any]], +) -> dict[str, dict[str, str]]: + previous_items = ( + previous.metric_aggregates + if previous and isinstance(previous.metric_aggregates, list) + else [] + ) + return _period_deltas_from_aggregates(previous_items, current_metric_aggregates) + + +def _sample_evidence_for_metrics( + rows: list[tuple[CallImportEvaluationRow, CallImportRow]], + metric_ids: set[str], +) -> dict[str, list[dict[str, str]]]: + samples: dict[str, list[dict[str, str]]] = {metric_id: [] for metric_id in metric_ids} + for eval_row, source_row in rows: + scores = eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} + for metric_id in metric_ids: + if len(samples.get(metric_id, [])) >= 4: + continue + score = scores.get(metric_id) + if not isinstance(score, dict): + continue + rationale = score.get("rationale") + transcript = source_row.diarised_transcript or source_row.transcript or "" + quote = rationale if isinstance(rationale, str) and rationale.strip() else transcript[:350] + if quote: + samples.setdefault(metric_id, []).append( + { + "conversation_id": source_row.conversation_id, + "quote": str(quote).strip()[:500], + } + ) + return samples + + +def _fallback_report_narrative( + insight_aggregates: list[dict[str, Any]], + evidence_samples: dict[str, list[dict[str, str]]], +) -> dict[str, Any]: + observations: dict[str, str] = {} + evidence: dict[str, dict[str, str]] = {} + design_notes: list[str] = [] + for aggregate in insight_aggregates: + metric_id = str(aggregate.get("metric_id") or "") + name = str(aggregate.get("metric_name") or "Insight") + counts = aggregate.get("value_counts") if isinstance(aggregate.get("value_counts"), list) else [] + if counts: + top = counts[0] + total = int(aggregate.get("count") or 0) or sum( + int(item.get("count") or 0) for item in counts if isinstance(item, dict) + ) + pct = (int(top.get("count") or 0) / total) * 100 if total else 0 + observations[metric_id] = ( + f"{top.get('label')} is the dominant {name.lower()} category at {pct:.1f}% of classified calls." + ) + design_notes.append( + f"{name}: {top.get('label')} is the largest segment and should be reviewed for workflow or prompt improvements." + ) + sample = (evidence_samples.get(metric_id) or [{}])[0] + if sample: + evidence[metric_id] = sample + return { + "observations": observations, + "evidence": evidence, + "design_notes": design_notes[:7], + "audit_summary": None, + } + + +def _generate_report_narrative( + db: Session, + organization_id: UUID, + *, + metric_aggregates: list[dict[str, Any]], + insight_aggregates: list[dict[str, Any]], + period_delta_by_metric: dict[str, dict[str, str]], + evidence_samples: dict[str, list[dict[str, str]]], + report_config: dict[str, Any], +) -> dict[str, Any]: + if not insight_aggregates: + return {"observations": {}, "evidence": {}, "design_notes": [], "audit_summary": None} + try: + from app.services.ai.llm_resolver import get_llm_provider_and_model + from app.services.ai.llm_service import llm_service + + provider_enum, model_str = get_llm_provider_and_model(organization_id, db, None, None) + prompt = ( + "You are writing a vendor-safe external call quality audit report. " + "Return strict JSON with keys observations (object keyed by metric_id), " + "evidence (object keyed by metric_id with conversation_id and quote), " + "design_notes (array of concise numbered-note strings), and audit_summary (string). " + "Use only the supplied aggregates and evidence samples.\n\n" + + json.dumps( + { + "metric_aggregates": metric_aggregates[:30], + "insight_aggregates": insight_aggregates, + "period_deltas": period_delta_by_metric, + "evidence_samples": evidence_samples, + "report_config": report_config, + }, + default=str, + ) + ) + llm_result = llm_service.generate_response( + messages=[ + {"role": "system", "content": "Return JSON only. No markdown."}, + {"role": "user", "content": prompt}, + ], + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + temperature=0.2, + max_tokens=1200, + ) + parsed = json.loads(str(llm_result.content or "{}")) + if isinstance(parsed, dict): + fallback = _fallback_report_narrative(insight_aggregates, evidence_samples) + return { + "observations": parsed.get("observations") or fallback["observations"], + "evidence": parsed.get("evidence") or fallback["evidence"], + "design_notes": parsed.get("design_notes") or fallback["design_notes"], + "audit_summary": parsed.get("audit_summary") or fallback["audit_summary"], + } + except Exception as exc: # noqa: BLE001 + logger.warning("Report narrative LLM generation fell back to deterministic text: {}", exc) + return _fallback_report_narrative(insight_aggregates, evidence_samples) + + +@router.get( + "/{eval_id}/baseline-candidates", + response_model=CallImportEvaluationBaselineCandidatesResponse, + operation_id="listCallImportEvaluationBaselineCandidates", +) +async def list_call_import_evaluation_baseline_candidates( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationBaselineCandidatesResponse: + del api_key + call_import = _require_import(db, call_import_id, organization_id) + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + rows = _evaluation_rows_for_period(db, evaluation.id) + period_start, _period_end, _derived_period_label, _period_display = _report_period_from_rows( + rows + ) + candidates = _baseline_candidate_evaluations( + db, + organization_id, + call_import.workspace_id, + evaluation, + period_start, + ) + default_evaluation_id = next( + (item["evaluation_id"] for item in candidates if item.get("is_default")), + None, + ) + return CallImportEvaluationBaselineCandidatesResponse( + items=[CallImportEvaluationBaselineCandidate(**item) for item in candidates], + default_evaluation_id=default_evaluation_id, + ) + + +@router.post( + "/{eval_id}/pdf-report", + operation_id="generateCallImportEvaluationPdfReport", + dependencies=[Depends(require_call_import_capability(REPORTS_GENERATE))], +) +async def generate_call_import_evaluation_pdf_report( + call_import_id: UUID, + eval_id: UUID, + payload: CallImportEvaluationPdfReportRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +): + del api_key + call_import = _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + is_internal = payload.report_type == "internal" + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + rows = sorted( + load_evaluation_row_pairs(db, eval_id), + key=lambda pair: int(pair[1].row_index or 0), + ) + else: + rows = ( + db.query(CallImportEvaluationRow, CallImportRow) + .join( + CallImportRow, + CallImportRow.id == CallImportEvaluationRow.call_import_row_id, + ) + .filter(CallImportEvaluationRow.evaluation_id == eval_id) + .order_by(CallImportRow.row_index.asc()) + .all() + ) + report_config = payload.report_config if isinstance(payload.report_config, dict) else {} + metrics = _display_metrics_for_pdf_report(db, organization_id, evaluation) + configured_quality_ids = { + str(item) + for item in report_config.get("quality_metric_ids", []) + if item + } + configured_insight_ids = { + str(item.get("metric_id") or item) + for item in report_config.get("insights", []) + if item + } + if configured_quality_ids or configured_insight_ids: + allowed_ids = configured_quality_ids | configured_insight_ids + metrics = [metric for metric in metrics if str(metric.id) in allowed_ids] + + eval_rows = [eval_row for eval_row, _source_row in rows] + aggregate_models = _compute_metric_aggregates(db, evaluation, eval_rows) + selected_report_metric_ids = {str(metric.id) for metric in metrics} + aggregate_dicts = [ + _aggregate_to_dict(aggregate) + for aggregate in aggregate_models + if aggregate.metric_id in selected_report_metric_ids + ] + insight_metric_ids = { + str(metric.id) + for metric in metrics + if _metric_is_user_insight(metric) + } + metric_aggregates = [ + item for item in aggregate_dicts if str(item.get("metric_id")) not in insight_metric_ids + ] + insight_aggregates = [ + item for item in aggregate_dicts if str(item.get("metric_id")) in insight_metric_ids + ] + period_start, period_end, derived_period_label, period_display = _report_period_from_rows(rows) + period_label = (payload.period_label or derived_period_label or "").strip() or None + include_period_delta = ( + payload.include_period_delta or payload.include_weekly_delta + ) + previous_snapshot = None + period_delta_by_metric: dict[str, dict[str, str]] = {} + baseline_evaluation: Optional[CallImportEvaluation] = None + if include_period_delta and period_start: + baseline_evaluation = _resolve_baseline_evaluation( + db, + organization_id, + call_import.workspace_id, + evaluation, + period_start, + payload.baseline_evaluation_id, + ) + if baseline_evaluation: + period_delta_by_metric = _period_deltas_from_evaluation( + db, + baseline_evaluation, + metric_aggregates, + evaluation, + [eval_row for eval_row, _ in rows], + ) + period_delta_by_metric = _period_deltas_with_explanations( + db, + organization_id, + evaluation, + baseline_evaluation, + period_delta_by_metric, + ) + benchmark_context = _benchmark_context_for_evaluation(db, baseline_evaluation) + evidence_samples = _sample_evidence_for_metrics(rows, insight_metric_ids) + cached_tldr_summary = _tldr_summary_payload(evaluation) + cached_user_insights = _user_insights_payload(evaluation) + cached_metric_clusters = _metric_clusters_payload(evaluation) + cached_prompt_improvements = _prompt_improvements_payload(evaluation) + generated_insights_for_pdf = _selected_generated_user_insights( + cached_user_insights, + report_config, + ) + metric_clusters_for_pdf = _selected_metric_clusters_for_pdf( + cached_metric_clusters, + report_config, + ) + prompt_improvements_for_pdf = _selected_prompt_improvements_for_pdf( + cached_prompt_improvements, + report_config, + ) + branding_images, custom_heading = _report_branding_for_import_workspace( + db, + organization_id, + call_import.workspace_id, + internal_brand_image_id=payload.internal_brand_image_id, + external_brand_image_id=payload.external_brand_image_id, + ) + eval_row_list = [eval_row for eval_row, _ in rows] + pdf_aggregates = _compute_metric_aggregates(db, evaluation, eval_row_list) + pdf_parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) + and not getattr(m, "parent_metric_id", None) + ] + pdf_child_map = _child_names_by_parent( + db, evaluation.organization_id, pdf_parent_ids + ) + failure_policies_for_pdf, _fp_source = effective_policies( + evaluation, + metrics, + pdf_aggregates, + child_names_by_parent=pdf_child_map, + ) + + from app.services.storage.s3_service import s3_service + + config_fingerprint = compute_pdf_report_config_fingerprint( + report_type=payload.report_type, + include_period_delta=bool(payload.include_period_delta), + include_weekly_delta=bool(payload.include_weekly_delta), + baseline_evaluation_id=payload.baseline_evaluation_id, + internal_brand_image_id=payload.internal_brand_image_id, + external_brand_image_id=payload.external_brand_image_id, + use_case=payload.use_case, + report_config=report_config, + report_heading=custom_heading, + vendor_name=payload.vendor_name, + platform_base_url=payload.platform_base_url, + period_label=period_label, + ) + content_fingerprint = compute_pdf_report_content_fingerprint( + evaluation_status=evaluation.status, + completed_rows=int(evaluation.completed_rows or 0), + total_rows=int(evaluation.total_rows or 0), + failed_rows=int(evaluation.failed_rows or 0), + metric_aggregates=metric_aggregates, + insight_aggregates=insight_aggregates, + period_delta_by_metric=period_delta_by_metric, + benchmark_context=benchmark_context, + metric_metadata=[ + { + "id": str(metric.id), + "name": metric.name, + "description": metric.description, + } + for metric in metrics + ], + failure_policies=failure_policies_for_pdf, + tldr_summary=cached_tldr_summary, + user_insights_for_pdf=generated_insights_for_pdf, + metric_clusters_for_pdf=metric_clusters_for_pdf, + prompt_improvements_for_pdf=prompt_improvements_for_pdf, + ) + cache_fingerprint = compute_pdf_report_cache_fingerprint( + config_fingerprint=config_fingerprint, + content_fingerprint=content_fingerprint, + ) + if s3_service.is_enabled(): + cached_pdf_report = find_cached_pdf_report( + db, + evaluation_id=evaluation.id, + organization_id=organization_id, + cache_fingerprint=cache_fingerprint, + ) + if cached_pdf_report is not None: + logger.info( + "Reusing stored PDF report {} for evaluation {} (cache fingerprint match)", + cached_pdf_report.id, + eval_id, + ) + return _pdf_report_response_from_row(cached_pdf_report, cache_hit=True) + + narrative = _generate_report_narrative( + db, + organization_id, + metric_aggregates=metric_aggregates, + insight_aggregates=insight_aggregates if is_internal else [], + period_delta_by_metric=period_delta_by_metric, + evidence_samples=evidence_samples if is_internal else {}, + report_config=report_config, + ) + + generated_at = datetime.now(timezone.utc) + try: + pdf_started = datetime.now(timezone.utc) + pdf_bytes = await asyncio.to_thread( + call_import_evaluation_pdf_report_service.render_pdf, + vendor_name=payload.vendor_name, + call_import=call_import, + evaluation=evaluation, + metrics=metrics, + rows=rows, + failure_policies=failure_policies_for_pdf, + generated_at=generated_at, + internal=is_internal, + logo_data_uris=branding_images, + custom_heading=custom_heading, + include_weekly_delta=include_period_delta, + period_delta_by_metric=period_delta_by_metric, + use_case=payload.use_case, + period_display=period_display, + total_metric_count=db.query(Metric) + .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) + .count(), + report_config=report_config, + narrative=narrative, + audit_summary=_audit_summary_text_from_tldr(cached_tldr_summary), + metric_insights=_metric_insights_from_tldr(cached_tldr_summary), + benchmark_context=benchmark_context, + generated_user_insights=generated_insights_for_pdf, + user_insights_overview=( + cached_user_insights.overview if cached_user_insights else None + ), + metric_clusters=metric_clusters_for_pdf, + metric_clusters_overview=( + cached_metric_clusters.overview if cached_metric_clusters else None + ), + prompt_improvements=prompt_improvements_for_pdf, + platform_base_url=payload.platform_base_url, + ) + logger.info( + "PDF report render finished in {:.1f}s for evaluation {}", + (datetime.now(timezone.utc) - pdf_started).total_seconds(), + eval_id, + ) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to generate PDF report for call import {} evaluation {}", + call_import_id, + eval_id, + ) + raise HTTPException( + status_code=500, + detail=f"Failed to generate PDF report: {exc}", + ) from exc + + snapshot = CallImportEvaluationReportSnapshot( + evaluation_id=evaluation.id, + call_import_id=call_import.id, + organization_id=organization_id, + workspace_id=call_import.workspace_id, + period_label=period_label, + period_start=period_start, + period_end=period_end, + report_config=report_config, + selected_metric_ids=[str(metric.id) for metric in metrics], + metric_aggregates=metric_aggregates, + insight_aggregates=insight_aggregates, + narrative=narrative, + total_calls=evaluation.total_rows, + selected_metric_count=len(metrics), + total_metric_count=db.query(Metric) + .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) + .count(), + ) + db.add(snapshot) + db.flush() + + filename = ( + f"{_report_filename_slug(payload.vendor_name)}-" + f"{payload.report_type}-quality-metric-audit-{eval_id}.pdf" + ) + + if not s3_service.is_enabled(): + db.commit() + return StreamingResponse( + iter([pdf_bytes]), + media_type="application/pdf", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + report_id = uuid4() + s3_key = build_pdf_report_s3_key( + organization_id=organization_id, + call_import_id=call_import.id, + evaluation_id=evaluation.id, + report_id=report_id, + ) + try: + s3_service.upload_file_by_key( + file_content=pdf_bytes, + key=s3_key, + content_type="application/pdf", + ) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to upload PDF report for evaluation {} to object storage", + eval_id, + ) + db.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to store PDF report: {exc}", + ) from exc + + created_by, created_by_user_id = _pdf_report_actor(principal) + pdf_report = CallImportEvaluationPdfReport( + id=report_id, + evaluation_id=evaluation.id, + call_import_id=call_import.id, + organization_id=organization_id, + workspace_id=call_import.workspace_id, + snapshot_id=snapshot.id, + vendor_name=payload.vendor_name, + report_type=payload.report_type, + filename=filename, + s3_key=s3_key, + report_config=report_config, + cache_fingerprint=cache_fingerprint, + created_by=created_by, + created_by_user_id=created_by_user_id, + ) + db.add(pdf_report) + try: + db.commit() + except IntegrityError: + db.rollback() + try: + s3_service.delete_file_by_key(s3_key) + except Exception: # noqa: BLE001 + logger.warning( + "Failed to delete orphan PDF after cache race for evaluation {}", + eval_id, + ) + raced_winner = find_cached_pdf_report( + db, + evaluation_id=evaluation.id, + organization_id=organization_id, + cache_fingerprint=cache_fingerprint, + ) + if raced_winner is not None: + logger.info( + "PDF report cache race resolved for evaluation {} (winner {})", + eval_id, + raced_winner.id, + ) + return _pdf_report_response_from_row(raced_winner, cache_hit=True) + raise HTTPException( + status_code=500, + detail="Failed to store PDF report due to a concurrent duplicate request.", + ) from None + db.refresh(pdf_report) + return _pdf_report_response_from_row(pdf_report) + + +@router.get( + "/{eval_id}/pdf-reports", + response_model=CallImportEvaluationPdfReportListResponse, + operation_id="listCallImportEvaluationPdfReports", +) +async def list_call_import_evaluation_pdf_reports( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationPdfReportListResponse: + del api_key + _require_import(db, call_import_id, organization_id) + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + rows = ( + db.query(CallImportEvaluationPdfReport) + .filter( + CallImportEvaluationPdfReport.evaluation_id == eval_id, + CallImportEvaluationPdfReport.organization_id == organization_id, + ) + .order_by(desc(CallImportEvaluationPdfReport.created_at)) + .all() + ) + return CallImportEvaluationPdfReportListResponse( + items=[_pdf_report_list_item_from_row(row) for row in rows], + ) + + +@router.get( + "/{eval_id}/pdf-reports/{report_id}", + response_model=CallImportEvaluationPdfReportResponse, + operation_id="getCallImportEvaluationPdfReport", +) +async def get_call_import_evaluation_pdf_report( + call_import_id: UUID, + eval_id: UUID, + report_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationPdfReportResponse: + del api_key + _require_import(db, call_import_id, organization_id) + row = ( + db.query(CallImportEvaluationPdfReport) + .filter( + CallImportEvaluationPdfReport.id == report_id, + CallImportEvaluationPdfReport.evaluation_id == eval_id, + CallImportEvaluationPdfReport.call_import_id == call_import_id, + CallImportEvaluationPdfReport.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="PDF report not found") + if not row.s3_key: + raise HTTPException( + status_code=404, + detail="PDF report file is not available in object storage", + ) + return _pdf_report_response_from_row(row) + + +@router.patch( + "/{eval_id}", + response_model=CallImportEvaluationResponse, + operation_id="updateCallImportEvaluation", +) +async def update_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + payload: CallImportEvaluationUpdate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationResponse: + """Edit metadata on an existing evaluation run (currently just ``name``).""" + + del api_key + _require_import(db, call_import_id, organization_id) + + row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + # Treat unset vs explicit ``None`` differently: unset = leave alone, + # explicit ``None`` or empty string = clear the name. + payload_data = payload.model_dump(exclude_unset=True) + if "name" in payload_data: + row.name = _normalize_name(payload_data["name"]) + + stamp_evaluation_actor(row, principal) + db.commit() + db.refresh(row) + return _serialize_eval(db, row) + + +def _revoke_pending_tasks(evaluation: CallImportEvaluation) -> None: + """Best-effort cancel of any in-flight Celery tasks for an evaluation.""" + + if not evaluation.celery_group_id and not any( + r.celery_task_id for r in evaluation.row_results + ): + return + try: + from app.workers.celery_app import celery_app + + pending_task_ids = [ + eval_row.celery_task_id + for eval_row in evaluation.row_results + if eval_row.celery_task_id + and eval_row.status in {"pending", "running"} + ] + if pending_task_ids: + celery_app.control.revoke(pending_task_ids, terminate=False) + except Exception: + # Best effort — DB delete remains the source of truth. + pass + + +# --------------------------------------------------------------------------- +# User-initiated cancel for in-flight evaluation rows +# --------------------------------------------------------------------------- +# +# Evaluation rows can sit in ``running`` for many minutes when the underlying +# LLM / audio metric call is slow or wedged (the worker carries an 8 min +# soft / 10 min hard time limit). Without a cancel affordance the operator's +# only recourse is to wait for Celery's time limit to fire — or to manually +# mutate the DB. These helpers + the two endpoints below give the UI a +# first-class "Abort" button mirroring the diarisation cancel pattern at +# ``app.api.v1.routes.call_imports`` (``_apply_diarisation_cancel`` etc.). +# +# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` above uses +# ``terminate=False`` because it's called from delete-flow paths where the +# task may simply not get to run (a worker pulls it off the queue and drops +# it). For a user-initiated cancel we want SIGTERM to interrupt the worker +# mid-LLM/audio call so the in-flight HTTP request actually aborts. +# ``terminate=True`` routes the signal to the executing process; we spell +# ``signal="SIGTERM"`` out for clarity even though it's the default. + +# Sentinel error message stamped on cancelled rows. Read by the eval worker's +# ``_was_cancelled_externally`` guard (see +# :mod:`app.workers.tasks.evaluate_call_import_row`) so a worker that's already +# past its slowest operation can't overwrite the cancelled state with its own +# terminal status. Touching either copy means touching both. +EVAL_CANCELLED_BY_USER_ERROR: str = "Evaluation cancelled by user" + + +def _cancellable_eval_states() -> Tuple[str, ...]: + """States that an evaluation row can be cancelled from. + + Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` + state only needs one edit. + """ + return ("pending", "running") + + +def _revoke_eval_task(eval_row: CallImportEvaluationRow) -> None: + """Best-effort revoke of a single eval row's Celery task. + + Always swallows control-plane exceptions — Celery's control bus is + inherently best-effort and a missed revoke is not catastrophic + because the DB row is already flipped to ``failed`` by the caller + before this runs (so the UI immediately reflects the cancel; if + the task happens to finish anyway, the worker's finaliser skips + over the row via :data:`EVAL_CANCELLED_BY_USER_ERROR`). + """ + task_id = (eval_row.celery_task_id or "").strip() + if not task_id: + return + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke( + task_id, terminate=True, signal="SIGTERM" + ) + logger.info( + "Revoked evaluation task {} for eval row {}", + task_id, + eval_row.id, + ) + except Exception as exc: # noqa: BLE001 — revoke is best-effort + logger.warning( + "Failed to revoke evaluation task {} for eval row {}: {}", + task_id, + eval_row.id, + exc, + ) + + +def _apply_evaluation_cancel( + eval_rows: List[CallImportEvaluationRow], +) -> Tuple[int, int]: + """Cancel every cancellable row in ``eval_rows``. + + Returns ``(cancelled, skipped)`` so the caller can build a typed + response without re-querying the DB. The caller is responsible for + ``db.commit()`` after this returns — we deliberately don't commit + here so a batch endpoint can flush all rows in one transaction. + """ + cancellable_states = _cancellable_eval_states() + cancelled = 0 + skipped = 0 + now = datetime.now(timezone.utc) + for eval_row in eval_rows: + if (eval_row.status or "").lower() not in cancellable_states: + skipped += 1 + continue + # Flip the row state BEFORE we revoke so the UI's next poll + # already shows the cancel, even if Celery's control plane is + # slow to ack. + eval_row.status = "failed" + eval_row.error_message = EVAL_CANCELLED_BY_USER_ERROR + eval_row.finished_at = now + _revoke_eval_task(eval_row) + # Drop the task id so a follow-up retry (or a stale poll) can't + # accidentally re-revoke or get confused. + eval_row.celery_task_id = None + cancelled += 1 + return cancelled, skipped + + +def _claim_evaluation_bulk_operation( + evaluation_id: UUID, + operation: str, +) -> None: + """Reserve the run for a single bulk worker pass; 409 if one is active.""" + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + try_set_evaluation_bulk_operation, + ) + + if try_set_evaluation_bulk_operation(evaluation_id, operation): # type: ignore[arg-type] + return + existing = get_evaluation_bulk_operation(evaluation_id) or operation + raise HTTPException( + status_code=409, + detail=( + f"A bulk {existing.replace('_', ' ')} operation is already in " + "progress for this evaluation. Wait for it to finish before " + "starting another action." + ), + ) + + +def _require_no_evaluation_bulk_operation(evaluation_id: UUID) -> None: + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + ) + + existing = get_evaluation_bulk_operation(evaluation_id) + if existing: + raise HTTPException( + status_code=409, + detail=( + f"A bulk {existing.replace('_', ' ')} operation is already in " + "progress for this evaluation. Wait for it to finish before " + "starting another action." + ), + ) + + +@router.post( + "/{eval_id}/cancel", + response_model=CallImportEvaluationBulkActionResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="cancelCallImportEvaluation", +) +async def cancel_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationBulkActionResponse: + """Abort all in-flight (or queued) rows in a single evaluation run. + + Idempotent: calling on a run whose rows are already terminal returns + ``target_count=0`` with 202 so the UI can fire this from an + "Abort" button without having to pre-check the state. + + Heavy row resets and Celery revokes run in a background worker so + large batches do not block the API thread. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets + + target_count = count_evaluation_cancel_targets(db, eval_id, mode="abort") + if target_count == 0: + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=0, + evaluation_id=eval_id, + ) + + _claim_evaluation_bulk_operation(eval_id, "abort") + evaluation.status = "cancelled" + stamp_evaluation_actor(evaluation, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import ( + cancel_call_import_evaluation_task, + ) + + cancel_call_import_evaluation_task.delay(str(eval_id), mode="abort") + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=target_count, + evaluation_id=eval_id, + ) + + +@router.post( + "/{eval_id}/force-fail-pending", + response_model=CallImportEvaluationBulkActionResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="forceFailCallImportEvaluationPending", +) +async def force_fail_pending_call_import_evaluation_rows( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationBulkActionResponse: + """Force-fail only rows currently in ``pending`` for a single run. + + This is narrower than :func:`cancel_call_import_evaluation`: it leaves + ``running`` rows untouched so operators can clear permanently queued rows + without interrupting in-flight evaluations. + + Row updates run in a background worker so large batches do not block + the API thread. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets + + target_count = count_evaluation_cancel_targets( + db, eval_id, mode="force_fail_pending" + ) + if target_count == 0: + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=0, + evaluation_id=eval_id, + ) + + _claim_evaluation_bulk_operation(eval_id, "force_fail_pending") + stamp_evaluation_actor(evaluation, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import ( + cancel_call_import_evaluation_task, + ) + + cancel_call_import_evaluation_task.delay( + str(eval_id), mode="force_fail_pending" + ) + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=target_count, + evaluation_id=eval_id, + ) + + +@router.post( + "/{eval_id}/rows/{eval_row_id}/cancel", + response_model=CallImportEvaluationRowResponse, + status_code=status.HTTP_200_OK, + operation_id="cancelCallImportEvaluationRow", +) +async def cancel_call_import_evaluation_row( + call_import_id: UUID, + eval_id: UUID, + eval_row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationRowResponse: + """Abort an in-flight (or queued) evaluation for a single row. + + Idempotent: calling on a row that's already terminal (``completed`` + / ``failed``) returns the row unchanged with a 200 so the UI can + wire this to a "Stop" button without having to pre-check the + state. Updates the parent run's rollup so its counters reflect + the cancel immediately. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + _require_no_evaluation_bulk_operation(eval_id) + + from app.db_sharding.eval_rows import evaluation_row_session + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + try: + with evaluation_row_session(eval_row_id) as ( + row_db, + _catalog_db, + eval_row, + source_row, + _shard_id, + ): + if eval_row.evaluation_id != eval_id: + raise HTTPException( + status_code=404, + detail="Evaluation row not found in this run", + ) + _apply_evaluation_cancel([eval_row]) + row_db.commit() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + row_db.refresh(eval_row) + return _to_evaluation_row_response(eval_row, source_row, evaluation) + except LookupError as exc: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) from exc + + eval_row = ( + db.query(CallImportEvaluationRow) + .filter( + CallImportEvaluationRow.id == eval_row_id, + CallImportEvaluationRow.evaluation_id == eval_id, + ) + .first() + ) + if not eval_row: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + + _apply_evaluation_cancel([eval_row]) + db.flush() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(eval_row) + + source_row = ( + db.query(CallImportRow) + .filter(CallImportRow.id == eval_row.call_import_row_id) + .first() + ) + + return _to_evaluation_row_response(eval_row, source_row, evaluation) + + +@router.delete( + "/{eval_id}", + status_code=status.HTTP_204_NO_CONTENT, + operation_id="deleteCallImportEvaluation", +) +async def delete_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Response: + del api_key + _require_import(db, call_import_id, organization_id) + + row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + _revoke_pending_tasks(row) + + db.delete(row) + db.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@router.post( + "/bulk-delete", + status_code=status.HTTP_200_OK, + operation_id="bulkDeleteCallImportEvaluations", +) +async def bulk_delete_call_import_evaluations( + call_import_id: UUID, + payload: CallImportEvaluationBulkDelete, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Dict[str, int]: + """Delete multiple evaluation runs scoped to one call import. + + Mirrors :func:`delete_call_import_evaluation` but in bulk so the UI + can clear out a multi-select. Unknown ids (already deleted, or + belonging to a different org/import) are silently skipped — the + response just reports how many actually went away. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + if not payload.evaluation_ids: + return {"deleted": 0} + + rows = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id.in_(payload.evaluation_ids), + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .all() + ) + deleted = 0 + for row in rows: + _revoke_pending_tasks(row) + db.delete(row) + deleted += 1 + db.commit() + return {"deleted": deleted} + + +# --------------------------------------------------------------------------- +# Aggregation: turns per-row metric scores into histograms / value counts. +# +# Designed to be cheap enough to call on every page load: we read each +# evaluation row once, bucket numeric values into a fixed 10-bin +# histogram, and tally the top categorical values. Scaling concerns +# (millions of rows) are deferred — at that point we'd push this into a +# Postgres aggregate query, but for typical CSV imports (<10k rows) the +# Python pass is fast enough and dramatically simpler. +# --------------------------------------------------------------------------- + + +_HISTOGRAM_BUCKETS = 10 +_TOP_VALUE_COUNTS = 10 + + +def _coerce_numeric(value: Any) -> Optional[float]: + """Return ``value`` as ``float`` when it's numeric; ``None`` otherwise.""" + if isinstance(value, bool): + # Booleans are ints in Python; treat them as categorical so + # pass/fail metrics show up in value_counts instead of becoming + # a degenerate {0,1} histogram. + return None + if isinstance(value, (int, float)) and math.isfinite(value): + return float(value) + if isinstance(value, str): + try: + f = float(value) + if math.isfinite(f): + return f + except ValueError: + return None + return None + + +def _coerce_category(value: Any) -> Optional[str]: + """Render ``value`` as a label suitable for a value_counts bucket.""" + if value is None: + return None + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, str): + text = value.strip() + return text or None + # Lists / dicts: stringify so they still group sensibly without + # exploding the cardinality (worst case: everything is "[…]" once). + return str(value) + + +def _build_histogram( + values: List[float], +) -> List[CallImportMetricHistogramBucket]: + """Fixed-bin histogram over ``values``; returns [] for <2 values.""" + if len(values) < 2: + return [] + lo = min(values) + hi = max(values) + if lo == hi: + # All values identical — render a single bucket so the UI shows a + # spike rather than empty space. + return [ + CallImportMetricHistogramBucket(x0=lo, x1=hi, count=len(values)) + ] + width = (hi - lo) / _HISTOGRAM_BUCKETS + buckets: List[List[float]] = [[] for _ in range(_HISTOGRAM_BUCKETS)] + for v in values: + # Right-edge inclusive on the last bucket so ``hi`` doesn't fall + # off into a non-existent bucket index. + idx = int((v - lo) / width) + if idx >= _HISTOGRAM_BUCKETS: + idx = _HISTOGRAM_BUCKETS - 1 + buckets[idx].append(v) + return [ + CallImportMetricHistogramBucket( + x0=lo + i * width, + x1=lo + (i + 1) * width, + count=len(bucket), + ) + for i, bucket in enumerate(buckets) + ] + + +def _percentile(values: List[float], pct: float) -> Optional[float]: + """Linear-interpolated percentile compatible with NumPy default.""" + if not values: + return None + sorted_vals = sorted(values) + if len(sorted_vals) == 1: + return sorted_vals[0] + rank = (pct / 100.0) * (len(sorted_vals) - 1) + lo = int(math.floor(rank)) + hi = int(math.ceil(rank)) + if lo == hi: + return sorted_vals[lo] + frac = rank - lo + return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * frac + + +def _compute_metric_aggregates( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], +) -> List[CallImportMetricAggregate]: + """Collapse per-row ``metric_scores`` into one aggregate per metric. + + Selected metrics are read fresh from the DB so the response always + surfaces the current ``metric.name`` / ``metric_type`` even when a + metric was renamed after the run finished. + """ + + selected_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) + # Include parent metrics from selected_metric_groups so they appear + # alongside their children in the aggregate response. Use ``getattr`` + # with a default so the helper still works for callers that pass + # lightweight objects (tests, in-memory shims) that don't carry the + # attribute at all. + groups_raw_candidate = getattr(evaluation, "selected_metric_groups", None) + groups_raw = ( + groups_raw_candidate if isinstance(groups_raw_candidate, dict) else {} + ) + for parent_str in groups_raw.keys(): + try: + pid = UUID(parent_str) + if pid not in selected_ids: + selected_ids.append(pid) + except (TypeError, ValueError): + continue + + metrics = _metrics_for_ids(db, evaluation.organization_id, selected_ids) + metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} + + # Default to selected metrics, but also include any metric ids that + # surface in row scores even if missing from the metric registry — + # otherwise renaming/deleting a metric mid-run would silently drop + # results from the chart. + discovered_ids: List[str] = list(metric_meta.keys()) + for row in eval_rows: + scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} + for metric_id_str in scores.keys(): + if metric_id_str not in metric_meta and metric_id_str not in discovered_ids: + discovered_ids.append(metric_id_str) + + results: List[CallImportMetricAggregate] = [] + + for metric_id_str in discovered_ids: + meta = metric_meta.get(metric_id_str) + numeric_values: List[float] = [] + category_counts: Dict[str, int] = {} + # For multi-label parents we still need to know how many rows + # were scored (each row votes for >=1 label) so the n-badge in + # the UI shows "n=50" instead of the misleading "n=208" sum. + multi_label_rows_scored = 0 + # Unordered pair tally for the co-occurrence heatmap. Keys are + # ``(label_a, label_b)`` with ``a < b`` so we never double-count + # the same unordered pair. Only populated for multi-label + # parents — every other metric leaves this empty. + multi_label_pair_counts: Dict[Tuple[str, str], int] = {} + skipped = 0 + errored = 0 + observed_metric_type: Optional[str] = None + observed_name: Optional[str] = None + + # ``meta`` is a real ``Metric`` row in production, but tests + # frequently pass a lightweight stub. Pull the two attributes + # we need via ``getattr`` so a stub that only sets ``id`` / + # ``name`` / ``metric_type`` doesn't blow up here. + is_multi_label_parent = bool( + meta + and getattr(meta, "selection_mode", None) == "multi_label" + and not getattr(meta, "parent_metric_id", None) + ) + + for row in eval_rows: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else {} + ) + entry = scores.get(metric_id_str) + if not isinstance(entry, dict): + continue + if entry.get("metric_name"): + observed_name = entry.get("metric_name") + if entry.get("type"): + observed_metric_type = entry.get("type") + if entry.get("skipped"): + skipped += 1 + continue + if entry.get("error"): + errored += 1 + continue + + # Multi-label parents store a comma-joined value that + # isn't useful as a single category; instead tally each + # selected child individually so the chart shows per-label + # counts that mirror the children's own boolean histograms. + if is_multi_label_parent: + selected = entry.get("selected_child_names") + if isinstance(selected, list) and selected: + multi_label_rows_scored += 1 + cleaned: List[str] = [] + for label in selected: + text_label = str(label).strip() or None + if text_label: + cleaned.append(text_label) + category_counts[text_label] = ( + category_counts.get(text_label, 0) + 1 + ) + # Emit one increment per unordered pair of distinct + # labels that fired together on this row. ``cleaned`` + # is deduplicated first because the LLM occasionally + # repeats a label inside ``selected_child_names``. + distinct = sorted(set(cleaned)) + for i in range(len(distinct)): + for j in range(i + 1, len(distinct)): + pair = (distinct[i], distinct[j]) + multi_label_pair_counts[pair] = ( + multi_label_pair_counts.get(pair, 0) + 1 + ) + continue + + value = entry.get("value") + numeric = _coerce_numeric(value) + if numeric is not None: + numeric_values.append(numeric) + continue + category = _coerce_category(value) + if category is not None: + category_counts[category] = category_counts.get(category, 0) + 1 + + # ``count`` is "rows scored". For numeric / single-choice + # metrics that's the same as ``len(numeric) + sum(categories)`` + # because each scored row contributes exactly one observation. + # Multi-label parents however contribute one observation per + # selected child, so summing ``category_counts`` over-counts — + # we tracked rows-scored separately above and use it here. + rows_scored = ( + multi_label_rows_scored + if is_multi_label_parent + else len(numeric_values) + sum(category_counts.values()) + ) + + # Build numeric stats first, then categorical (both can coexist). + agg = CallImportMetricAggregate( + metric_id=metric_id_str, + metric_name=( + (meta.name if meta else observed_name) or "Unknown metric" + ), + metric_type=( + meta.metric_type if meta else observed_metric_type + ), + metric_category=( + "user_insight" + if meta is not None and _metric_is_user_insight(meta) + else "quality" + ) + or "quality", + is_multi_label_parent=is_multi_label_parent, + count=rows_scored, + skipped_count=skipped, + error_count=errored, + ) + if numeric_values: + agg.mean = float(statistics.fmean(numeric_values)) + agg.median = float(statistics.median(numeric_values)) + agg.min = min(numeric_values) + agg.max = max(numeric_values) + agg.stddev = ( + float(statistics.pstdev(numeric_values)) + if len(numeric_values) > 1 + else 0.0 + ) + agg.p25 = _percentile(numeric_values, 25) + agg.p75 = _percentile(numeric_values, 75) + agg.p95 = _percentile(numeric_values, 95) + agg.histogram_buckets = _build_histogram(numeric_values) + if category_counts: + sorted_counts = sorted( + category_counts.items(), key=lambda kv: kv[1], reverse=True + ) + agg.value_counts = [ + CallImportMetricValueCount(label=label, count=count) + for label, count in sorted_counts[:_TOP_VALUE_COUNTS] + ] + # Restrict the heatmap to pairs of labels we actually + # rendered above so the frontend never has to match + # against truncated/missing rows. Sorted desc by pair + # count to keep the most informative cells in the + # response when ``_TOP_VALUE_COUNTS`` clipped the matrix. + if is_multi_label_parent and multi_label_pair_counts: + kept_labels = { + label for label, _ in sorted_counts[:_TOP_VALUE_COUNTS] + } + pair_items = [ + (a, b, count) + for (a, b), count in multi_label_pair_counts.items() + if a in kept_labels and b in kept_labels + ] + pair_items.sort(key=lambda t: t[2], reverse=True) + agg.co_occurrence = [ + CallImportMetricLabelPair(a=a, b=b, count=count) + for a, b, count in pair_items + ] + + results.append(agg) + + # Sort so each parent metric immediately precedes its children. + # The Visualizations grid renders metrics top-to-bottom in this + # order, so multi-label parents (the "summary" chart) sit above + # the per-child boolean histograms that drill into them. Metrics + # whose ``meta`` row was deleted mid-run (``meta is None``) sink + # to the bottom but keep their relative order. + enumerated = list(enumerate(results)) + + def _sort_key(item: Tuple[int, CallImportMetricAggregate]): + original_idx, agg = item + meta = metric_meta.get(agg.metric_id) + if meta is None: + return (1, "", 1, "", original_idx) + parent_id = getattr(meta, "parent_metric_id", None) + # Group key: a child shares its parent's UUID; a parent + # uses its own UUID. Within a group, depth=0 (parent) sorts + # before depth=1 (child); ties break alphabetically by name + # so children render in a stable order regardless of which + # row scored which label first. + if parent_id is None: + group_key = str(meta.id) + depth = 0 + else: + group_key = str(parent_id) + depth = 1 + return ( + 0, + group_key, + depth, + (getattr(meta, "name", "") or "").lower(), + original_idx, + ) + + enumerated.sort(key=_sort_key) + return [agg for _idx, agg in enumerated] + + +@router.get( + "/{eval_id}/aggregate", + response_model=CallImportEvaluationAggregateResponse, + operation_id="getCallImportEvaluationAggregate", +) +async def get_call_import_evaluation_aggregate( + call_import_id: UUID, + eval_id: UUID, + baseline_evaluation_id: Optional[UUID] = Query(None), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationAggregateResponse: + """Return per-metric distributions for the Visualizations tab. + + The shape is intentionally chart-friendly: histograms for numeric + metrics, top-N value counts for categorical/text metrics, plus + summary stats (mean/p50/p95) so the UI can render summary cards + without recomputing on the client. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + + metrics = _compute_metric_aggregates(db, evaluation, eval_rows) + + period_deltas: dict[str, MetricPeriodDelta] = {} + resolved_baseline_id: Optional[UUID] = None + if baseline_evaluation_id is not None: + call_import = _require_import(db, call_import_id, organization_id) + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + + rows = load_evaluation_row_pairs(db, eval_id) + period_start, _, _, _ = _report_period_from_rows(rows) + baseline_evaluation = _resolve_baseline_evaluation( + db, + organization_id, + call_import.workspace_id, + evaluation, + period_start, + str(baseline_evaluation_id), + ) + if baseline_evaluation: + resolved_baseline_id = baseline_evaluation.id + metric_aggregates_dicts = [ + _aggregate_to_dict(agg) for agg in metrics + ] + raw_deltas = _period_deltas_from_evaluation( + db, + baseline_evaluation, + metric_aggregates_dicts, + evaluation, + eval_rows, + ) + raw_deltas = _period_deltas_with_explanations( + db, + organization_id, + evaluation, + baseline_evaluation, + raw_deltas, + ) + period_deltas = { + metric_id: MetricPeriodDelta( + label=delta.get("label") or "", + detail=delta.get("detail") or "", + why=(delta.get("why") or "").strip() or None, + ) + for metric_id, delta in raw_deltas.items() + } + + _fp_stored, failure_policies_source = policies_from_evaluation_raw( + evaluation.metric_clusters + ) + return CallImportEvaluationAggregateResponse( + evaluation_id=eval_id, + total_rows=evaluation.total_rows, + completed_rows=evaluation.completed_rows, + failed_rows=evaluation.failed_rows, + metrics=metrics, + period_deltas=period_deltas, + baseline_evaluation_id=resolved_baseline_id, + failure_policies_source=failure_policies_source, + ) + + +# --------------------------------------------------------------------------- +# TLDR insights: LLM-generated narrative + bullet patterns rendered above +# the Visualizations charts. Cached on ``CallImportEvaluation.tldr_summary`` +# so the page never auto-burns LLM tokens; the user explicitly clicks +# "Generate summary" or "Regenerate" from the empty-state CTA. +# --------------------------------------------------------------------------- + + +_INSIGHTS_SYSTEM_PROMPT = ( + "You are a senior conversation-analytics reviewer. You will be " + "given aggregated metric statistics + a sample of rationales for " + "the rows of a single call-import evaluation. Identify the most " + "useful PATTERNS that hold ACROSS the calls -- not just per-metric " + "numbers. Look for combinations (e.g. `when X happens, Y also " + "tends to happen`), notable outliers, frequent failure modes, and " + "any signal that would change how a reviewer triages the run.\n\n" + "Return STRICT JSON only, with this shape and no extra keys:\n" + "{\n" + ' "narrative": "",\n' + ' "patterns": ["", "", ...],\n' + ' "metric_insights": {"": "<2-3 line business meaning>"}\n' + "}\n\n" + "Constraints:\n" + "- narrative is the ONLY text shown in the external audit summary and " + "Visualizations TLDR; keep it to at most 3 short sentences (~300 chars).\n" + "- patterns are optional supporting notes and are NOT rendered in the " + "audit summary; keep 0 to 3 bullets if supplied, each <= 120 characters.\n" + "- metric_insights must include one entry for each top-level metric id supplied.\n" + "- Each metric insight should explain what the metric means for the business and what the current distribution suggests, not restate the metric rubric.\n" + "- Avoid restating raw counts unless they reveal a pattern.\n" + "- Use neutral, factual language ('frustration appeared in...') " + "rather than judgemental ('the agents failed to...')." +) + + +def _tldr_summary_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationTldrSummary]: + """Return the cached TLDR (with ``is_stale`` set) or ``None``. + + ``CallImportEvaluation.tldr_summary`` is a ``JSON`` column so we + have to validate shape defensively -- a half-written or hand-edited + blob should not break the aggregate response. Returns ``None`` when + no cached summary exists. + """ + raw = evaluation.tldr_summary + if not isinstance(raw, dict): + return None + narrative = raw.get("narrative") + if not isinstance(narrative, str) or not narrative.strip(): + return None + patterns_raw = raw.get("patterns") + patterns = ( + [str(p) for p in patterns_raw if isinstance(p, str) and p.strip()] + if isinstance(patterns_raw, list) + else [] + ) + metric_insights_raw = raw.get("metric_insights") + metric_insights = ( + { + str(metric_id): str(insight).strip() + for metric_id, insight in metric_insights_raw.items() + if str(metric_id).strip() + and isinstance(insight, str) + and insight.strip() + } + if isinstance(metric_insights_raw, dict) + else {} + ) + generated_at_raw = raw.get("generated_at") + try: + generated_at = ( + datetime.fromisoformat(generated_at_raw) + if isinstance(generated_at_raw, str) + else evaluation.updated_at or datetime.now(timezone.utc) + ) + except ValueError: + generated_at = evaluation.updated_at or datetime.now(timezone.utc) + snapshot = raw.get("generated_at_completed_rows") + snapshot_int = int(snapshot) if isinstance(snapshot, (int, float)) else 0 + return EvaluationTldrSummary( + narrative=_clamp_prose_to_sentences(narrative.strip()), + patterns=patterns, + metric_insights=metric_insights, + generated_at=generated_at, + generated_at_completed_rows=snapshot_int, + provider=raw.get("provider") if isinstance(raw.get("provider"), str) else None, + model=raw.get("model") if isinstance(raw.get("model"), str) else None, + is_stale=evaluation.completed_rows > snapshot_int, + ) + + +def _sample_rationales_per_metric( + eval_rows: List[CallImportEvaluationRow], + *, + per_metric_cap: int = 3, + rationale_char_cap: int = 600, +) -> Dict[str, List[str]]: + """Collect up to ``per_metric_cap`` distinct rationales per metric. + + Distinctness is case- and whitespace-insensitive. We truncate each + rationale to ``rationale_char_cap`` so a few unusually verbose rows + can't dominate the prompt budget. Empty / non-string rationales are + skipped. + """ + out: Dict[str, List[str]] = {} + seen: Dict[str, set[str]] = {} + for row in eval_rows: + scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} + for metric_id, entry in scores.items(): + if not isinstance(entry, dict): + continue + rationale = entry.get("rationale") + if not isinstance(rationale, str): + continue + text = rationale.strip() + if not text: + continue + bucket = out.setdefault(metric_id, []) + if len(bucket) >= per_metric_cap: + continue + key = " ".join(text.lower().split()) + seen_set = seen.setdefault(metric_id, set()) + if key in seen_set: + continue + seen_set.add(key) + bucket.append(text[:rationale_char_cap]) + return out + + +def _build_insights_messages( + evaluation: CallImportEvaluation, + aggregate: List[CallImportMetricAggregate], + rationale_samples: Dict[str, List[str]], + metric_meta: Dict[str, Metric], +) -> List[Dict[str, str]]: + """Render the user prompt fed to the LLM. + + The shape is plain markdown-ish text instead of JSON so the LLM can + skim it without us spending tokens on verbose schema delimiters. + Parent metrics surface their child metrics nested underneath so the + model sees the hierarchy and can talk about "X often co-occurred + with Y" rather than treating sub-labels as standalone metrics. + """ + name = evaluation.name or f"Run {str(evaluation.id)[:8]}" + lines: List[str] = [ + f"Evaluation: {name}", + ( + f"Rows: total={evaluation.total_rows} " + f"completed={evaluation.completed_rows} " + f"failed={evaluation.failed_rows}" + ), + "", + "## Per-metric aggregate", + ] + + # Group metrics by parent so the prompt mirrors the hierarchy. Any + # aggregate row whose ``metric_id`` is missing from ``metric_meta`` + # is rendered as a leaf at the top-level list (handles renamed / + # deleted parents). + children_by_parent: Dict[str, List[CallImportMetricAggregate]] = {} + top_level: List[CallImportMetricAggregate] = [] + for agg in aggregate: + meta = metric_meta.get(agg.metric_id) + parent_id = ( + str(meta.parent_metric_id) + if meta is not None and getattr(meta, "parent_metric_id", None) + else None + ) + if parent_id: + children_by_parent.setdefault(parent_id, []).append(agg) + else: + top_level.append(agg) + + def _format_metric_block(agg: CallImportMetricAggregate, indent: int) -> List[str]: + prefix = " " * indent + "- " + bits: List[str] = [f"{prefix}{agg.metric_name} [id={agg.metric_id}] (n={agg.count}"] + if agg.skipped_count: + bits.append(f", skipped={agg.skipped_count}") + if agg.error_count: + bits.append(f", errors={agg.error_count}") + bits.append(")") + meta = metric_meta.get(agg.metric_id) + description = (meta.description or "").strip() if meta else "" + if description: + bits.append(f" | definition={description[:500]}") + if agg.mean is not None: + mean_s = f"{agg.mean:.2f}" + stddev_s = f"{agg.stddev:.2f}" if agg.stddev is not None else "-" + bits.append(f" | mean={mean_s} stddev={stddev_s}") + if agg.min is not None and agg.max is not None: + bits.append(f" range=[{agg.min:.2f}, {agg.max:.2f}]") + if agg.value_counts: + total = sum(v.count for v in agg.value_counts) or 1 + top = agg.value_counts[:3] + shares = ", ".join( + f'"{v.label}"={v.count}/{total}' for v in top + ) + bits.append(f" | top={shares}") + result = ["".join(bits)] + rationales = rationale_samples.get(agg.metric_id, []) + for r in rationales: + result.append(" " * (indent + 1) + f"- rationale: {r}") + return result + + for agg in top_level: + lines.extend(_format_metric_block(agg, indent=0)) + meta = metric_meta.get(agg.metric_id) + children = children_by_parent.get(str(meta.id), []) if meta else [] + for child in children: + lines.extend(_format_metric_block(child, indent=1)) + + lines.append("") + top_level_ids = [agg.metric_id for agg in top_level] + if top_level_ids: + lines.append( + "metric_insights keys must exactly use these top-level metric ids: " + + ", ".join(top_level_ids) + ) + lines.append("") + lines.append( + "Write the JSON object as instructed. Do not include " + "preamble, code fences, or trailing commentary." + ) + + return [ + {"role": "system", "content": _INSIGHTS_SYSTEM_PROMPT}, + {"role": "user", "content": "\n".join(lines)}, + ] + + +def _parse_insights_response(text: str) -> EvaluationTldrSummary: + """Coerce the LLM response into ``narrative`` + ``patterns``. + + Matches the JSON-with-fallback pattern used by + ``app.api.v1.routes.metrics._parse_metric_generation_response``: try + ``json.loads`` first, then fall back to regex extraction of the + first ``{...}`` block. Raises ``HTTPException`` with a 502 when the + response can't be parsed at all. + """ + cleaned = (text or "").strip() + if not cleaned: + raise HTTPException( + status_code=502, detail="LLM returned an empty insights response" + ) + try: + parsed = json.loads(cleaned) + except json.JSONDecodeError: + import re + + match = re.search(r"\{.*\}", cleaned, re.DOTALL) + if not match: + raise HTTPException( + status_code=502, + detail="Could not parse LLM insights response as JSON", + ) + try: + parsed = json.loads(match.group(0)) + except json.JSONDecodeError as e: + raise HTTPException( + status_code=502, + detail=f"Could not parse LLM insights response: {e}", + ) + + if not isinstance(parsed, dict): + raise HTTPException( + status_code=502, detail="LLM insights JSON was not an object" + ) + + narrative = parsed.get("narrative") + if not isinstance(narrative, str) or not narrative.strip(): + raise HTTPException( + status_code=502, + detail="LLM insights JSON missing 'narrative' string", + ) + + patterns_raw = parsed.get("patterns") + if patterns_raw is None: + patterns: List[str] = [] + elif isinstance(patterns_raw, list): + patterns = [ + str(p).strip() + for p in patterns_raw + if isinstance(p, str) and p.strip() + ] + else: + raise HTTPException( + status_code=502, + detail="LLM insights JSON 'patterns' must be a list of strings", + ) + metric_insights_raw = parsed.get("metric_insights") + if metric_insights_raw is None: + metric_insights: Dict[str, str] = {} + elif isinstance(metric_insights_raw, dict): + metric_insights = { + str(metric_id): str(insight).strip() + for metric_id, insight in metric_insights_raw.items() + if str(metric_id).strip() + and isinstance(insight, str) + and insight.strip() + } + else: + raise HTTPException( + status_code=502, + detail="LLM insights JSON 'metric_insights' must be an object", + ) + + return EvaluationTldrSummary( + narrative=_clamp_prose_to_sentences(narrative.strip()), + patterns=patterns, + metric_insights=metric_insights, + generated_at=datetime.now(timezone.utc), + generated_at_completed_rows=0, # filled in by caller + is_stale=False, + ) + + +def _generate_and_persist_tldr_summary( + db: Session, + evaluation: CallImportEvaluation, + *, + organization_id: UUID, + provider: Optional[str] = None, + model: Optional[str] = None, +) -> EvaluationTldrSummary: + """LLM TLDR generation used by the imports-queue Celery worker.""" + eval_id = evaluation.id + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + + pairs = load_evaluation_row_pairs(db, eval_id) + eval_rows = [eval_row for eval_row, _ in pairs] + aggregate = _compute_metric_aggregates(db, evaluation, eval_rows) + if not aggregate: + raise HTTPException( + status_code=400, + detail=( + "No metric data yet. Wait for at least one row to " + "finish scoring before generating a summary." + ), + ) + + metric_ids: List[UUID] = [] + for agg in aggregate: + try: + metric_ids.append(UUID(agg.metric_id)) + except (TypeError, ValueError): + continue + metrics = _metrics_for_ids(db, organization_id, metric_ids) + metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} + + rationale_samples = _sample_rationales_per_metric(eval_rows) + messages = _build_insights_messages( + evaluation, aggregate, rationale_samples, metric_meta + ) + + from app.services.ai.llm_resolver import get_llm_provider_and_model + from app.services.ai.llm_service import llm_service + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, provider, model + ) + + try: + llm_result = llm_service.generate_response( + messages=messages, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + temperature=0.4, + max_tokens=1400, + ) + except Exception as e: + logger.error(f"[CallImportInsights] LLM call failed: {e}") + raise HTTPException( + status_code=502, detail=f"LLM call failed: {e}" + ) from e + + summary = _parse_insights_response(llm_result.get("text", "")) + total = int(evaluation.total_rows or 0) + ui_completed = min(int(evaluation.completed_rows or 0), total) if total else int( + evaluation.completed_rows or 0 + ) + summary.generated_at_completed_rows = ui_completed + summary.provider = provider_enum.value + summary.model = model_str + summary.is_stale = False + + evaluation.tldr_summary = { + "narrative": summary.narrative, + "patterns": summary.patterns, + "metric_insights": summary.metric_insights, + "generated_at": summary.generated_at.isoformat(), + "generated_at_completed_rows": summary.generated_at_completed_rows, + "provider": summary.provider, + "model": summary.model, + } + flag_modified(evaluation, "tldr_summary") + db.commit() + db.refresh(evaluation) + return summary + + +@router.get( + "/{eval_id}/insights", + response_model=Optional[EvaluationTldrSummary], + operation_id="getCallImportEvaluationInsights", +) +async def get_call_import_evaluation_insights( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationTldrSummary]: + """Return the cached TLDR (or ``null``) without contacting the LLM. + + Used by the Visualizations tab on first paint so the empty-state + CTA can show up before the user opts into generation. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _tldr_summary_payload(evaluation) + + +@router.post( + "/{eval_id}/insights", + response_model=EvaluationTldrSummary, + operation_id="generateCallImportEvaluationInsights", +) +async def generate_call_import_evaluation_insights( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationInsightsRequest = Body(default_factory=EvaluationInsightsRequest), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationTldrSummary: + """Generate (or return-cached) the LLM TLDR for an evaluation run. + + Behavior: + + * ``body.regenerate=False`` and a cached summary at the current + ``completed_rows`` watermark exists -> return it as-is. + * ``body.regenerate=False`` and a stale cached summary exists + (``generated_at_completed_rows < completed_rows``) -> return it + with ``is_stale=True``; the UI prompts the user to regenerate. + * Otherwise -> resolve provider+model (auto-detect when omitted), + call the LLM, persist the new summary, return it. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not body.regenerate: + cached = _tldr_summary_payload(evaluation) + if cached is not None: + return cached + + # Run the TLDR LLM on the imports worker (not the default worker or API). + from app.workers.tasks.generate_evaluation_tldr_insights import ( + generate_evaluation_tldr_insights_task, + ) + + try: + task_result = generate_evaluation_tldr_insights_task.apply_async( + kwargs={ + "evaluation_id": str(eval_id), + "call_import_id": str(call_import_id), + "organization_id": str(organization_id), + "provider": body.provider, + "model": body.model, + }, + ).get(timeout=25 * 60) + except Exception as exc: + logger.error( + "[CallImportInsights] TLDR task failed for evaluation {}: {}", + eval_id, + exc, + ) + raise HTTPException( + status_code=502, + detail=f"Summary generation failed: {exc}", + ) from exc + + if isinstance(task_result, dict) and task_result.get("error"): + status_code = int(task_result.get("status_code") or 502) + raise HTTPException( + status_code=status_code, + detail=str(task_result["error"]), + ) + + summary = EvaluationTldrSummary.model_validate(task_result) + db.refresh(evaluation) + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) + + from app.services.ai.llm_resolver import get_llm_provider_and_model + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, body.provider, body.model, body.credential_id + ) + + _enqueue_user_insights_job( + evaluation, + provider=summary.provider or provider_enum.value, + model=summary.model or model_str, + force=body.regenerate, + max_llm_calls=body.max_llm_calls, + db=db, + principal=principal, + ) + + return summary + + +def _user_insights_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationUserInsightsState]: + raw = getattr(evaluation, "user_insights", None) + if raw is None: + return None + return user_insights_state_from_raw( + raw, + completed_rows=evaluation.completed_rows, + ) + + +def _selected_generated_user_insights( + state: Optional[EvaluationUserInsightsState], + report_config: dict[str, Any], +) -> list[dict[str, Any]]: + """Filter and order generated insights for PDF section 03.""" + if state is None or state.status != "completed" or not state.insights: + return [] + + selected_ids = report_config.get("user_insight_ids") + if isinstance(selected_ids, list) and selected_ids: + allowed = {str(item) for item in selected_ids if item} + items = [item for item in state.insights if item.id in allowed] + else: + items = list(state.insights) + + order_raw = report_config.get("order") + order_ids: list[str] = [] + if isinstance(order_raw, dict): + user_order = order_raw.get("user_insights") + if isinstance(user_order, list): + order_ids = [str(item) for item in user_order if item] + + if order_ids: + by_id = {item.id: item for item in items} + ordered = [by_id[iid] for iid in order_ids if iid in by_id] + seen = set(order_ids) + ordered.extend(item for item in items if item.id not in seen) + items = ordered + + return [item.model_dump(mode="json") for item in items] + + +def _enqueue_user_insights_job( + evaluation: CallImportEvaluation, + *, + provider: Optional[str] = None, + model: Optional[str] = None, + force: bool = False, + max_llm_calls: Optional[int] = None, + db: Optional[Session] = None, + principal: Optional[Principal] = None, +) -> None: + """Enqueue background user-insights generation unless already running.""" + current = _user_insights_payload(evaluation) + if current is not None and current.status == "running" and not force: + return + + llm_budget = normalize_max_llm_calls(max_llm_calls) + + completed_count = ( + _count_completed_eval_rows(db, evaluation.id) + if db is not None + else evaluation.completed_rows + ) + total_calls = total_llm_calls_for_rows(completed_count, max_llm_calls=llm_budget) + evaluation.user_insights = { + "status": "running", + "insights": ( + (evaluation.user_insights or {}).get("insights", []) + if isinstance(evaluation.user_insights, dict) + else [] + ), + "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at_completed_rows": evaluation.completed_rows, + "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, + "provider": provider, + "model": model, + "max_llm_calls": llm_budget, + "llm_calls_used": 0, + "error_message": None, + } + if db is not None: + if principal is not None: + stamp_evaluation_actor(evaluation, principal) + flag_modified(evaluation, "user_insights") + db.commit() + + from app.workers.tasks.generate_evaluation_user_insights import ( + generate_evaluation_user_insights_task, + ) + + generate_evaluation_user_insights_task.delay( + str(evaluation.id), + provider=provider, + model=model, + max_llm_calls=llm_budget, + ) + + +@router.get( + "/{eval_id}/user-insights", + response_model=Optional[EvaluationUserInsightsState], + operation_id="getCallImportEvaluationUserInsights", +) +async def get_call_import_evaluation_user_insights( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationUserInsightsState]: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _user_insights_payload(evaluation) + + +@router.post( + "/{eval_id}/user-insights", + response_model=EvaluationUserInsightsState, + operation_id="generateCallImportEvaluationUserInsights", +) +async def generate_call_import_evaluation_user_insights( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationUserInsightsRequest = Body( + default_factory=EvaluationUserInsightsRequest + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationUserInsightsState: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not body.regenerate and not body.force: + cached = _user_insights_payload(evaluation) + if cached is not None and cached.status in {"running", "completed"}: + return cached + + eval_rows = _load_eval_rows(db, eval_id) + if not any(row.status == "completed" for row in eval_rows): + raise HTTPException( + status_code=400, + detail=( + "No completed rows yet. Wait for at least one row to " + "finish scoring before generating user insights." + ), + ) + + from app.services.ai.llm_resolver import get_llm_provider_and_model + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, body.provider, body.model, body.credential_id + ) + + _enqueue_user_insights_job( + evaluation, + provider=provider_enum.value, + model=model_str, + force=body.force or body.regenerate, + max_llm_calls=body.max_llm_calls, + db=db, + principal=principal, + ) + + db.refresh(evaluation) + return _user_insights_payload(evaluation) or EvaluationUserInsightsState( + status="running" + ) + + +def _metric_clusters_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationMetricClustersState]: + raw = getattr(evaluation, "metric_clusters", None) + if raw is None: + return None + return metric_clusters_state_from_raw( + raw, + completed_rows=evaluation.completed_rows, + ) + + +def _selected_metric_clusters_for_pdf( + state: Optional[EvaluationMetricClustersState], + report_config: dict[str, Any], +) -> dict[str, Any]: + if state is None or state.status != "completed": + return {} + sections = report_config.get("sections") + if isinstance(sections, dict) and sections.get("failure_diagnostics") is False: + return {} + payload: dict[str, Any] = { + "groups": [g.model_dump(mode="json") for g in state.groups], + "discovered_problems": [ + d.model_dump(mode="json") for d in state.discovered_problems + ], + } + if state.rca_summary is not None: + payload["rca_summary"] = state.rca_summary.model_dump(mode="json") + return payload + + +def _prompt_improvements_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationPromptImprovementsState]: + from app.services.call_import_prompt_improvements import ( + prompt_improvements_state_from_raw, + ) + + raw = getattr(evaluation, "prompt_improvements", None) + if raw is None: + return None + return prompt_improvements_state_from_raw( + raw, + completed_rows=evaluation.completed_rows, + ) + + +def _selected_prompt_improvements_for_pdf( + state: Optional[EvaluationPromptImprovementsState], + report_config: dict[str, Any], +) -> dict[str, Any]: + if state is None or state.status != "completed": + return {} + sections = report_config.get("sections") + if isinstance(sections, dict) and sections.get("prompt_improvements") is False: + return {} + return { + "imported_agent_id": state.imported_agent_id, + "imported_agent_name": state.imported_agent_name, + "overview": state.overview, + "suggestions": [s.model_dump(mode="json") for s in state.suggestions], + } + + +def _enqueue_prompt_improvements_job( + evaluation: CallImportEvaluation, + *, + imported_agent_id: UUID, + imported_agent_name: str, + provider: Optional[str] = None, + model: Optional[str] = None, + credential_id: Optional[UUID] = None, + force: bool = False, + db: Optional[Session] = None, + principal: Optional[Principal] = None, +) -> None: + current = _prompt_improvements_payload(evaluation) + if current is not None and current.status == "running" and not force: + return + + evaluation.prompt_improvements = { + "status": "running", + "imported_agent_id": str(imported_agent_id), + "imported_agent_name": imported_agent_name, + "suggestions": [], + "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at_completed_rows": evaluation.completed_rows, + "provider": provider, + "model": model, + "error_message": None, + } + if db is not None: + if principal is not None: + stamp_evaluation_actor(evaluation, principal) + flag_modified(evaluation, "prompt_improvements") + db.commit() + + from app.workers.tasks.generate_evaluation_prompt_improvements import ( + generate_evaluation_prompt_improvements_task, + ) + + async_result = generate_evaluation_prompt_improvements_task.apply_async( + kwargs={ + "evaluation_id": str(evaluation.id), + "imported_agent_id": str(imported_agent_id), + "provider": provider, + "model": model, + "credential_id": str(credential_id) if credential_id else None, + }, + queue="imports", + ) + if db is not None and isinstance(evaluation.prompt_improvements, dict): + evaluation.prompt_improvements["celery_task_id"] = async_result.id + flag_modified(evaluation, "prompt_improvements") + db.commit() + + +def _load_eval_rows(db: Session, evaluation_id: UUID) -> List[CallImportEvaluationRow]: + from app.db_sharding.eval_rows import load_evaluation_rows_for_run + + return load_evaluation_rows_for_run(db, evaluation_id) + + +def _count_completed_eval_rows(db: Session, evaluation_id: UUID) -> int: + from app.db_sharding.eval_rows import count_evaluation_rows_for_run + + return count_evaluation_rows_for_run( + db, evaluation_id, statuses=["completed"] + ) + + +def _completed_row_pairs_for_evaluation( + db: Session, + evaluation_id: UUID, +) -> List[Tuple[CallImportEvaluationRow, CallImportRow]]: + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + + row_pairs = load_evaluation_row_pairs(db, evaluation_id) + return [ + (eval_row, source_row) + for eval_row, source_row in row_pairs + if eval_row.status == "completed" + ] + + +def _resolve_metric_cluster_row_selection( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], + evaluation_row_ids: Optional[List[UUID]], + *, + row_limit: Optional[int] = None, + policies: Optional[Dict[str, MetricFailurePolicy]] = None, +) -> Tuple[List[Tuple[CallImportEvaluationRow, CallImportRow]], List[str]]: + """Return filtered completed row pairs and the selected row id strings.""" + completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) + metrics = _metrics_for_clustering(db, evaluation, eval_rows) + if policies is None: + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) + and not getattr(m, "parent_metric_id", None) + ] + child_names_by_parent = _child_names_by_parent( + db, evaluation.organization_id, parent_ids + ) + policies, _ = effective_policies( + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + eligible = list_eligible_cluster_rows( + evaluation, completed_pairs, metrics, policies + ) + eligible_ordered_ids = [str(item["evaluation_row_id"]) for item in eligible] + eligible_id_set = set(eligible_ordered_ids) + + if evaluation_row_ids is None and row_limit is not None: + selected_ids = eligible_ordered_ids[:row_limit] + filtered = filter_completed_row_pairs( + completed_pairs, + [UUID(rid) for rid in selected_ids], + ) + return filtered, selected_ids + + if evaluation_row_ids is None: + selected_ids = eligible_ordered_ids + filtered = filter_completed_row_pairs( + completed_pairs, + [UUID(rid) for rid in selected_ids], + ) + return filtered, selected_ids + + requested = {str(rid) for rid in evaluation_row_ids} + completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} + unknown = sorted(requested - completed_id_set) + if unknown: + raise HTTPException( + status_code=400, + detail=( + "One or more evaluation_row_ids are missing or not completed: " + + ", ".join(unknown[:5]) + + ("…" if len(unknown) > 5 else "") + ), + ) + not_eligible = sorted(requested - eligible_id_set) + if not_eligible: + raise HTTPException( + status_code=400, + detail=( + "Each selected row must have at least one flagged quality metric. " + "Ineligible row(s): " + + ", ".join(not_eligible[:5]) + + ("…" if len(not_eligible) > 5 else "") + ), + ) + selected_ids = sorted(requested) + filtered = filter_completed_row_pairs(completed_pairs, evaluation_row_ids) + return filtered, selected_ids + + +def _enqueue_metric_clusters_job( + evaluation: CallImportEvaluation, + *, + provider: Optional[str] = None, + model: Optional[str] = None, + credential_id: Optional[UUID] = None, + force: bool = False, + max_llm_calls: Optional[int] = None, + evaluation_row_ids: Optional[List[UUID]] = None, + selected_evaluation_row_ids: Optional[List[str]] = None, + failure_policies: Optional[Dict[str, MetricFailurePolicy]] = None, + db: Optional[Session] = None, + principal: Optional[Principal] = None, +) -> None: + current = _metric_clusters_payload(evaluation) + if current is not None and current.status == "running" and not force: + return + + llm_budget = normalize_max_llm_calls(max_llm_calls) + total_calls = 1 + row_ids_for_task: Optional[List[str]] = None + if db is not None: + eval_rows = _load_eval_rows(db, evaluation.id) + if selected_evaluation_row_ids is None: + _, selected_evaluation_row_ids = _resolve_metric_cluster_row_selection( + db, + evaluation, + eval_rows, + evaluation_row_ids, + ) + completed_pairs = filter_completed_row_pairs( + _completed_row_pairs_for_evaluation(db, evaluation.id), + [UUID(rid) for rid in selected_evaluation_row_ids], + ) + metrics = _metrics_for_clustering(db, evaluation, eval_rows) + policies_for_estimate = failure_policies + if policies_for_estimate is None: + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) + and not getattr(m, "parent_metric_id", None) + ] + child_names_by_parent = _child_names_by_parent( + db, evaluation.organization_id, parent_ids + ) + policies_for_estimate, _ = effective_policies( + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + _, total_calls = estimate_metric_clusters_llm_calls( + evaluation, + metrics, + completed_pairs, + policies_for_estimate, + max_llm_calls=llm_budget, + ) + row_ids_for_task = list(selected_evaluation_row_ids) + + prior_raw = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + policy_blob: Dict[str, Any] = {} + if failure_policies: + policy_blob = failure_policies_to_db(failure_policies, source="user") + + evaluation.metric_clusters = { + "status": "running", + "groups": prior_raw.get("groups", []) if isinstance(prior_raw, dict) else [], + "discovered_problems": ( + prior_raw.get("discovered_problems", []) + if isinstance(prior_raw, dict) + else [] + ), + "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at_completed_rows": evaluation.completed_rows, + "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, + "provider": provider, + "model": model, + "max_llm_calls": llm_budget, + "llm_calls_used": 0, + "error_message": None, + "selected_evaluation_row_ids": selected_evaluation_row_ids or [], + **policy_blob, + } + if db is not None: + if principal is not None: + stamp_evaluation_actor(evaluation, principal) + flag_modified(evaluation, "metric_clusters") + db.commit() + + from app.workers.tasks.generate_evaluation_metric_clusters import ( + generate_evaluation_metric_clusters_task, + ) + + async_result = generate_evaluation_metric_clusters_task.apply_async( + kwargs={ + "evaluation_id": str(evaluation.id), + "provider": provider, + "model": model, + "credential_id": str(credential_id) if credential_id else None, + "max_llm_calls": llm_budget, + "evaluation_row_ids": row_ids_for_task, + }, + queue="imports", + ) + if db is not None and isinstance(evaluation.metric_clusters, dict): + evaluation.metric_clusters["celery_task_id"] = async_result.id + flag_modified(evaluation, "metric_clusters") + if principal is not None: + stamp_evaluation_actor(evaluation, principal) + db.commit() + + +def _revoke_metric_clusters_task(evaluation: CallImportEvaluation) -> None: + """Best-effort SIGTERM revoke of the in-flight clustering Celery task.""" + raw = evaluation.metric_clusters + if not isinstance(raw, dict): + return + task_id = str(raw.get("celery_task_id") or "").strip() + if not task_id: + return + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(task_id, terminate=True, signal="SIGTERM") + logger.info( + "Revoked metric-clusters task {} for evaluation {}", + task_id, + evaluation.id, + ) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Failed to revoke metric-clusters task {} for evaluation {}: {}", + task_id, + evaluation.id, + exc, + ) + + +def _apply_metric_clusters_cancel(evaluation: CallImportEvaluation) -> bool: + """Mark clustering as cancelled and revoke the worker task. + + Returns True if a running job was cancelled, False if already terminal. + """ + raw = evaluation.metric_clusters + if not isinstance(raw, dict): + return False + if (raw.get("status") or "").lower() != "running": + return False + + _revoke_metric_clusters_task(evaluation) + progress = raw.get("progress") if isinstance(raw.get("progress"), dict) else {} + evaluation.metric_clusters = { + **raw, + "status": "cancelled", + "error_message": METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + "progress": progress, + "celery_task_id": None, + } + return True + + +@router.get( + "/{eval_id}/metric-clusters/failure-policies", + response_model=MetricFailurePoliciesResponse, + operation_id="getCallImportEvaluationMetricClusterFailurePolicies", +) +async def get_call_import_evaluation_metric_cluster_failure_policies( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> MetricFailurePoliciesResponse: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + metrics, aggregates, policies, source, child_names_by_parent = _clustering_context( + db, evaluation, eval_rows + ) + previews = build_failure_policy_previews( + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + effective=policies, + ) + updated_at = None + raw_mc = evaluation.metric_clusters + if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): + try: + updated_at = datetime.fromisoformat( + str(raw_mc["failure_policies_updated_at"]) + ) + except ValueError: + updated_at = None + return MetricFailurePoliciesResponse( + previews=previews, + policies=policies, + source=source, + updated_at=updated_at, + ) + + +@router.put( + "/{eval_id}/metric-clusters/failure-policies", + response_model=MetricFailurePoliciesResponse, + operation_id="saveCallImportEvaluationMetricClusterFailurePolicies", +) +async def save_call_import_evaluation_metric_cluster_failure_policies( + call_import_id: UUID, + eval_id: UUID, + body: MetricFailurePoliciesSaveRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> MetricFailurePoliciesResponse: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + metrics, aggregates, _existing, _source, child_names_by_parent = _clustering_context( + db, evaluation, eval_rows + ) + try: + validate_failure_policies_for_metrics(body.policies, metrics) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + prior = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + evaluation.metric_clusters = merge_failure_policies_into_raw( + prior, + body.policies, + source="user", + ) + flag_modified(evaluation, "metric_clusters") + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) + + policies, source = policies_from_evaluation_raw(evaluation.metric_clusters) + if source != "user": + source = "user" + previews = build_failure_policy_previews( + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + effective=policies, + ) + updated_at = None + raw_mc = evaluation.metric_clusters + if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): + try: + updated_at = datetime.fromisoformat( + str(raw_mc["failure_policies_updated_at"]) + ) + except ValueError: + updated_at = None + return MetricFailurePoliciesResponse( + previews=previews, + policies=policies, + source="user", + updated_at=updated_at, + ) + + +@router.get( + "/{eval_id}/metric-clusters/eligible-rows", + response_model=MetricClusterEligibleRowsResponse, + operation_id="listCallImportEvaluationMetricClusterEligibleRows", +) +async def list_call_import_evaluation_metric_cluster_eligible_rows( + call_import_id: UUID, + eval_id: UUID, + limit: Optional[int] = Query(default=None, ge=1), + count_only: bool = Query(default=False), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> MetricClusterEligibleRowsResponse: + """Completed rows that have at least one flagged quality metric.""" + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + completed_pairs = _completed_row_pairs_for_evaluation(db, eval_id) + metrics, _aggregates, policies, _source, _child_map = _clustering_context( + db, evaluation, eval_rows + ) + all_eligible = list_eligible_cluster_rows( + evaluation, completed_pairs, metrics, policies + ) + total = len(all_eligible) + if count_only: + return MetricClusterEligibleRowsResponse(items=[], total=total) + raw_items = all_eligible if limit is None else all_eligible[:limit] + items = [MetricClusterEligibleRow.model_validate(item) for item in raw_items] + return MetricClusterEligibleRowsResponse(items=items, total=total) + + +@router.get( + "/{eval_id}/metric-clusters", + response_model=Optional[EvaluationMetricClustersState], + operation_id="getCallImportEvaluationMetricClusters", +) +async def get_call_import_evaluation_metric_clusters( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationMetricClustersState]: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _metric_clusters_payload(evaluation) + + +@router.post( + "/{eval_id}/metric-clusters", + response_model=EvaluationMetricClustersState, + operation_id="generateCallImportEvaluationMetricClusters", +) +async def generate_call_import_evaluation_metric_clusters( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationMetricClustersRequest = Body( + default_factory=EvaluationMetricClustersRequest + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationMetricClustersState: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not body.regenerate and not body.force: + cached = _metric_clusters_payload(evaluation) + if cached is not None and cached.status in {"running", "completed"}: + return cached + + eval_rows = _load_eval_rows(db, eval_id) + if not any(row.status == "completed" for row in eval_rows): + raise HTTPException( + status_code=400, + detail=( + "No completed rows yet. Wait for at least one row to " + "finish scoring before generating metric clusters." + ), + ) + + if body.evaluation_row_ids and body.row_limit is not None: + raise HTTPException( + status_code=400, + detail="Specify either evaluation_row_ids or row_limit, not both.", + ) + + if body.evaluation_row_ids: + completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) + completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} + requested = {str(rid) for rid in body.evaluation_row_ids} + unknown = sorted(requested - completed_id_set) + if unknown: + raise HTTPException( + status_code=400, + detail=( + "One or more evaluation_row_ids are missing or not completed: " + + ", ".join(unknown[:5]) + + ("…" if len(unknown) > 5 else "") + ), + ) + + from app.services.ai.llm_resolver import get_llm_provider_and_model + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, body.provider, body.model, body.credential_id + ) + + metrics, aggregates, _inferred, _source, child_names_by_parent = _clustering_context( + db, evaluation, eval_rows + ) + merged_policies = merge_clustering_policies( + body.failure_policies, + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + try: + validate_failure_policies_for_metrics( + body.failure_policies or merged_policies, metrics + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + if not has_clusterable_metrics(metrics, merged_policies, eval_rows): + raise HTTPException( + status_code=400, + detail=( + "No calls match any failure policy. Select failure values on " + "metrics that have matching rows, or leave metrics with no " + "failures unchecked — they are skipped automatically." + ), + ) + + filtered_pairs, selected_row_ids = _resolve_metric_cluster_row_selection( + db, + evaluation, + eval_rows, + body.evaluation_row_ids, + row_limit=body.row_limit, + policies=merged_policies, + ) + if not selected_row_ids: + raise HTTPException( + status_code=400, + detail=( + "No eligible rows to cluster. Select completed calls that match " + "at least one configured failure policy." + ), + ) + if not filtered_pairs: + raise HTTPException( + status_code=400, + detail="No completed rows match the selected evaluation_row_ids.", + ) + + _enqueue_metric_clusters_job( + evaluation, + provider=provider_enum.value, + model=model_str, + credential_id=body.credential_id, + force=body.force or body.regenerate, + max_llm_calls=body.max_llm_calls, + evaluation_row_ids=body.evaluation_row_ids, + selected_evaluation_row_ids=selected_row_ids, + failure_policies=merged_policies, + db=db, + principal=principal, + ) + + db.refresh(evaluation) + return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( + status="running" + ) + + +@router.post( + "/{eval_id}/metric-clusters/cancel", + response_model=EvaluationMetricClustersState, + operation_id="cancelCallImportEvaluationMetricClusters", +) +async def cancel_call_import_evaluation_metric_clusters( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationMetricClustersState: + """Abort in-flight failure-diagnostics clustering. + + Idempotent: if clustering is not ``running``, returns the current state + unchanged. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + _apply_metric_clusters_cancel(evaluation) + flag_modified(evaluation, "metric_clusters") + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) + + return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( + status="idle" + ) + + +@router.get( + "/{eval_id}/prompt-improvements", + response_model=Optional[EvaluationPromptImprovementsState], + operation_id="getCallImportEvaluationPromptImprovements", +) +async def get_call_import_evaluation_prompt_improvements( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationPromptImprovementsState]: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _prompt_improvements_payload(evaluation) + + +@router.post( + "/{eval_id}/prompt-improvements", + response_model=EvaluationPromptImprovementsState, + operation_id="generateCallImportEvaluationPromptImprovements", +) +async def generate_call_import_evaluation_prompt_improvements( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationPromptImprovementsRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationPromptImprovementsState: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + clusters = _metric_clusters_payload(evaluation) + if clusters is None or clusters.status != "completed": + raise HTTPException( + status_code=400, + detail=( + "Metric clusters must be completed before generating prompt " + "improvements. Run failure diagnostics first." + ), + ) + + from app.services.call_import_prompt_improvements import is_imported_agent + from app.services.ai.llm_resolver import get_llm_provider_and_model + + imported_agent = ( + db.query(PromptPartial) + .filter( + PromptPartial.id == body.imported_agent_id, + PromptPartial.organization_id == organization_id, + PromptPartial.workspace_id == workspace_id, + ) + .first() + ) + if imported_agent is None or not is_imported_agent(imported_agent): + raise HTTPException( + status_code=404, + detail="Imported agent not found in the active workspace", + ) + + if not body.regenerate and not body.force: + cached = _prompt_improvements_payload(evaluation) + if ( + cached is not None + and cached.status in {"running", "completed"} + and cached.imported_agent_id == str(body.imported_agent_id) + ): + return cached + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, body.provider, body.model, body.credential_id + ) + + _enqueue_prompt_improvements_job( + evaluation, + imported_agent_id=body.imported_agent_id, + imported_agent_name=imported_agent.name, + provider=provider_enum.value, + model=model_str, + credential_id=body.credential_id, + force=body.force or body.regenerate, + db=db, + principal=principal, + ) + + db.refresh(evaluation) + return _prompt_improvements_payload(evaluation) or EvaluationPromptImprovementsState( + status="running", + imported_agent_id=str(body.imported_agent_id), + imported_agent_name=imported_agent.name, + ) + + +# --------------------------------------------------------------------------- +# Flow chart: turns per-row LLM-inferred ``sequence`` arrays into a +# directed graph of (label -> label) transitions across the whole run. +# Powers the aggregate Sankey-style React Flow chart on the evaluation +# overview; per-call flow charts are built client-side from the same +# ``sequence`` field on a single row's metric_scores entry. +# --------------------------------------------------------------------------- + + +_FLOW_TERMINAL_THRESHOLD = 0.2 # Mark as terminal when >=20% of sequences end here. +_FLOW_START_NODE_ID = "__START__" +_DISCOVERED_NODE_PREFIX = "disc:" + + +def _slug_label(value: Any) -> str: + """Lowercase + whitespace-collapse + underscore-join. + + Used everywhere we need a stable key for a metric/label name — + matching the same convention the worker uses when emitting + ``sequence`` entries and discovered keys. + """ + if value is None: + return "" + return "_".join(str(value).strip().lower().split()) + + +def _resolve_alias(alias_map: Dict[str, str], key: str) -> str: + """Walk the alias map until we hit a slug that doesn't redirect. + + The merge endpoint stores ``from_slug -> to_slug`` pairs. The delete + endpoint stores ``from_slug -> ""`` (empty string sentinel) to mark + a slug as tombstoned. Chains can accumulate when the user merges + A→B and later merges B→C; this helper collapses them so callers + always land on the final canonical slug. + + Returns: + * the canonical slug if it still resolves to a real label, + * an empty string if the slug has been tombstoned (callers MUST + treat an empty result as "drop this entry entirely"), + * the input ``key`` if it isn't aliased. + + Cycles are guarded by a hard step limit since the alias map is + user-driven. + """ + if not key: + return "" + if not alias_map: + return key + current = key + seen: set[str] = set() + for _ in range(16): + if current in seen: + return current + seen.add(current) + if current not in alias_map: + return current + nxt = alias_map[current] + if nxt == current: + return current + if nxt == "": + # Deletion sentinel — the user has explicitly retired this + # slug. Propagate the empty string up so callers drop it. + return "" + current = nxt + return current + + +# Reserved JSON key under which the worker stores top-level metric +# discoveries on each row's ``metric_scores`` dict. Mirrors the constant +# in ``app/workers/tasks/helpers/llm_evaluation.py`` — kept local here to +# avoid a worker import cycle from the routes module. +DISCOVERED_METRICS_KEY = "__discovered_metrics__" + +# Allowed values for an LLM-suggested top-level metric type. Kept in +# sync with ``DiscoveredMetricSuggestedType`` in +# ``app/models/schemas.py``. +_DISCOVERED_METRIC_TYPES = ("boolean", "rating", "category") + + +def normalize_scores_with_aliases( + metric_scores: Dict[str, Any], + evaluation: CallImportEvaluation, + db: Session, + organization_id: UUID, +) -> Dict[str, Any]: + """Rewrite per-row ``metric_scores`` to honor merges + promotions. + + Called by the worker right after ``evaluate_with_llm`` returns so + every row that finishes AFTER a user has merged or promoted a + discovered label persists data already reflecting that decision. + Without this hook, a worker holding a stale prompt could re-emit a + ``from_key`` slug long after the user merged it away. + + For every parent entry (``selection_mode != null`` and a + ``discovered_labels`` / ``sequence`` field) we: + + * resolve discovered slugs through the evaluation's + ``discovered_label_aliases`` map (transitively), + * drop any discovered_labels entry whose canonical slug now + matches a real promoted child of the parent (merging them out + of the panel for free), and + * collapse adjacent duplicate sequence entries that result. + + Returns ``metric_scores`` (mutated in place) for chaining. + """ + if not isinstance(metric_scores, dict): + return metric_scores + + aliases_top = ( + evaluation.discovered_label_aliases + if isinstance(evaluation.discovered_label_aliases, dict) + else {} + ) + + # Identify the parent entries inside metric_scores. They're the + # dicts that carry a ``selection_mode`` key (set by the LLM + # hierarchy parser) and either a ``sequence`` or a + # ``discovered_labels`` list. + for key, entry in list(metric_scores.items()): + if not isinstance(entry, dict): + continue + if entry.get("type") != "category" and not entry.get("selection_mode"): + continue + try: + parent_uuid = UUID(str(key)) + except (TypeError, ValueError): + continue + + alias_map = {} + sub = aliases_top.get(str(parent_uuid)) + if isinstance(sub, dict): + alias_map = { + str(k): str(v) + for k, v in sub.items() + if isinstance(k, str) and isinstance(v, str) + } + promoted = _promoted_child_slugs(db, parent_uuid, organization_id) + + # Rewrite discovered_labels: alias-resolve keys, drop duplicates + # post-resolution, and drop entries that have been promoted. + discovered = entry.get("discovered_labels") + if isinstance(discovered, list): + kept_disc: List[Dict[str, Any]] = [] + seen: set[str] = set() + for d in discovered: + if not isinstance(d, dict): + continue + slug = _slug_label(d.get("key") or d.get("name")) + slug = _resolve_alias(alias_map, slug) + if not slug or slug in promoted or slug in seen: + continue + seen.add(slug) + new_entry = dict(d) + new_entry["key"] = slug + kept_disc.append(new_entry) + entry["discovered_labels"] = kept_disc + + # Rewrite sequence: alias-resolve every entry; collapse adjacent + # duplicates that result. We DON'T drop slugs that match + # promoted children — the promoted child slug is still a valid + # sequence entry; the flow chart will resolve it to the real + # child node. + seq = entry.get("sequence") + if isinstance(seq, list): + new_seq: List[str] = [] + last: Optional[str] = None + for item in seq: + if not isinstance(item, str): + continue + slug = _resolve_alias(alias_map, _slug_label(item)) + if not slug or slug == last: + continue + new_seq.append(slug) + last = slug + entry["sequence"] = new_seq + + # Top-level metric discoveries live alongside the parent entries + # under the reserved ``DISCOVERED_METRICS_KEY`` slot. Apply the + # flat evaluation-level alias/tombstone map + suppress slugs that + # already correspond to a real top-level Metric so workers that + # finish AFTER the user has merged / deleted / promoted can't + # resurrect a retired candidate. + discovered_metrics_payload = metric_scores.get(DISCOVERED_METRICS_KEY) + if isinstance(discovered_metrics_payload, list): + flat_alias_map = ( + evaluation.discovered_metric_aliases + if isinstance(evaluation.discovered_metric_aliases, dict) + else {} + ) + promoted_metric_slugs = _promoted_top_level_metric_slugs( + db, organization_id + ) + kept_metrics: List[Dict[str, Any]] = [] + seen_metrics: set[str] = set() + for d in discovered_metrics_payload: + if not isinstance(d, dict): + continue + slug = _slug_label(d.get("key") or d.get("name")) + slug = _resolve_alias(flat_alias_map, slug) + if ( + not slug + or slug in promoted_metric_slugs + or slug in seen_metrics + ): + continue + seen_metrics.add(slug) + new_entry = dict(d) + new_entry["key"] = slug + kept_metrics.append(new_entry) + if kept_metrics: + metric_scores[DISCOVERED_METRICS_KEY] = kept_metrics + else: + # No survivors — drop the empty array so empty-discovery rows + # keep their pre-feature payload shape. + metric_scores.pop(DISCOVERED_METRICS_KEY, None) + + return metric_scores + + +def _alias_map_for_parent( + evaluation: CallImportEvaluation, parent_metric_id: UUID +) -> Dict[str, str]: + """Pull ``{from_slug: to_slug}`` for one parent out of the eval's blob. + + Stored shape on the evaluation row is + ``{parent_id_str: {from_slug: to_slug, ...}}``. Returns an empty + dict for parents that have never had a merge applied. + """ + raw = getattr(evaluation, "discovered_label_aliases", None) + if not isinstance(raw, dict): + return {} + submap = raw.get(str(parent_metric_id)) + if not isinstance(submap, dict): + return {} + return { + str(k): str(v) + for k, v in submap.items() + if isinstance(k, str) and isinstance(v, str) + } + + +def _promoted_child_slugs( + db: Session, parent_metric_id: UUID, organization_id: UUID +) -> set[str]: + """Slugs of every real child currently sitting under the parent. + + The Discovered Labels panel hides any candidate whose slug already + matches a real child — that covers both freshly-promoted candidates + and legacy children the LLM happened to re-discover. We pull from + the live ``metrics`` table rather than the eval's + ``selected_metric_groups`` snapshot so newly-promoted children take + effect immediately, even on evaluations that ran before the + promotion. + """ + children = ( + db.query(Metric.name) + .filter( + Metric.parent_metric_id == parent_metric_id, + Metric.organization_id == organization_id, + ) + .all() + ) + out: set[str] = set() + for (name,) in children: + slug = _slug_label(name) + if slug: + out.add(slug) + return out + + +def _promoted_top_level_metric_slugs( + db: Session, organization_id: UUID +) -> set[str]: + """Slugs of every top-level (non-child) Metric in the organization. + + Used to suppress discovered-metric candidates whose slug already + matches a real standalone metric. We intentionally include both + standalone metrics AND parent category metrics — a top-level + discovery that collides with either name is a duplicate by + definition. + """ + rows = ( + db.query(Metric.name) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id.is_(None), + ) + .all() + ) + out: set[str] = set() + for (name,) in rows: + slug = _slug_label(name) + if slug: + out.add(slug) + return out + + +def _get_running_discovered_labels( + db: Session, + eval_id: UUID, + parent_metric_id: UUID, + organization_id: Optional[UUID] = None, + alias_map: Optional[Dict[str, str]] = None, +) -> List[Dict[str, Any]]: + """Slug-deduped view of every discovered label seen in this eval so far. + + Walks each ``call_import_evaluation_rows`` row's + ``metric_scores[parent_id]["discovered_labels"]`` and folds entries + that share the same slug. Returns a list ordered by descending + count and stable on label key, shaped like:: + + [{"key": "customer_on_hold", "name": "Customer put on hold", + "description": "...", "sample_rationale": "...", "count": 12}] + + Powers two callers: + * The worker prompt builder ("REUSE the existing key if it fits") + — invoked just before each row's LLM call to feed the model the + running list of previously-discovered labels in this evaluation. + * The ``/discovered-labels`` API surface used by the frontend + Discovered Labels panel to render candidates with counts + + sample rationales. + + Non-completed rows are skipped: an in-flight row's discoveries are + not yet reliable (the row could fail and never produce final + metric_scores). We accept the tradeoff that rows running + concurrently won't see each other's labels — slug-collision dedup + catches identical re-inventions, and near-paraphrases surface in + the UI panel where the user can manually merge. + """ + + parent_id_str = str(parent_metric_id) + from app.db_sharding.eval_rows import load_evaluation_rows_for_run + + eval_rows = load_evaluation_rows_for_run(db, eval_id) + rows = [ + (row.metric_scores,) + for row in eval_rows + if row.status == CallImportRowStatus.COMPLETED.value + ] + + # Suppress slugs that have either: + # * been promoted to a real child of the parent (so the panel doesn't + # keep nagging the user about a candidate they've already + # accepted), or + # * been merged INTO another slug (the "from" side of a merge) — + # those occurrences fold into the canonical target instead. + promoted_slugs: set[str] = set() + if organization_id is not None: + promoted_slugs = _promoted_child_slugs( + db, parent_metric_id, organization_id + ) + aliases = alias_map or {} + + by_key: Dict[str, Dict[str, Any]] = {} + for (scores,) in rows: + if not isinstance(scores, dict): + continue + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + continue + discovered = parent_entry.get("discovered_labels") + if not isinstance(discovered, list): + continue + for entry in discovered: + if not isinstance(entry, dict): + continue + raw_key = entry.get("key") or entry.get("name") + key = _slug_label(raw_key) + if not key: + continue + # Apply user merges + deletions first, THEN drop anything + # that ended up on a real child slug. Order matters: a + # candidate that was merged into a slug which has since + # been promoted should disappear, not show up at the + # canonical slug. An empty resolved key means the slug was + # tombstoned via the delete endpoint. + key = _resolve_alias(aliases, key) + if not key or key in promoted_slugs: + continue + name = (entry.get("name") or "").strip() or key.replace("_", " ") + description = (entry.get("description") or "").strip() or None + sample = (entry.get("rationale") or "").strip() or None + + existing = by_key.get(key) + if existing is None: + # Track up to N=3 distinct rationales per candidate so + # the Promote-to-child flow can pre-fill the new + # sub-metric's rubric with concrete LLM examples + # without the user copy-pasting from the row table. + # ``sample_rationale`` is preserved for back-compat + # with older clients; ``examples`` is the new field. + examples = [sample] if sample else [] + by_key[key] = { + "key": key, + "name": name, + "description": description, + "sample_rationale": sample, + "examples": examples, + "count": 1, + } + continue + + existing["count"] += 1 + if not existing["description"] and description: + existing["description"] = description + if not existing["sample_rationale"] and sample: + existing["sample_rationale"] = sample + # Append distinct rationales (case-insensitive trim) up + # to a small cap. Headroom is intentionally one above + # what the UI surfaces (2) so we have a backup when the + # first rationale is unhelpful. + if sample: + ex_list: List[str] = existing.setdefault("examples", []) + if len(ex_list) < 3 and not any( + s.strip().lower() == sample.strip().lower() for s in ex_list + ): + ex_list.append(sample) + + return sorted( + by_key.values(), + key=lambda item: (-item["count"], item["key"]), + ) + + +def _get_running_discovered_metrics( + db: Session, + eval_id: UUID, + organization_id: Optional[UUID] = None, + alias_map: Optional[Dict[str, str]] = None, +) -> List[Dict[str, Any]]: + """Slug-deduped view of every discovered top-level metric in this eval. + + Mirrors :func:`_get_running_discovered_labels` but is keyed at the + evaluation level (no ``parent_metric_id``). Walks each completed + row's ``metric_scores[DISCOVERED_METRICS_KEY]`` list, folds entries + that share the same slug (post-alias resolution), and suppresses + slugs that already correspond to a real top-level :class:`Metric` + in the organization. + + Each returned entry is shaped:: + + {"key": "customer_satisfaction", + "name": "Customer Satisfaction", + "description": "...", + "suggested_type": "boolean" | "rating" | "category", + "sample_rationale": "...", + "examples": ["..."], + "count": 12} + """ + + from app.db_sharding.eval_rows import load_evaluation_rows_for_run + + eval_rows = load_evaluation_rows_for_run(db, eval_id) + rows = [ + (row.metric_scores,) + for row in eval_rows + if row.status == CallImportRowStatus.COMPLETED.value + ] + + promoted_slugs: set[str] = set() + if organization_id is not None: + promoted_slugs = _promoted_top_level_metric_slugs( + db, organization_id + ) + aliases = alias_map or {} + + by_key: Dict[str, Dict[str, Any]] = {} + for (scores,) in rows: + if not isinstance(scores, dict): + continue + discovered = scores.get(DISCOVERED_METRICS_KEY) + if not isinstance(discovered, list): + continue + for entry in discovered: + if not isinstance(entry, dict): + continue + raw_key = entry.get("key") or entry.get("name") + key = _slug_label(raw_key) + if not key: + continue + # Apply user merges + deletions first, THEN drop anything + # that ended up on an already-existing top-level metric + # slug. Empty resolved key = tombstoned. + key = _resolve_alias(aliases, key) + if not key or key in promoted_slugs: + continue + name = (entry.get("name") or "").strip() or key.replace( + "_", " " + ) + description = (entry.get("description") or "").strip() or None + sample = (entry.get("rationale") or "").strip() or None + raw_type = str(entry.get("suggested_type") or "").strip().lower() + if raw_type not in _DISCOVERED_METRIC_TYPES: + raw_type = "boolean" + + existing = by_key.get(key) + if existing is None: + examples = [sample] if sample else [] + by_key[key] = { + "key": key, + "name": name, + "description": description, + "suggested_type": raw_type, + "sample_rationale": sample, + "examples": examples, + "count": 1, + } + continue + + existing["count"] += 1 + if not existing["description"] and description: + existing["description"] = description + if not existing["sample_rationale"] and sample: + existing["sample_rationale"] = sample + # Keep the most-frequently-suggested type. We don't track + # per-type frequency yet; defer to the first non-default + # type encountered when the existing entry has the default. + if existing.get("suggested_type") == "boolean" and raw_type != "boolean": + existing["suggested_type"] = raw_type + if sample: + ex_list: List[str] = existing.setdefault("examples", []) + if len(ex_list) < 3 and not any( + s.strip().lower() == sample.strip().lower() for s in ex_list + ): + ex_list.append(sample) + + return sorted( + by_key.values(), + key=lambda item: (-item["count"], item["key"]), + ) + + +def _build_flow_graph( + eval_rows: List[CallImportEvaluationRow], + parent_metric: Metric, + children: List[Metric], + alias_map: Optional[Dict[str, str]] = None, + extra_children: Optional[List[Metric]] = None, +) -> MetricFlowResponse: + """Walk per-row ``sequence`` arrays and produce aggregate nodes/edges. + + A synthetic ``START`` node is prepended to every sequence so the + diagram has a single origin. Children that never appear in any + sequence are still emitted as nodes (count=0) so the UI can render + them in the legend. + + ``alias_map`` lets callers fold merged-out discovered slugs into + their canonical target before building the graph; ``extra_children`` + are children of the parent that aren't in the legend list (e.g. + children promoted *after* the evaluation was created and therefore + missing from ``selected_metric_groups``) but should still resolve in + sequences so the slug doesn't get redrawn as a discovered candidate. + """ + parent_id_str = str(parent_metric.id) + aliases = alias_map or {} + # Build a fast lookup keyed by both the lower_snake child key (what the + # LLM emits in ``sequence``) and the child UUID (what some clients may + # store) so legacy / drifted payloads still resolve. + child_lookup: Dict[str, Metric] = {} + for child in children: + slug = _slug_label(child.name) + child_lookup[slug] = child + child_lookup[str(child.id)] = child + # ``extra_children`` are resolved-only — they shouldn't add legend + # nodes (those come from the explicit ``children`` argument), but + # they need to be in ``child_lookup`` so a sequence step that + # matches a freshly-promoted child resolves to the real child UUID + # instead of falling through to ``discovered_lookup`` and rendering + # as a "discovered" node. + if extra_children: + for child in extra_children: + slug = _slug_label(child.name) + if slug and slug not in child_lookup: + child_lookup[slug] = child + cid = str(child.id) + child_lookup.setdefault(cid, child) + + # Discovered labels: walk every row's discovered_labels first so we + # know which discovered slugs are valid before resolving sequences. + # Discovered nodes get a ``disc:`` prefixed id so they can't collide + # with real child UUIDs in the node/edge graph. We apply + # ``alias_map`` first so merged-out source slugs fold into their + # canonical target — preserving the user's "merge" intent on still- + # in-flight rows whose JSON wasn't rewritten by the merge endpoint. + discovered_lookup: Dict[str, Dict[str, Any]] = {} + for row in eval_rows: + scores = ( + row.metric_scores if isinstance(row.metric_scores, dict) else {} + ) + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + continue + raw_discovered = parent_entry.get("discovered_labels") + if not isinstance(raw_discovered, list): + continue + for entry in raw_discovered: + if not isinstance(entry, dict): + continue + slug = _slug_label(entry.get("key") or entry.get("name")) + slug = _resolve_alias(aliases, slug) + if not slug or slug in child_lookup: + continue + name = (entry.get("name") or "").strip() or slug.replace("_", " ") + existing = discovered_lookup.get(slug) + if existing is None: + discovered_lookup[slug] = { + "id": f"{_DISCOVERED_NODE_PREFIX}{slug}", + "name": name, + } + + node_counts: Dict[str, int] = {} + edge_counts: Dict[tuple[str, str], int] = {} + terminal_counts: Dict[str, int] = {} + + total_rows = len(eval_rows) + rows_with_sequence = 0 + + for row in eval_rows: + scores = ( + row.metric_scores if isinstance(row.metric_scores, dict) else {} + ) + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + continue + raw_sequence = parent_entry.get("sequence") + if not isinstance(raw_sequence, list): + continue + + resolved_ids: List[str] = [] + last_resolved: Optional[str] = None + for item in raw_sequence: + if not isinstance(item, str): + continue + normalized = _resolve_alias(aliases, _slug_label(item)) + child = child_lookup.get(normalized) or child_lookup.get(item) + if child is not None: + cid = str(child.id) + # Adjacent dedupe AFTER alias resolution so two + # different raw slugs that fold to the same target + # don't draw a self-edge through the chart. + if cid == last_resolved: + continue + resolved_ids.append(cid) + last_resolved = cid + continue + disc = discovered_lookup.get(normalized) + if disc is not None: + if disc["id"] == last_resolved: + continue + resolved_ids.append(disc["id"]) + last_resolved = disc["id"] + + if not resolved_ids: + continue + + rows_with_sequence += 1 + for nid in resolved_ids: + node_counts[nid] = node_counts.get(nid, 0) + 1 + + edge_counts[(_FLOW_START_NODE_ID, resolved_ids[0])] = ( + edge_counts.get((_FLOW_START_NODE_ID, resolved_ids[0]), 0) + 1 + ) + for src, tgt in zip(resolved_ids, resolved_ids[1:]): + if src == tgt: + continue + edge_counts[(src, tgt)] = edge_counts.get((src, tgt), 0) + 1 + + terminal_id = resolved_ids[-1] + terminal_counts[terminal_id] = terminal_counts.get(terminal_id, 0) + 1 + + nodes: List[MetricFlowNode] = [] + # Always include a START node so the UI has a stable entry point. + nodes.append( + MetricFlowNode( + id=_FLOW_START_NODE_ID, + label="Start", + count=rows_with_sequence, + is_terminal=False, + ) + ) + + def _emit_child_node(child: Metric) -> None: + cid = str(child.id) + count = node_counts.get(cid, 0) + terminal_count = terminal_counts.get(cid, 0) + is_terminal = False + if rows_with_sequence > 0: + is_terminal = ( + terminal_count / rows_with_sequence + ) >= _FLOW_TERMINAL_THRESHOLD + nodes.append( + MetricFlowNode( + id=cid, + label=child.name, + count=count, + is_terminal=is_terminal, + ) + ) + + emitted_child_ids: set[str] = set() + for child in children: + cid = str(child.id) + if cid in emitted_child_ids: + continue + emitted_child_ids.add(cid) + _emit_child_node(child) + # Extra children (promoted after the eval was created) only get + # legend nodes if they actually appear in the data — otherwise we'd + # pollute the diagram with every standalone promotion the user has + # ever made under this parent. + if extra_children: + for child in extra_children: + cid = str(child.id) + if cid in emitted_child_ids: + continue + if node_counts.get(cid, 0) == 0: + continue + emitted_child_ids.add(cid) + _emit_child_node(child) + # Append discovered nodes after the real children so legend ordering + # keeps user-defined labels first. + for slug, info in discovered_lookup.items(): + nid = info["id"] + count = node_counts.get(nid, 0) + terminal_count = terminal_counts.get(nid, 0) + is_terminal = False + if rows_with_sequence > 0: + is_terminal = ( + terminal_count / rows_with_sequence + ) >= _FLOW_TERMINAL_THRESHOLD + nodes.append( + MetricFlowNode( + id=nid, + label=info["name"], + count=count, + is_terminal=is_terminal, + is_discovered=True, + ) + ) + + edges: List[MetricFlowEdge] = [ + MetricFlowEdge(source=src, target=tgt, count=count) + for (src, tgt), count in sorted( + edge_counts.items(), key=lambda kv: kv[1], reverse=True + ) + ] + + return MetricFlowResponse( + parent_metric_id=parent_id_str, + parent_metric_name=parent_metric.name, + selection_mode=parent_metric.selection_mode, + nodes=nodes, + edges=edges, + total_rows=total_rows, + rows_with_sequence=rows_with_sequence, + ) + + +@router.get( + "/{eval_id}/flow", + response_model=MetricFlowResponse, + operation_id="getCallImportEvaluationFlow", +) +async def get_call_import_evaluation_flow( + call_import_id: UUID, + eval_id: UUID, + parent_metric_id: UUID = Query( + ..., + description=( + "Parent (category) metric whose children's sequences should be " + "aggregated into a flow graph." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> MetricFlowResponse: + """Aggregate the LLM-inferred per-row sequences into one flow graph. + + Returns ``nodes`` (one per child of the parent metric, plus a + synthetic ``START`` node) and ``edges`` (counts of consecutive + label transitions across every row that produced a sequence). The + frontend feeds this directly into a React Flow / xyflow canvas; + edge thickness should scale with ``count / total_rows`` and + ``is_terminal`` nodes should be styled as outcomes. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + if not parent.selection_mode: + raise HTTPException( + status_code=400, + detail=( + "Flow charts are only meaningful for parent metrics " + "(selection_mode set). This metric is standalone." + ), + ) + + # Children are taken from selected_metric_groups when present so the + # flow chart reflects exactly the subset that ran in this + # evaluation; otherwise fall back to every enabled child of the + # parent. + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + parent_id_str = str(parent.id) + children: List[Metric] = [] + if parent_id_str in groups_raw and isinstance( + groups_raw[parent_id_str], list + ): + child_ids: List[UUID] = [] + for c in groups_raw[parent_id_str]: + try: + child_ids.append(UUID(str(c))) + except (TypeError, ValueError): + continue + if child_ids: + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.id.in_(child_ids), + ) + .order_by(Metric.created_at.asc()) + .all() + ) + if not children: + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id == parent.id, + ) + .order_by(Metric.created_at.asc()) + .all() + ) + + # Children promoted AFTER this evaluation was created aren't in + # ``selected_metric_groups`` but their slugs still appear in already- + # scored rows' sequences. Pass them as ``extra_children`` so those + # sequence entries resolve against the real (now promoted) child + # instead of being redrawn as discovered candidates. + extra_children: List[Metric] = [] + if children: + existing_ids = {child.id for child in children} + all_children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id == parent.id, + ) + .all() + ) + extra_children = [c for c in all_children if c.id not in existing_ids] + + eval_rows = _load_eval_rows(db, eval_id) + + alias_map = _alias_map_for_parent(evaluation, parent.id) + return _build_flow_graph( + eval_rows, + parent, + children, + alias_map=alias_map, + extra_children=extra_children, + ) + + +@router.get( + "/{eval_id}/discovered-labels", + response_model=DiscoveredLabelsResponse, + operation_id="getCallImportEvaluationDiscoveredLabels", +) +async def get_call_import_evaluation_discovered_labels( + call_import_id: UUID, + eval_id: UUID, + parent_metric_id: UUID = Query( + ..., + description=( + "Parent (category) metric whose LLM-discovered candidate " + "sub-labels should be aggregated across rows." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> DiscoveredLabelsResponse: + """Aggregate candidate sub-labels the LLM discovered during this eval. + + Only meaningful for parents with ``allow_discovery=true``; for other + parents we just return an empty ``items`` list rather than 400-ing + so the frontend can call the endpoint unconditionally for every + parent on the Flow tab without branching. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + + alias_map = _alias_map_for_parent(evaluation, parent_metric_id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + parent_metric_id, + organization_id=organization_id, + alias_map=alias_map, + ) + items = [DiscoveredLabelItem(**item) for item in items_raw] + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), items=items + ) + + +@router.post( + "/{eval_id}/discovered-labels/merge", + response_model=DiscoveredLabelsResponse, + operation_id="mergeCallImportEvaluationDiscoveredLabels", +) +async def merge_call_import_evaluation_discovered_labels( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredLabelMergeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredLabelsResponse: + """Rewrite every row's ``discovered_labels`` entry from from_key -> to_key. + + Idempotent — re-merging the same pair is a no-op. Discovered slugs + inside per-row ``sequence`` arrays are also rewritten so the flow + chart stays consistent with the panel. When a row already has + ``to_key`` and we're merging ``from_key`` into it, we drop the + ``from_key`` entry instead of producing two entries with the same + slug. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == body.parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + + from_key = _slug_label(body.from_key) + to_key = _slug_label(body.to_key) + if not from_key or not to_key: + raise HTTPException( + status_code=400, + detail="from_key and to_key must be non-empty slugs.", + ) + if from_key == to_key: + # No-op; just return the current aggregate so the client can + # refresh its view. + alias_map_existing = _alias_map_for_parent(evaluation, parent.id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + body.parent_metric_id, + organization_id=organization_id, + alias_map=alias_map_existing, + ) + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), + items=[DiscoveredLabelItem(**item) for item in items_raw], + ) + + parent_id_str = str(parent.id) + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _merge_discovered_label_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + return False + + mutated = False + discovered = parent_entry.get("discovered_labels") + if isinstance(discovered, list): + kept: List[Dict[str, Any]] = [] + existing_to = next( + ( + e + for e in discovered + if isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) == to_key + ), + None, + ) + for entry in discovered: + if not isinstance(entry, dict): + kept.append(entry) + continue + key = _slug_label(entry.get("key") or entry.get("name")) + if key == from_key: + if existing_to is not None: + mutated = True + continue + new_entry = dict(entry) + new_entry["key"] = to_key + kept.append(new_entry) + mutated = True + else: + kept.append(entry) + if mutated: + parent_entry["discovered_labels"] = kept + + seq = parent_entry.get("sequence") + if isinstance(seq, list): + new_seq: List[str] = [] + seq_changed = False + last_added: Optional[str] = None + for item in seq: + if isinstance(item, str) and _slug_label(item) == from_key: + seq_changed = True + if last_added == to_key: + continue + new_seq.append(to_key) + last_added = to_key + else: + new_seq.append(item) + last_added = ( + _slug_label(item) if isinstance(item, str) else None + ) + if seq_changed: + parent_entry["sequence"] = new_seq + mutated = True + + if mutated: + row.metric_scores = dict(scores) + return mutated + + foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_label_row) + + # Persist the merge at the evaluation level too. This is what makes + # the merge survive future scoring: rows that finish AFTER this + # call (e.g. retries, in-flight workers) will go through the + # alias map in the API surface even if the per-row JSON they + # write still mentions ``from_key``. We chain through any existing + # alias so merging A→B and then B→C resolves A→C in the panel. + raw_aliases = ( + evaluation.discovered_label_aliases + if isinstance(evaluation.discovered_label_aliases, dict) + else {} + ) + aliases_top = dict(raw_aliases) + parent_aliases = dict(aliases_top.get(parent_id_str) or {}) + # Resolve transitively: if to_key itself was previously merged into + # something else, point from_key at the canonical end-of-chain. + canonical_to = _resolve_alias(parent_aliases, to_key) + parent_aliases[from_key] = canonical_to + # Re-target any earlier aliases that pointed AT from_key — without + # this, A→B and then B→C would leave A still pointing to B (now a + # broken pointer because B is gone). Rewriting them keeps the + # alias map self-consistent. + for k, v in list(parent_aliases.items()): + if v == from_key: + parent_aliases[k] = canonical_to + aliases_top[parent_id_str] = parent_aliases + evaluation.discovered_label_aliases = aliases_top + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + alias_map_after = _alias_map_for_parent(evaluation, parent.id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + body.parent_metric_id, + organization_id=organization_id, + alias_map=alias_map_after, + ) + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), + items=[DiscoveredLabelItem(**item) for item in items_raw], + ) + + +@router.post( + "/{eval_id}/discovered-labels/delete", + response_model=DiscoveredLabelsResponse, + operation_id="deleteCallImportEvaluationDiscoveredLabel", +) +async def delete_call_import_evaluation_discovered_label( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredLabelDeleteRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredLabelsResponse: + """Tombstone a single LLM-discovered candidate for this evaluation. + + Symmetric with the merge endpoint, but instead of redirecting the + slug at another candidate we mark it as deleted. After this call: + + * the slug is stripped from every row's + ``metric_scores[parent].discovered_labels`` list, and from + every row's ``sequence`` array (so the flow chart no longer + draws a node for it); + * the slug is recorded in + ``evaluation.discovered_label_aliases[parent][slug] = ""`` + so any worker that finishes a row AFTER this call (e.g. a row + still in flight when the user clicked Delete) silently drops + the slug instead of resurrecting it. + + Idempotent: deleting an already-deleted slug is a no-op. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == body.parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + + target_key = _slug_label(body.key) + if not target_key: + raise HTTPException( + status_code=400, + detail="key must be a non-empty slug.", + ) + + parent_id_str = str(parent.id) + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _delete_discovered_label_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + return False + + mutated = False + discovered = parent_entry.get("discovered_labels") + if isinstance(discovered, list): + kept = [ + e + for e in discovered + if not ( + isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) + == target_key + ) + ] + if len(kept) != len(discovered): + parent_entry["discovered_labels"] = kept + mutated = True + + seq = parent_entry.get("sequence") + if isinstance(seq, list): + new_seq: List[str] = [] + seq_changed = False + last_added: Optional[str] = None + for item in seq: + if isinstance(item, str) and _slug_label(item) == target_key: + seq_changed = True + continue + if isinstance(item, str): + norm = _slug_label(item) + if norm == last_added: + seq_changed = True + continue + last_added = norm + new_seq.append(item) + if seq_changed: + parent_entry["sequence"] = new_seq + mutated = True + + if mutated: + row.metric_scores = dict(scores) + return mutated + + foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_label_row) + + # 3. Persist the tombstone on the evaluation so workers that finish + # later don't re-surface the deleted slug. We also retarget any + # existing aliases whose ``to_key`` was the deleted slug — without + # this, a previous merge that pointed at this slug would leave a + # dangling pointer. + raw_aliases = ( + evaluation.discovered_label_aliases + if isinstance(evaluation.discovered_label_aliases, dict) + else {} + ) + aliases_top = dict(raw_aliases) + parent_aliases = dict(aliases_top.get(parent_id_str) or {}) + parent_aliases[target_key] = "" # deletion sentinel + for k, v in list(parent_aliases.items()): + if v == target_key: + parent_aliases[k] = "" + aliases_top[parent_id_str] = parent_aliases + evaluation.discovered_label_aliases = aliases_top + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + alias_map_after = _alias_map_for_parent(evaluation, parent.id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + body.parent_metric_id, + organization_id=organization_id, + alias_map=alias_map_after, + ) + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), + items=[DiscoveredLabelItem(**item) for item in items_raw], + ) + + +# --------------------------------------------------------------------------- +# Discovered TOP-LEVEL METRICS (per-evaluation discovery) +# +# These endpoints are the parallel of the discovered-labels trio above but +# scoped to the evaluation as a whole instead of to a parent metric. They +# all live under ``/{eval_id}/discovered-metrics`` and operate on the +# reserved ``DISCOVERED_METRICS_KEY`` slot of each per-row +# ``metric_scores`` plus the flat ``CallImportEvaluation.discovered_metric_aliases`` +# map (no parent-id nesting). +# --------------------------------------------------------------------------- + + +def _flat_metric_aliases( + evaluation: CallImportEvaluation, +) -> Dict[str, str]: + """Pull the flat ``{from_slug: to_slug}`` map for an evaluation.""" + raw = getattr(evaluation, "discovered_metric_aliases", None) + if not isinstance(raw, dict): + return {} + return { + str(k): str(v) + for k, v in raw.items() + if isinstance(k, str) and isinstance(v, str) + } + + +@router.get( + "/{eval_id}/discovered-metrics", + response_model=DiscoveredMetricsResponse, + operation_id="getCallImportEvaluationDiscoveredMetrics", +) +async def get_call_import_evaluation_discovered_metrics( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> DiscoveredMetricsResponse: + """Aggregate top-level metric candidates the LLM discovered during this eval. + + Returns an empty ``items`` list when the evaluation did not opt + into top-level metric discovery; this keeps the frontend able to + call the endpoint unconditionally without branching on the + evaluation's ``discover_new_metrics`` flag. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not bool(getattr(evaluation, "discover_new_metrics", False)): + return DiscoveredMetricsResponse(evaluation_id=evaluation.id, items=[]) + + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + +@router.post( + "/{eval_id}/discovered-metrics/merge", + response_model=DiscoveredMetricsResponse, + operation_id="mergeCallImportEvaluationDiscoveredMetrics", +) +async def merge_call_import_evaluation_discovered_metrics( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredMetricMergeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredMetricsResponse: + """Rewrite every row's ``__discovered_metrics__`` entry from→to. + + Mirrors the discovered-labels merge endpoint but operates on the + flat top-level metric list. Idempotent — re-merging is a no-op. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + from_key = _slug_label(body.from_key) + to_key = _slug_label(body.to_key) + if not from_key or not to_key: + raise HTTPException( + status_code=400, + detail="from_key and to_key must be non-empty slugs.", + ) + if from_key == to_key: + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _merge_discovered_metric_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + discovered = scores.get(DISCOVERED_METRICS_KEY) + if not isinstance(discovered, list): + return False + + kept: List[Dict[str, Any]] = [] + mutated = False + existing_to = next( + ( + e + for e in discovered + if isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) == to_key + ), + None, + ) + for entry in discovered: + if not isinstance(entry, dict): + kept.append(entry) + continue + key = _slug_label(entry.get("key") or entry.get("name")) + if key == from_key: + if existing_to is not None: + mutated = True + continue + new_entry = dict(entry) + new_entry["key"] = to_key + kept.append(new_entry) + mutated = True + else: + kept.append(entry) + if mutated: + scores[DISCOVERED_METRICS_KEY] = kept + row.metric_scores = dict(scores) + return mutated + + foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_metric_row) + + raw_aliases = ( + evaluation.discovered_metric_aliases + if isinstance(evaluation.discovered_metric_aliases, dict) + else {} + ) + aliases = dict(raw_aliases) + canonical_to = _resolve_alias(aliases, to_key) + aliases[from_key] = canonical_to + for k, v in list(aliases.items()): + if v == from_key: + aliases[k] = canonical_to + evaluation.discovered_metric_aliases = aliases + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + +@router.post( + "/{eval_id}/discovered-metrics/delete", + response_model=DiscoveredMetricsResponse, + operation_id="deleteCallImportEvaluationDiscoveredMetric", +) +async def delete_call_import_evaluation_discovered_metric( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredMetricDeleteRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredMetricsResponse: + """Tombstone a single LLM-discovered top-level metric candidate.""" + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + target_key = _slug_label(body.key) + if not target_key: + raise HTTPException( + status_code=400, + detail="key must be a non-empty slug.", + ) + + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _delete_discovered_metric_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + discovered = scores.get(DISCOVERED_METRICS_KEY) + if not isinstance(discovered, list): + return False + kept = [ + e + for e in discovered + if not ( + isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) + == target_key + ) + ] + if len(kept) == len(discovered): + return False + if kept: + scores[DISCOVERED_METRICS_KEY] = kept + else: + scores.pop(DISCOVERED_METRICS_KEY, None) + row.metric_scores = dict(scores) + return True + + foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_metric_row) + + raw_aliases = ( + evaluation.discovered_metric_aliases + if isinstance(evaluation.discovered_metric_aliases, dict) + else {} + ) + aliases = dict(raw_aliases) + aliases[target_key] = "" # tombstone + for k, v in list(aliases.items()): + if v == target_key: + aliases[k] = "" + evaluation.discovered_metric_aliases = aliases + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + +@router.delete( + "/{eval_id}/rows/{eval_row_id}", + status_code=status.HTTP_204_NO_CONTENT, + operation_id="deleteCallImportEvaluationRow", +) +async def delete_call_import_evaluation_row( + call_import_id: UUID, + eval_id: UUID, + eval_row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Response: + """Delete a single per-row scoring entry within an evaluation run. + + Useful when the user wants to drop a noisy row before re-exporting + the CSV — e.g. a row whose audio was corrupt and skewed the + aggregate. Counters on the parent are recomputed so the rolled-up + status stays accurate. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + from app.db_sharding.eval_rows import delete_evaluation_row_on_shards + + if not delete_evaluation_row_on_shards(eval_row_id, eval_id): + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + + eval_row = ( + db.query(CallImportEvaluationRow) + .filter( + CallImportEvaluationRow.id == eval_row_id, + CallImportEvaluationRow.evaluation_id == eval_id, + ) + .first() + ) + if not eval_row: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + + # If the row was still in flight, best-effort revoke the worker task + # so it doesn't try to write into a deleted DB row mid-execution. + if eval_row.celery_task_id and eval_row.status in {"pending", "running"}: + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(eval_row.celery_task_id, terminate=False) + except Exception: + pass + + db.delete(eval_row) + db.flush() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +# --------------------------------------------------------------------------- +# Retry endpoints +# --------------------------------------------------------------------------- +# +# The create endpoint enqueues every row of a fresh run; these endpoints +# let the user re-enqueue a *subset* of rows in an existing run — most +# commonly the ones that failed. We keep the worker contract identical +# (``evaluate_call_import_row_task(eval_row_id)``), so the retry path +# only has to reset row state and re-fan-out. When a row is missing its +# diarised transcript and the run was configured for diarised +# transcripts, we chain through ``transcribe_call_import_row_task`` the +# same way the create endpoint does — that's what makes "retry" feel +# like "just fix it" instead of "fail again immediately". + + +def _prepare_source_row_for_retry( + source_row: CallImportRow, + *, + transcribe_overwrite: bool, +) -> None: + """Clear stale diarisation markers so retry dispatch can re-run the pipeline.""" + source_row.celery_task_id = None + + # Re-fetch recordings when a prior import failed or stalled without S3 audio. + # Mirrors retry_failed_call_import_rows so eval retry can re-enqueue imports. + if ( + source_row.status + in (CallImportRowStatus.FAILED, CallImportRowStatus.PROCESSING) + and not (source_row.recording_s3_key or "").strip() + ): + source_row.status = CallImportRowStatus.PENDING + source_row.error_message = None + + if transcribe_overwrite and (source_row.diarised_transcript or "").strip(): + source_row.diarised_transcript = None + + has_dia = bool((source_row.diarised_transcript or "").strip()) + dia_status = (source_row.diarised_transcript_status or "").strip().lower() + + if has_dia and not transcribe_overwrite: + source_row.diarised_transcript_status = "completed" + source_row.diarised_transcript_error = None + return + + if dia_status in {"failed", "pending", "running", "idle"}: + source_row.diarised_transcript_status = "idle" + source_row.diarised_transcript_error = None + + +def _reset_eval_row_for_retry( + eval_row: CallImportEvaluationRow, + *, + metric_ids: Optional[List[UUID]] = None, + skip_revoke: bool = False, +) -> None: + """Wipe per-row state so the worker can re-run it cleanly. + + Mirrors the initial state used by ``create_call_import_evaluation`` + when it first inserts a row, with the addition of revoking any + lingering Celery task id. + + When ``metric_ids`` is provided, this is a **metric-subset retry**: + only the scores for those metrics are removed from + ``metric_scores`` (other metrics' previously-computed values are + preserved so the worker's partial-merge write keeps them intact). + Otherwise the entire ``metric_scores`` dict is reset, matching the + legacy behaviour. + """ + if ( + not skip_revoke + and eval_row.celery_task_id + and eval_row.status in {"pending", "running"} + ): + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(eval_row.celery_task_id, terminate=False) + except Exception: # noqa: BLE001 — revoke is best-effort + pass + eval_row.status = "pending" + eval_row.error_message = None + if metric_ids: + # Strip ONLY the targeted metric keys. Both string and UUID + # forms can appear in ``metric_scores`` depending on which + # code path wrote the dict, so we normalise to lower-case + # strings for the comparison. + existing = ( + eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} + ) + target_keys = {str(mid).lower() for mid in metric_ids} + eval_row.metric_scores = { + key: value + for key, value in existing.items() + if str(key).lower() not in target_keys + } + else: + eval_row.metric_scores = {} + eval_row.started_at = None + eval_row.finished_at = None + eval_row.celery_task_id = None + + +def _enqueue_eval_rows_with_optional_transcribe( + db: Session, + evaluation: CallImportEvaluation, + eval_rows_with_source: List[ + Tuple[CallImportEvaluationRow, CallImportRow] + ], + *, + transcribe_overwrite: bool = False, + restricted_metric_ids: Optional[List[UUID]] = None, +) -> Tuple[int, int]: + """Schedule throttled evaluation dispatch for pending eval rows. + + Returns ``(evaluate_only_count, transcribe_then_evaluate_count)`` for + logging/UI compatibility. Actual Celery fan-out is handled by + :func:`dispatch_evaluation_rows_task` under Redis fair-share limits. + """ + from app.workers.concurrency.eval_dispatch import _needs_transcribe_for_eval + from app.workers.concurrency.fair_dispatch import ( + schedule_fair_dispatch, + store_evaluation_transcribe_overwrite, + store_row_restricted_metrics, + ) + + eval_only_count = 0 + transcribe_count = 0 + if eval_rows_with_source: + for eval_row, source_row in eval_rows_with_source: + if _needs_transcribe_for_eval( + evaluation, + source_row, + transcribe_overwrite=transcribe_overwrite, + ): + transcribe_count += 1 + else: + eval_only_count += 1 + + restricted_metric_ids_str: Optional[List[str]] = ( + [str(mid) for mid in restricted_metric_ids] + if restricted_metric_ids + else None + ) + if restricted_metric_ids_str: + for eval_row, _ in eval_rows_with_source: + store_row_restricted_metrics(eval_row.id, restricted_metric_ids_str) + else: + restricted_metric_ids_str = ( + [str(mid) for mid in restricted_metric_ids] if restricted_metric_ids else None + ) + store_evaluation_transcribe_overwrite( + evaluation.id, + overwrite=transcribe_overwrite, + ) + schedule_fair_dispatch(max_workspace_turns=999) + return eval_only_count, transcribe_count + + +def _apply_telephony_retry_overrides( + db: Session, + *, + call_import: CallImport, + organization_id: UUID, + payload: CallImportEvaluationRetryRequest, +) -> None: + """Pin or clear telephony credentials on the batch for this retry pass.""" + fields_set = payload.model_fields_set + if ( + "provider" not in fields_set + and "telephony_integration_id" not in fields_set + ): + return + + from app.api.v1.routes.call_imports import _resolve_telephony_integration + + if payload.telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + else: + call_import.provider = None + call_import.telephony_integration_id = None + db.flush() + + +def _apply_retry_overrides( + db: Session, + evaluation: CallImportEvaluation, + organization_id: UUID, + payload: CallImportEvaluationRetryRequest, +) -> None: + """Validate + persist the LLM/STT override fields on the run. + + Mirrors the validation in ``create_call_import_evaluation`` but + only touches the fields the caller actually sent — leaving any + field ``None`` preserves the run's existing value. Raises + ``HTTPException(400)`` on bad input so the route handler can let + FastAPI turn it into a clean 400 response. + """ + # --- LLM provider + model (must be sent together) --- + if payload.llm_provider is not None or payload.llm_model is not None: + if not (payload.llm_provider and payload.llm_model): + raise HTTPException( + status_code=400, + detail=( + "Both llm_provider and llm_model are required when " + "overriding the run LLM on retry." + ), + ) + try: + evaluation.llm_provider = ModelProvider( + payload.llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Unknown LLM provider '{payload.llm_provider}'. " + "Valid keys are documented in ModelProvider." + ), + ) + new_model = payload.llm_model.strip() or None + if not new_model: + raise HTTPException( + status_code=400, detail="llm_model cannot be empty." + ) + evaluation.llm_model = new_model + + # --- LLM credential pin --- + if payload.llm_credential_id is not None: + cred = ( + db.query(AIProvider) + .filter( + AIProvider.id == payload.llm_credential_id, + AIProvider.organization_id == organization_id, + ) + .first() + ) + if not cred: + raise HTTPException( + status_code=400, + detail=( + "The provided llm_credential_id does not exist in " + "this organization." + ), + ) + evaluation.llm_credential_id = payload.llm_credential_id + + if payload.llm_config is not None: + evaluation.llm_config = payload.llm_config + + # --- Per-metric LLM overrides --- + # We accept the same dict shape as the create endpoint but + # constrain keys to leaf metrics that are actually in this run. + # Passing an empty dict explicitly clears existing overrides. + if payload.metric_llm_overrides is not None: + valid_leaf_ids = { + str(mid) for mid in (evaluation.selected_metric_ids or []) + } + overrides_payload: Dict[str, Dict[str, Any]] = {} + for metric_id, override in payload.metric_llm_overrides.items(): + if metric_id not in valid_leaf_ids: + raise HTTPException( + status_code=400, + detail=( + "metric_llm_overrides references metric " + f"{metric_id} which is not a leaf metric in " + "this run." + ), + ) + override_dict: Dict[str, Any] = {} + if override.provider is not None: + if not override.model: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a " + "provider but no model." + ), + ) + try: + override_dict["provider"] = ModelProvider( + override.provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} uses " + f"unknown provider '{override.provider}'." + ), + ) + override_dict["model"] = override.model.strip() + elif override.model: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a model " + "but no provider." + ), + ) + if override.credential_id is not None: + override_dict["credential_id"] = str(override.credential_id) + if override.llm_config is not None: + override_dict["llm_config"] = override.llm_config + if override_dict: + overrides_payload[metric_id] = override_dict + evaluation.metric_llm_overrides = overrides_payload or None + + # --- STT provider + model (must be sent together) --- + if payload.stt_provider is not None or payload.stt_model is not None: + if not (payload.stt_provider and payload.stt_model): + raise HTTPException( + status_code=400, + detail=( + "Both stt_provider and stt_model are required " + "when overriding the run STT on retry." + ), + ) + try: + evaluation.stt_provider = ModelProvider( + payload.stt_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unknown STT provider '{payload.stt_provider}'.", + ) + new_stt_model = payload.stt_model.strip() or None + if not new_stt_model: + raise HTTPException( + status_code=400, detail="stt_model cannot be empty." + ) + evaluation.stt_model = new_stt_model + + # --- STT credential pin --- + if payload.stt_credential_id is not None: + evaluation.stt_credential_id = payload.stt_credential_id + + # --- LLM diariser provider + model (must be sent together) --- + if ( + payload.diarization_llm_provider is not None + or payload.diarization_llm_model is not None + ): + if not ( + payload.diarization_llm_provider + and payload.diarization_llm_model + ): + raise HTTPException( + status_code=400, + detail=( + "Both diarization_llm_provider and " + "diarization_llm_model are required when overriding " + "the run diariser on retry." + ), + ) + try: + evaluation.diarisation_llm_provider = ModelProvider( + payload.diarization_llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + "Unknown diarisation LLM provider " + f"'{payload.diarization_llm_provider}'." + ), + ) + new_diariser_model = ( + payload.diarization_llm_model.strip() or None + ) + if not new_diariser_model: + raise HTTPException( + status_code=400, + detail="diarization_llm_model cannot be empty.", + ) + evaluation.diarisation_llm_model = new_diariser_model + + if payload.diarization_llm_credential_id is not None: + evaluation.diarisation_llm_credential_id = ( + payload.diarization_llm_credential_id + ) + + # ``diarization_prompt`` semantics: None = leave untouched; + # empty string = clear (fall back to the canonical default at + # worker time); anything else = persist verbatim. + if payload.diarization_prompt is not None: + cleaned = payload.diarization_prompt.strip() + evaluation.diarisation_prompt = cleaned or None + + if payload.transcribe_mode is not None: + mode = payload.transcribe_mode.strip().lower() + if mode not in {"stt_llm", "llm_only"}: + raise HTTPException( + status_code=400, + detail=( + f"Unknown transcribe_mode '{payload.transcribe_mode}'. " + "Valid values are 'stt_llm' and 'llm_only'." + ), + ) + evaluation.transcribe_mode = mode + + +def _gather_retry_targets( + db: Session, + evaluation: CallImportEvaluation, + requested_ids: Optional[List[UUID]], + *, + include_completed: bool = False, +) -> Tuple[ + List[Tuple[CallImportEvaluationRow, CallImportRow]], + List[CallImportEvaluationRetrySkippedItem], +]: + """Resolve which rows to retry + reasons for any we refuse. + + When ``requested_ids`` is None we retry every row whose status is + ``failed`` (or every row when ``include_completed`` is also set — + used by the metric-subset retry path which legitimately wants to + recompute a metric on already-successful rows). When the caller + passes ids explicitly we still filter out rows that are currently + in flight; ``include_completed`` controls whether previously- + successful rows are eligible. + """ + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + from app.db_sharding.eval_rows import gather_retry_targets_sharded + + return gather_retry_targets_sharded( + db, + evaluation, + requested_ids, + include_completed=include_completed, + ) + + eval_rows_query = db.query(CallImportEvaluationRow).filter( + CallImportEvaluationRow.evaluation_id == evaluation.id + ) + + targets: List[Tuple[CallImportEvaluationRow, CallImportRow]] = [] + skipped: List[CallImportEvaluationRetrySkippedItem] = [] + + if requested_ids is None: + if include_completed: + # "Retry everything" path used by the metric-subset re-run + # UI. Still skip in-flight rows below so we don't trample + # work the worker is actively doing. + candidate_rows = eval_rows_query.filter( + CallImportEvaluationRow.status.in_(["failed", "completed"]) + ).all() + else: + candidate_rows = eval_rows_query.filter( + CallImportEvaluationRow.status == "failed" + ).all() + else: + requested_set = set(requested_ids) + candidate_rows = eval_rows_query.filter( + CallImportEvaluationRow.id.in_(requested_set) + ).all() + found_ids = {row.id for row in candidate_rows} + for missing in requested_set - found_ids: + skipped.append( + CallImportEvaluationRetrySkippedItem( + eval_row_id=missing, + reason="unknown", + ) + ) + + if not candidate_rows: + return targets, skipped + + source_row_ids = [row.call_import_row_id for row in candidate_rows] + source_rows = ( + db.query(CallImportRow) + .filter(CallImportRow.id.in_(source_row_ids)) + .all() + ) + source_by_id = {row.id: row for row in source_rows} + + for eval_row in candidate_rows: + if eval_row.status in {"pending", "running"}: + skipped.append( + CallImportEvaluationRetrySkippedItem( + eval_row_id=eval_row.id, + reason="in_progress", + ) + ) + continue + if eval_row.status == "completed" and not include_completed: + skipped.append( + CallImportEvaluationRetrySkippedItem( + eval_row_id=eval_row.id, + reason="completed", + ) + ) + continue + source_row = source_by_id.get(eval_row.call_import_row_id) + if source_row is None: + skipped.append( + CallImportEvaluationRetrySkippedItem( + eval_row_id=eval_row.id, + reason="source_row_missing", + ) + ) + continue + targets.append((eval_row, source_row)) + + return targets, skipped + + +@router.post( + "/{eval_id}/retry", + response_model=CallImportEvaluationRetryResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="retryCallImportEvaluation", +) +async def retry_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + payload: Optional[CallImportEvaluationRetryRequest] = Body(default=None), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationRetryResponse: + """Re-enqueue failed rows in an evaluation run. + + Default behavior (no body) is "retry every row that failed". Pass + ``eval_row_ids`` to scope the retry to a specific subset (e.g. the + single row a user clicked in the UI). Rows that are still + in-flight or already completed are returned in ``skipped`` rather + than re-enqueued, so this endpoint is always safe to call. + + When ``metric_ids`` is set in the payload, this is a **metric- + subset retry**: only the listed metrics are recomputed (and merged + into the row's existing ``metric_scores`` — other metrics' values + are preserved). The route auto-flips ``include_completed=True`` in + that case so previously-successful rows are eligible for re- + scoring; without it the call would no-op because every row would + be skipped as ``completed``. + + The worker contract is the same as the create endpoint: + ``evaluate_call_import_row_task(eval_row_id, [restricted_metric_ids])``. + When the run is configured for diarised transcripts and the row's + diarised transcript is missing, we chain through + ``transcribe_call_import_row_task`` first — matching the + auto-transcribe behavior of POST ``/evaluations``. + """ + del api_key + call_import = _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + requested_ids = payload.eval_row_ids if payload else None + # Metric-subset retry: validate that every metric is something this + # run actually scored. Empty list is rejected too — callers that + # want a full re-run should omit the field entirely. + # + # ``selected_metric_ids`` holds the LEAVES only (children for + # hierarchical / category metrics, standalone metrics otherwise) — + # see ``leaf_metric_ids`` in :func:`create_call_import_evaluation`. + # Parent IDs for hierarchical metrics live separately in + # ``selected_metric_groups`` (``{parent_id: [child_ids]}``) so the + # UI can reconstruct the tree without round-tripping through the + # metric table. + # + # The Re-run-metrics modal surfaces PARENTS for hierarchical + # metrics (it suppresses individual children via + # ``childrenInGroups`` in ``CallImportEvaluationDetail.tsx``), so a + # naive ``metric_ids ⊆ selected_metric_ids`` check rejects every + # parent-ID request with a misleading "unknown ids" 400. We accept + # both shapes here and then EXPAND any parent IDs into + # ``{parent_id, *child_ids}`` so the downstream helpers see the + # full set of keys that need clearing + the full set of leaves + # that need re-scoring. + metric_ids: Optional[List[UUID]] = ( + payload.metric_ids if payload else None + ) + if metric_ids is not None: + if not metric_ids: + raise HTTPException( + status_code=400, + detail=( + "metric_ids must be a non-empty list. Omit the " + "field to re-run all metrics." + ), + ) + + leaf_set: Set[str] = { + str(item).lower() + for item in (evaluation.selected_metric_ids or []) + } + # ``selected_metric_groups`` is a dict ``{parent_id_str: + # [child_id_str, ...]}`` (see line ~487 in + # ``create_call_import_evaluation``). We tolerate stale data + # (string / UUID / non-dict) without crashing the retry path — + # if it's malformed we just treat it as "no parents" and fall + # back to the leaf-only check. + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + parent_to_children_str: Dict[str, List[str]] = {} + for parent_key, children_raw in groups_raw.items(): + if not isinstance(children_raw, (list, tuple)): + continue + children_norm = [ + str(c).lower() for c in children_raw if c is not None + ] + parent_to_children_str[str(parent_key).lower()] = children_norm + parent_set = set(parent_to_children_str.keys()) + + unknown = [ + mid for mid in metric_ids + if str(mid).lower() not in leaf_set + and str(mid).lower() not in parent_set + ] + if unknown: + raise HTTPException( + status_code=400, + detail=( + "metric_ids must be a subset of this evaluation's " + f"selected metrics; unknown ids: {[str(u) for u in unknown]}." + ), + ) + + # Expand parent IDs into ``{parent, *children}`` so: + # * ``_reset_eval_row_for_retry`` strips BOTH the parent + # entry (with ``chosen_child_id`` / rationale) AND every + # per-child boolean entry that the LLM evaluator wrote + # under each child's ID (see + # ``app/workers/tasks/helpers/llm_evaluation.py`` lines + # 1584 and 1649). + # * ``_enqueue_eval_rows_with_optional_transcribe`` → + # ``evaluate_call_import_row_task`` filters the work-list + # off ``selected_metric_ids`` (leaves), so we MUST hand it + # the child IDs for the parent to actually get re-scored. + # Leaves pass through unchanged. + expanded: List[UUID] = [] + seen: Set[str] = set() + for mid in metric_ids: + mid_norm = str(mid).lower() + children_str = parent_to_children_str.get(mid_norm) + if children_str is not None: + # Parent: include the parent ID itself (so the parent + # entry in ``metric_scores`` is also cleared) and all + # of its children. + candidates = [mid_norm, *children_str] + else: + candidates = [mid_norm] + for candidate in candidates: + if candidate in seen: + continue + try: + expanded.append(UUID(candidate)) + except (TypeError, ValueError): + # Defensive: skip junk values rather than 500. + continue + seen.add(candidate) + metric_ids = expanded + + # ``include_completed`` is auto-enabled when the caller asked for a + # metric subset (otherwise the metric-subset retry would always + # no-op on a green run, which is the whole reason this feature + # exists). The explicit payload flag wins for full-row retries. + include_completed = bool( + (payload.include_completed if payload else False) + or (metric_ids is not None) + ) + + transcribe_overwrite = bool( + payload.transcribe_overwrite if payload else False + ) + + skipped: List[CallImportEvaluationRetrySkippedItem] = [] + if requested_ids is None: + from app.db_sharding.eval_rows import count_evaluation_rows_for_run + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + statuses = ( + ["failed", "completed"] if include_completed else ["failed"] + ) + target_count = count_evaluation_rows_for_run( + db, eval_id, statuses=statuses + ) + else: + from sqlalchemy import func + + count_query = db.query(func.count(CallImportEvaluationRow.id)).filter( + CallImportEvaluationRow.evaluation_id == eval_id + ) + if include_completed: + count_query = count_query.filter( + CallImportEvaluationRow.status.in_(["failed", "completed"]) + ) + else: + count_query = count_query.filter( + CallImportEvaluationRow.status == "failed" + ) + target_count = int(count_query.scalar() or 0) + if target_count == 0: + return CallImportEvaluationRetryResponse( + requeued=0, + transcribe_requeued=0, + skipped=skipped, + ) + else: + targets, skipped = _gather_retry_targets( + db, + evaluation, + requested_ids, + include_completed=include_completed, + ) + if not targets: + return CallImportEvaluationRetryResponse( + requeued=0, + transcribe_requeued=0, + skipped=skipped, + ) + target_count = len(targets) + + # Apply LLM / STT overrides BEFORE enqueueing so the persisted run + # config is correct by the time the worker reads it. + if payload is not None: + _apply_retry_overrides(db, evaluation, organization_id, payload) + _apply_telephony_retry_overrides( + db, + call_import=call_import, + organization_id=organization_id, + payload=payload, + ) + + evaluation.error_message = None + evaluation.finished_at = None + evaluation.status = "running" + if not evaluation.started_at: + from datetime import datetime, timezone + + evaluation.started_at = datetime.now(timezone.utc) + + _claim_evaluation_bulk_operation(eval_id, "retry") + stamp_evaluation_actor(evaluation, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import ( + retry_call_import_evaluation_task, + ) + + retry_call_import_evaluation_task.delay( + str(eval_id), + { + "eval_row_ids": [str(rid) for rid in requested_ids] + if requested_ids + else None, + "metric_ids": [str(mid) for mid in metric_ids] if metric_ids else None, + "include_completed": include_completed, + "transcribe_overwrite": transcribe_overwrite, + }, + ) + + return CallImportEvaluationRetryResponse( + requeued=target_count, + transcribe_requeued=0, + skipped=skipped, + ) + + +@router.post( + "/{eval_id}/rows/{eval_row_id}/retry", + response_model=CallImportEvaluationRowResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="retryCallImportEvaluationRow", +) +async def retry_call_import_evaluation_row( + call_import_id: UUID, + eval_id: UUID, + eval_row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationRowResponse: + """Re-enqueue a single failed evaluation row. + + Convenience wrapper around ``retry_call_import_evaluation`` for the + "Retry this row" affordance in the row table. Returns the + refreshed row so the UI can update its badge immediately, without + waiting for the next polling tick. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + _require_no_evaluation_bulk_operation(eval_id) + + from app.db_sharding.eval_rows import ( + evaluation_row_session, + find_evaluation_row_in_run, + ) + from app.db_sharding.sessions import is_sharding_enabled + + eval_row, _source_stub = find_evaluation_row_in_run(db, eval_id, eval_row_id) + if eval_row is None: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + + if eval_row.status in {"pending", "running"}: + raise HTTPException( + status_code=409, + detail=( + "This row is still in progress — wait for it to finish " + "before retrying." + ), + ) + + targets, _ = _gather_retry_targets(db, evaluation, [eval_row.id]) + if not targets: + raise HTTPException( + status_code=409, + detail=( + "This row cannot be retried in its current state " + f"(status={eval_row.status})." + ), + ) + + if is_sharding_enabled(): + with evaluation_row_session(eval_row_id) as ( + row_db, + _catalog_db, + eval_row, + source_row, + _shard_id, + ): + _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) + _reset_eval_row_for_retry(eval_row) + row_db.commit() + targets = [(eval_row, source_row)] + else: + for er, source_row in targets: + _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) + _reset_eval_row_for_retry(er) + + evaluation.error_message = None + evaluation.finished_at = None + evaluation.status = "running" + if not evaluation.started_at: + from datetime import datetime, timezone + + evaluation.started_at = datetime.now(timezone.utc) + db.flush() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + + try: + _enqueue_eval_rows_with_optional_transcribe(db, evaluation, targets) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to re-enqueue retry for evaluation row {}", eval_row_id + ) + if is_sharding_enabled(): + with evaluation_row_session(eval_row_id) as ( + row_db, + _catalog_db, + eval_row, + _source_row, + _shard_id, + ): + eval_row.status = "failed" + eval_row.error_message = f"Failed to re-enqueue retry: {exc}" + row_db.commit() + else: + eval_row.status = "failed" + eval_row.error_message = f"Failed to re-enqueue retry: {exc}" + _rollup_evaluation_status(evaluation, db) + db.commit() + raise HTTPException( + status_code=500, + detail=f"Failed to re-enqueue retry: {exc}", + ) + + if is_sharding_enabled(): + with evaluation_row_session(eval_row_id) as ( + _row_db, + _catalog_db, + eval_row, + source_row, + _shard_id, + ): + return _to_evaluation_row_response(eval_row, source_row, evaluation) + + db.refresh(eval_row) + source_row = targets[0][1] + return _to_evaluation_row_response(eval_row, source_row, evaluation) + + +from app.core.auth.capabilities import EVALS_RUN, EVALS_VIEW, REPORTS_GENERATE +from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities + +apply_workspace_route_capabilities( + router, + view_capability=EVALS_VIEW, + manage_capability=EVALS_RUN, + run_capability=EVALS_RUN, + report_capability=REPORTS_GENERATE, +) diff --git a/app/api/v1/routes/call_imports.py b/app/api/v1/routes/call_imports.py index e7461504..3b7bab55 100644 --- a/app/api/v1/routes/call_imports.py +++ b/app/api/v1/routes/call_imports.py @@ -1,3947 +1,4016 @@ -"""CSV-driven call import routes. - -Users upload a CSV plus a per-batch column mapping (CSV header -> system -field). The backend persists a CallImport batch + one CallImportRow per -line, then fans the rows out to the Celery ``imports`` queue where each -row is downloaded using the telephony credential pinned on the batch. -Exotel credentialed imports require a ``recording_url`` on every row; -direct-URL imports (no credential) also require a mapped recording URL. -""" - -from __future__ import annotations - -import csv -import io -import json -import re -from dataclasses import dataclass, field -from datetime import date, datetime, time, timedelta -from typing import Any, Dict, Iterable, List, Optional, Tuple -from uuid import UUID, uuid4 - -from fastapi import APIRouter, Body, BackgroundTasks, Depends, File, Form, HTTPException, Query, Response, UploadFile, status -from loguru import logger -from sqlalchemy import desc, func, or_ -from sqlalchemy.orm import Session - -from app.config import settings -from app.core.auth.rbac import require_admin -from app.database import get_db -from app.db_sharding.sessions import is_sharding_enabled -from app.dependencies import ( - get_api_key, - get_organization_id, - get_workspace_id, - require_enterprise_feature, -) -from app.services.billing.flexprice_service import record_call_import_batch_created -from app.services.call_imports.dispatch_diagnostics import ( - build_call_import_dispatch_diagnostics, -) -from app.models.database import ( - CallImport, - CallImportRow, - CallImportSchema, - CallImportSchemaParameter, - CallImportTag, - TelephonyIntegration, -) -from app.models.enums import ( - CallImportParameterType, - CallImportRowStatus, - CallImportStatus, -) -from app.models.schemas import ( - CallImportCancelDiarisationRequest, - CallImportCancelDiarisationResponse, - CallImportDetailResponse, - CallImportDeleteResponse, - CallImportDiarisationPromptDefaultResponse, - CallImportDispatchDiagnosticsResponse, - CallImportInsightsMetric, - CallImportInsightsResponse, - CallImportInsightsRunPoint, - CallImportListResponse, - CallImportMappingUpdate, - CallImportMetricAggregate, - CallImportPreviewResponse, - CallImportPreviewSheet, - CallImportRetryFailedRowsRequest, - CallImportRetryFailedRowsResponse, - CallImportResponse, - CallImportRowIdsResponse, - CallImportRowBulkDelete, - CallImportRowBulkDeleteResponse, - CallImportRowResponse, - CallImportStartRequest, - CallImportTranscribeRequest, - CallImportTranscribeResponse, - CallImportUpdate, - CallImportUploadResponse, -) - - -router = APIRouter( - prefix="/call-imports", - tags=["Call Imports"], - dependencies=[Depends(require_enterprise_feature("call_imports"))], -) - - -@dataclass(frozen=True) -class CallImportParseSkip: - """One source row excluded during CSV/Excel parse (identity / recording URL).""" - - source_row: int - reason: str - message: str - - -@dataclass -class CallImportParseResult: - rows: List[Dict[str, Any]] = field(default_factory=list) - skipped: List[CallImportParseSkip] = field(default_factory=list) - - -def parse_skips_to_json(skips: List[CallImportParseSkip]) -> List[Dict[str, Any]]: - """Persistable JSON shape for ``CallImport.source_row_skips``.""" - return [ - { - "source_row": item.source_row, - "reason": item.reason, - "message": item.message, - } - for item in skips - ] - - -def _normalize_dataset(raw: Optional[str]) -> Optional[str]: - """Trim and treat empty strings as 'no dataset' (NULL).""" - if raw is None: - return None - cleaned = raw.strip() - return cleaned or None - - -def _serialize_call_import(db: Session, call_import: CallImport) -> CallImportResponse: - """Catalog parent fields; counters come from SQL rollup (not Redis merge).""" - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - from app.services.call_imports.progress_counters import ( - clear_import_progress_redis, - read_import_progress, - ) - - redis_completed, redis_failed = read_import_progress(call_import.id) - if ( - redis_completed - or redis_failed - or int(call_import.completed_rows or 0) > int(call_import.total_rows or 0) - or int(call_import.failed_rows or 0) > int(call_import.total_rows or 0) - ): - rollup_call_import_batch_status(db, call_import) - db.flush() - - clear_import_progress_redis(call_import.id) - db.refresh(call_import) - total = int(call_import.total_rows or 0) - completed = min(int(call_import.completed_rows or 0), total) if total else int( - call_import.completed_rows or 0 - ) - failed = min(int(call_import.failed_rows or 0), total) if total else int( - call_import.failed_rows or 0 - ) - base = CallImportResponse.model_validate(call_import) - return base.model_copy(update={"completed_rows": completed, "failed_rows": failed}) - - -def _resolve_tags( - db: Session, organization_id: UUID, tag_ids: Optional[List[UUID]] -) -> List[CallImportTag]: - """Look up tag rows by id, scoped to the organization. - - Raises HTTPException(400) if any id is unknown for the org. - """ - if not tag_ids: - return [] - rows = ( - db.query(CallImportTag) - .filter( - CallImportTag.organization_id == organization_id, - CallImportTag.id.in_(tag_ids), - ) - .all() - ) - found_ids = {row.id for row in rows} - missing = [str(tag_id) for tag_id in tag_ids if tag_id not in found_ids] - if missing: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unknown call_import_tag id(s): {missing}", - ) - return rows - - -MAX_UPLOAD_BYTES = 15 * 1024 * 1024 # 15 MB upload cap (CSV or Excel) - -# File extensions accepted by the upload + preview endpoints. Keep in -# lockstep with the frontend ``accept`` attribute on the file picker. -CSV_EXTENSIONS = (".csv",) -XLSX_EXTENSIONS = (".xlsx", ".xlsm") -ALLOWED_EXTENSIONS = CSV_EXTENSIONS + XLSX_EXTENSIONS - -AUDIO_CONTENT_TYPES = { - "wav": "audio/wav", - "mp3": "audio/mpeg", - "flac": "audio/flac", - "m4a": "audio/mp4", -} - - -def _file_format(filename: Optional[str]) -> Optional[str]: - """Classify ``filename`` as ``'csv'`` / ``'xlsx'`` or ``None`` if unsupported.""" - if not filename: - return None - name = filename.lower() - if name.endswith(CSV_EXTENSIONS): - return "csv" - if name.endswith(XLSX_EXTENSIONS): - return "xlsx" - return None - - -def _audio_extension(filename: Optional[str]) -> Optional[str]: - """Return the validated lower-case extension for a manual recording.""" - if not filename or "." not in filename: - return None - ext = filename.rsplit(".", 1)[-1].lower().strip() - allowed = {fmt.lower().lstrip(".") for fmt in settings.ALLOWED_AUDIO_FORMATS} - return ext if ext in allowed else None - - -def _audio_content_type(ext: str, upload_content_type: Optional[str]) -> str: - """Prefer the browser-supplied audio content type, with a safe fallback.""" - supplied = (upload_content_type or "").strip() - if supplied and supplied != "application/octet-stream": - return supplied - return AUDIO_CONTENT_TYPES.get(ext.lower(), "application/octet-stream") - - -def _audio_s3_key( - organization_id: UUID, call_import_id: UUID, row_id: UUID, ext: str -) -> str: - """Build the canonical S3 key for a manually uploaded recording.""" - from app.services.storage.s3_service import s3_service - - return ( - f"{s3_service.prefix}organizations/{organization_id}/call_imports/" - f"{call_import_id}/{row_id}.{ext}" - ) - - -def _filename_stem(filename: Optional[str]) -> str: - """Extract a cross-platform filename stem from an UploadFile name.""" - raw = (filename or "").strip() - basename = re.split(r"[\\/]", raw)[-1] if raw else "" - if "." in basename: - basename = basename.rsplit(".", 1)[0] - return basename.strip() - - -def _sanitize_conversation_id(raw: str) -> str: - """Turn a filename stem into a stable conversation_id.""" - cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", raw.strip()) - cleaned = re.sub(r"_+", "_", cleaned).strip("._-") - return (cleaned or "recording")[:255] - - -def _dedupe_conversation_id( - base: str, counts: Dict[str, int] -) -> str: - """Make conversation ids unique within one manual upload batch.""" - count = counts.get(base, 0) + 1 - counts[base] = count - if count == 1: - return base - suffix = f"-{count}" - return f"{base[: 255 - len(suffix)]}{suffix}" - - -def _normalize_header(name: str) -> str: - return (name or "").strip().lower() - - -def _header_lookup(fieldnames: List[str]) -> Dict[str, str]: - """Map normalized header -> original header for case-insensitive lookup.""" - return {_normalize_header(h): h for h in fieldnames or []} - - -def _resolve_mapped_header( - mapping_value: Optional[str], header_lookup: Dict[str, str] -) -> Optional[str]: - """Translate a user-supplied CSV header into the actual column key. - - The frontend sends headers exactly as they appear in the source file, - but we still normalize on the server so trailing whitespace / casing - doesn't break matching. Returns the canonical fieldname or ``None`` - if not present in the file. - """ - if not mapping_value: - return None - return header_lookup.get(_normalize_header(mapping_value)) - - -def _xlsx_cell_to_str(value: Any) -> str: - """Coerce an openpyxl cell value to the string the rest of the - pipeline expects. - - openpyxl returns native Python types (int, float, datetime, bool, - None). The CSV path always works with strings, so we mirror that: - integers stringify cleanly (no ``.0`` suffix on whole-number floats), - datetimes use ISO-8601, booleans use SQL-style ``TRUE`` / ``FALSE``. - """ - if value is None: - return "" - if isinstance(value, bool): - return "TRUE" if value else "FALSE" - if isinstance(value, int): - return str(value) - if isinstance(value, float): - if value.is_integer(): - return str(int(value)) - return str(value) - if isinstance(value, datetime): - return value.isoformat() - if isinstance(value, date): - return value.isoformat() - if isinstance(value, time): - return value.isoformat() - if isinstance(value, timedelta): - return str(value) - return str(value) - - -def _parse_recording_date_cell(cell: str) -> date: - """Parse day-first dates with one/two digit day-month parts.""" - match = re.fullmatch(r"\s*(\d{1,2})[/-](\d{1,2})[/-](\d{4})\s*", cell) - if match: - day, month, year = (int(part) for part in match.groups()) - return date(year, month, day) - - # Native Excel date cells arrive from ``_xlsx_cell_to_str`` as ISO - # datetimes (e.g. ``2026-01-04T00:00:00``). Accept that resolved date, - # while keeping plain ISO dates rejected for hand-entered text/CSV cells. - if "T" in cell: - return datetime.fromisoformat(cell.replace("Z", "+00:00")).date() - - raise ValueError("expected D/M/YYYY or D-M-YYYY") - - -def _coerce_parameter_value( - raw: str, - param_type: CallImportParameterType, - *, - row_idx: int, - param_name: str, -) -> Any: - """Validate + coerce a single CSV cell against its declared type. - - Returns the typed Python value to surface in ``raw_columns``. Empty - strings are returned as ``None`` regardless of the parameter type so - optional cells stay null end-to-end. Coercion failures raise a - 400 with a row-anchored message. - """ - cell = (raw or "").strip() - if not cell: - return None - - if param_type == CallImportParameterType.CONVERSATION_ID: - return cell - if param_type == CallImportParameterType.RECORDING_URL: - # Recording URLs are exercised by the worker (which downloads - # them); we only do a light "starts with http" check here so a - # paste-error surfaces immediately at upload time. - lower = cell.lower() - if not (lower.startswith("http://") or lower.startswith("https://")): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - "valid recording URL (must start with http:// or https://)." - ), - ) - return cell - if param_type == CallImportParameterType.RECORDING_DATE: - try: - parsed_date = _parse_recording_date_cell(cell) - except ValueError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid recording date ({cell!r}); expected day-first " - "D/M/YYYY or D-M-YYYY." - ), - ) - return parsed_date.strftime("%d/%m/%Y") - if param_type == CallImportParameterType.TRANSCRIPT: - return cell - if param_type == CallImportParameterType.TEXT: - return cell - if param_type == CallImportParameterType.NUMBER: - try: - value = float(cell) - except ValueError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid number ({cell!r})." - ), - ) - if value.is_integer(): - return int(value) - return value - if param_type == CallImportParameterType.BOOLEAN: - truthy = {"true", "yes", "y", "1", "t"} - falsy = {"false", "no", "n", "0", "f"} - norm = cell.lower() - if norm in truthy: - return True - if norm in falsy: - return False - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid boolean ({cell!r})." - ), - ) - if param_type == CallImportParameterType.DATETIME: - try: - parsed = datetime.fromisoformat(cell.replace("Z", "+00:00")) - except ValueError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid ISO-8601 date/time ({cell!r})." - ), - ) - return parsed.isoformat() - if param_type == CallImportParameterType.URL: - lower = cell.lower() - if not (lower.startswith("http://") or lower.startswith("https://")): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - "valid URL (must start with http:// or https://)." - ), - ) - return cell - # Unknown types: store as text and let the next migration catch up. - return cell - - -def _recording_url_cell_is_valid_http(raw: str) -> bool: - cell = (raw or "").strip() - if not cell: - return False - lower = cell.lower() - return lower.startswith("http://") or lower.startswith("https://") - - -def _parameter_is_required(param: CallImportSchemaParameter) -> bool: - """Return whether a schema parameter must be mapped on every upload.""" - if param.is_required: - return True - try: - param_type = CallImportParameterType(param.type) - except ValueError: - return False - return param_type in ( - CallImportParameterType.CONVERSATION_ID, - CallImportParameterType.RECORDING_URL, - ) - - -def _apply_schema_mapping( - fieldnames: List[str], - rows_iter: Iterable[Dict[str, str]], - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, str], - skipped_columns: List[str], - *, - source_label: str = "CSV", - validate_only: bool = False, -) -> CallImportParseResult: - """Schema-driven row projection: parameter -> CSV header -> typed value. - - Validates that every required schema parameter is mapped to a CSV - header that actually exists in the file, and that every CSV header - is either mapped to a parameter or explicitly listed in - ``skipped_columns``. Returns one dict per non-empty data row with: - - * ``conversation_id`` (str, mandatory) - * ``recording_date`` (Optional[str], DD/MM/YYYY date) - * ``recording_url`` (Optional[str]) - * ``transcript`` (Optional[str]) - * ``parameter_values`` (Dict[str, Any]) of typed values keyed by - parameter name (drives ``raw_columns`` so the export can - reproduce the source). - - ``validate_only=True`` runs the header / mapping / skipped-column - checks (every check that doesn't need to read row data) and then - returns an empty list — used by the MAP stage to validate a - mapping payload against the cached sheet snapshot without - re-fetching the source bytes from S3. - """ - header_lookup = _header_lookup(list(fieldnames)) - - # 1. Look up the conversation_id parameter so we can address it - # directly while building each row. - conv_param = next( - (p for p in parameters if p.type == CallImportParameterType.CONVERSATION_ID), - None, - ) - if conv_param is None: - # The schema invariant should have caught this on create/update, - # but a defensive 400 here keeps us safe against hand-rolled - # API callers that bypassed validation. - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Selected schema is missing the mandatory conversation_id parameter.", - ) - # 2. Resolve every mapped parameter to a canonical fieldname. - # Required parameters MUST resolve; optional ones may resolve to - # None if the user left them blank (no mapping). - canonical_by_param: Dict[str, Optional[str]] = {} - recording_date_param_name: Optional[str] = None - rec_url_param_name: Optional[str] = None - transcript_param_name: Optional[str] = None - for param in parameters: - mapped_header = parameter_mapping.get(param.name) - canonical = ( - _resolve_mapped_header(mapped_header, header_lookup) - if mapped_header - else None - ) - if _parameter_is_required(param) and canonical is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"{source_label} does not contain the column " - f"'{mapped_header or ''}' mapped to required parameter " - f"'{param.name}'." - ), - ) - canonical_by_param[param.name] = canonical - if param.type == CallImportParameterType.RECORDING_DATE: - recording_date_param_name = param.name - elif param.type == CallImportParameterType.RECORDING_URL: - rec_url_param_name = param.name - elif param.type == CallImportParameterType.TRANSCRIPT: - transcript_param_name = param.name - - # 3. Every CSV column must either be mapped to a parameter or - # explicitly skipped. Catches "I forgot to skip the email - # column" gracefully instead of dropping data silently. - mapped_canonicals = {c for c in canonical_by_param.values() if c} - skipped_canonicals = { - _resolve_mapped_header(h, header_lookup) - for h in skipped_columns - } - skipped_canonicals.discard(None) - unhandled = [ - h - for h in fieldnames - if h not in mapped_canonicals and h not in skipped_canonicals - ] - if unhandled: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"{source_label} columns must either be mapped to a schema " - f"parameter or explicitly skipped. Unhandled: {unhandled}." - ), - ) - - conv_canonical = canonical_by_param[conv_param.name] - rec_canonical = ( - canonical_by_param.get(rec_url_param_name) - if rec_url_param_name - else None - ) - recording_date_canonical = ( - canonical_by_param.get(recording_date_param_name) - if recording_date_param_name - else None - ) - transcript_canonical = ( - canonical_by_param.get(transcript_param_name) - if transcript_param_name - else None - ) - - if validate_only: - # MAP-stage validation: every header check above has already - # run; the row loop only matters at IMPORT time. Skip it (and - # the "no data rows" guard at the bottom of the function) so - # the caller gets a clean pass when the mapping is shaped right. - return CallImportParseResult() - - parsed: List[Dict[str, Any]] = [] - skipped: List[CallImportParseSkip] = [] - for idx, row in enumerate(rows_iter): - # Drop fully-blank lines - matches the legacy parser behavior so - # trailing-newline edge cases don't fail an otherwise-good upload. - non_blank = any( - (row.get(c) or "").strip() - for c in mapped_canonicals - if c - ) - if not non_blank: - continue - - source_row = idx + 1 - conv_value = (row.get(conv_canonical) or "").strip() if conv_canonical else "" - if not conv_value: - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="missing_conversation_id", - message=( - f"Row {source_row} is missing the '{conv_param.name}' " - "(conversation_id) value." - ), - ) - ) - continue - - if rec_canonical and rec_url_param_name: - rec_param = next( - (p for p in parameters if p.name == rec_url_param_name), - None, - ) - if rec_param is not None and _parameter_is_required(rec_param): - rec_raw = (row.get(rec_canonical) or "").strip() - if not rec_raw: - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="missing_recording_url", - message=( - f"Row {source_row} is missing the required " - f"'{rec_url_param_name}' value." - ), - ) - ) - continue - if not _recording_url_cell_is_valid_http(rec_raw): - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="invalid_recording_url", - message=( - f"Row {source_row}: value for " - f"'{rec_url_param_name}' is not a valid recording " - "URL (must start with http:// or https://)." - ), - ) - ) - continue - - # Materialize every mapped parameter into the per-row snapshot, - # running per-type coercion so a bad cell aborts the upload - # rather than silently storing garbage. - parameter_values: Dict[str, Any] = {} - row_skipped = False - for param in parameters: - canonical = canonical_by_param[param.name] - if canonical is None: - continue - try: - param_type = CallImportParameterType(param.type) - except ValueError: - param_type = CallImportParameterType.TEXT - coerced = _coerce_parameter_value( - row.get(canonical) or "", - param_type, - row_idx=idx, - param_name=param.name, - ) - if _parameter_is_required(param) and coerced is None: - if param_type == CallImportParameterType.RECORDING_URL: - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="missing_recording_url", - message=( - f"Row {source_row} is missing the required " - f"'{param.name}' value." - ), - ) - ) - row_skipped = True - break - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {source_row} is missing the required " - f"'{param.name}' value." - ), - ) - parameter_values[param.name] = coerced - if row_skipped: - continue - - rec_value = ( - (row.get(rec_canonical) or "").strip() if rec_canonical else "" - ) - transcript_value = ( - (row.get(transcript_canonical) or "").strip() - if transcript_canonical - else "" - ) - recording_date_value = ( - parameter_values.get(recording_date_param_name) - if recording_date_param_name - else None - ) - - parsed.append( - { - "conversation_id": conv_value, - "recording_date": recording_date_value, - "recording_url": rec_value or None, - "transcript": transcript_value or None, - "parameter_values": parameter_values, - } - ) - - if not parsed and not skipped: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"{source_label} did not contain any data rows.", - ) - - return CallImportParseResult(rows=parsed, skipped=skipped) - - -def _raise_if_no_importable_rows( - result: CallImportParseResult, *, source_label: str = "CSV" -) -> None: - """Sync upload / API callers fail fast when every data row was skipped.""" - if result.rows: - return - if result.skipped: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"No importable rows. {len(result.skipped)} row(s) skipped due " - "to missing or invalid conversation ID or recording URL." - ), - ) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"{source_label} did not contain any data rows.", - ) - - -def _parse_csv( - file_bytes: bytes, - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, str], - skipped_columns: List[str], -) -> CallImportParseResult: - """Parse a CSV file using the resolved schema parameters.""" - if not file_bytes: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Uploaded CSV is empty.", - ) - - try: - text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) - except UnicodeDecodeError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV must be UTF-8 encoded.", - ) - - reader = csv.DictReader(text_stream) - if not reader.fieldnames: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV is missing a header row.", - ) - - return _apply_schema_mapping( - list(reader.fieldnames), - reader, - parameters, - parameter_mapping, - skipped_columns, - source_label="CSV", - ) - - -def _open_xlsx_workbook(file_bytes: bytes): - """Open an xlsx/xlsm workbook from in-memory bytes (read-only stream). - - Imports openpyxl lazily so the module loads even in environments that - haven't installed the optional dep yet (e.g. lightweight tooling - images). Surfaces a clean 400 if openpyxl is missing or the file is - not a valid Office Open XML workbook. - """ - if not file_bytes: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Uploaded Excel file is empty.", - ) - try: - from openpyxl import load_workbook # type: ignore - from openpyxl.utils.exceptions import InvalidFileException # type: ignore - except ImportError as exc: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=( - "Excel uploads require the 'openpyxl' package which is " - "not installed in this environment." - ), - ) from exc - - try: - return load_workbook( - io.BytesIO(file_bytes), - read_only=True, - data_only=True, - ) - except InvalidFileException as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"File is not a valid .xlsx workbook: {exc}", - ) from exc - except Exception as exc: # zipfile.BadZipFile etc. - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Could not open Excel workbook: {exc}", - ) from exc - - -def _xlsx_sheet_headers_and_rows( - worksheet, -) -> Tuple[List[str], List[Dict[str, str]]]: - """Read row 1 as headers and the rest as dicts of stringified cells. - - Empty trailing header cells are dropped. Duplicate headers preserve - the first occurrence (matches ``csv.DictReader`` behavior, which - silently drops duplicates). - """ - iterator = worksheet.iter_rows(values_only=True) - try: - header_row = next(iterator) - except StopIteration: - return [], [] - - headers: List[str] = [] - seen: set[str] = set() - for cell in header_row: - name = _xlsx_cell_to_str(cell).strip() - if not name: - # Stop at the first blank header — treats trailing empty - # columns as not part of the table (matches typical Excel - # workbook conventions). - break - norm = name.lower() - if norm in seen: - continue - seen.add(norm) - headers.append(name) - - rows: List[Dict[str, str]] = [] - for row in iterator: - if row is None: - continue - # Pad / truncate to the header length so dict construction is - # stable even when a row has fewer / extra cells than the header. - cells = list(row[: len(headers)]) - if len(cells) < len(headers): - cells.extend([None] * (len(headers) - len(cells))) - if not any(_xlsx_cell_to_str(c).strip() for c in cells): - # Skip fully-blank rows (openpyxl read_only routinely yields - # trailing empties when the worksheet's used range exceeds - # the actual data). - continue - rows.append( - { - header: _xlsx_cell_to_str(value) - for header, value in zip(headers, cells) - } - ) - - return headers, rows - - -def _parse_xlsx( - file_bytes: bytes, - sheet_name: Optional[str], - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, str], - skipped_columns: List[str], -) -> CallImportParseResult: - """Parse a single worksheet from an xlsx/xlsm workbook. - - ``sheet_name`` must match one of the workbook's sheets (case - insensitive whitespace-trimmed match). Returns the same shape as - :func:`_parse_csv` so the upload handler can persist either format - through the same code path. - """ - if not sheet_name or not sheet_name.strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is required when uploading an Excel workbook.", - ) - - workbook = _open_xlsx_workbook(file_bytes) - try: - sheet_names = list(workbook.sheetnames) - target_norm = sheet_name.strip().lower() - match = next( - (s for s in sheet_names if s.strip().lower() == target_norm), - None, - ) - if match is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Sheet '{sheet_name}' not found in workbook. " - f"Available sheets: {sheet_names}" - ), - ) - worksheet = workbook[match] - headers, rows = _xlsx_sheet_headers_and_rows(worksheet) - finally: - workbook.close() - - if not headers: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Sheet '{sheet_name}' is missing a header row.", - ) - - return _apply_schema_mapping( - headers, - rows, - parameters, - parameter_mapping, - skipped_columns, - source_label=f"Sheet '{sheet_name}'", - ) - - -def _csv_preview_sheets( - file_bytes: bytes, filename: Optional[str] -) -> List[CallImportPreviewSheet]: - """Build the synthetic single-sheet preview entry for a CSV upload.""" - if not file_bytes: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Uploaded CSV is empty.", - ) - try: - text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) - except UnicodeDecodeError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV must be UTF-8 encoded.", - ) - reader = csv.DictReader(text_stream) - if not reader.fieldnames: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV is missing a header row.", - ) - headers = list(reader.fieldnames) - row_count = 0 - for row in reader: - # Match the parse-time skip: ignore fully blank rows so the - # count the user sees lines up with what /upload will ingest. - if any((v or "").strip() for v in row.values()): - row_count += 1 - - sheet_label = (filename or "sheet1").rsplit("/", 1)[-1] or "sheet1" - return [ - CallImportPreviewSheet( - name=sheet_label, - headers=headers, - row_count=row_count, - ) - ] - - -def _xlsx_preview_sheets(file_bytes: bytes) -> List[CallImportPreviewSheet]: - """List every worksheet in the workbook with its headers and row count.""" - workbook = _open_xlsx_workbook(file_bytes) - sheets: List[CallImportPreviewSheet] = [] - try: - for name in workbook.sheetnames: - worksheet = workbook[name] - headers, rows = _xlsx_sheet_headers_and_rows(worksheet) - sheets.append( - CallImportPreviewSheet( - name=name, - headers=headers, - row_count=len(rows), - ) - ) - finally: - workbook.close() - return sheets - - -def _parse_json_form_field(name: str, raw: Optional[str], default): - """Decode a JSON-encoded form field with a friendly 400 on bad JSON.""" - if raw is None or raw == "": - return default - try: - return json.loads(raw) - except json.JSONDecodeError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"{name} must be valid JSON: {exc}", - ) - - -# --------------------------------------------------------------------------- -# Shared helpers used by the staged endpoints (UPLOAD / MAP / IMPORT) and the -# legacy one-shot ``POST /upload`` shim. Extracted here so each stage and the -# back-compat path operate on the exact same validation + persistence code. -# --------------------------------------------------------------------------- - - -def _source_content_type(fmt: str) -> str: - """Return the canonical ``Content-Type`` for a parsed file format.""" - if fmt == "csv": - return "text/csv" - if fmt == "xlsx": - return ( - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ) - return "application/octet-stream" - - -def _source_s3_key( - organization_id: UUID, call_import_id: UUID, fmt: str -) -> str: - """Build the canonical S3 key for an upload's source file. - - Mirrors the per-row recording key convention used by - ``process_call_import_row`` so a single prefix sweep on delete still - cleans up both the source artefact and every fetched recording. - """ - from app.services.storage.s3_service import s3_service - - ext = "xlsx" if fmt == "xlsx" else "csv" - return ( - f"{s3_service.prefix}organizations/{organization_id}/call_imports/" - f"{call_import_id}/source.{ext}" - ) - - -def _build_available_sheets( - file_bytes: bytes, fmt: str, filename: Optional[str] -) -> List[CallImportPreviewSheet]: - """Snapshot of sheets + headers cached on the batch at UPLOAD time.""" - if fmt == "csv": - return _csv_preview_sheets(file_bytes, filename) - return _xlsx_preview_sheets(file_bytes) - - -def _resolve_schema( - db: Session, - organization_id: UUID, - workspace_id: UUID, - schema_id: UUID, -) -> CallImportSchema: - """Fetch + validate a schema row in the active workspace. - - Eager-loads ``parameters`` so callers can iterate without re-querying. - """ - from sqlalchemy.orm import selectinload as _selectinload - - schema = ( - db.query(CallImportSchema) - .options(_selectinload(CallImportSchema.parameters)) - .filter( - CallImportSchema.id == schema_id, - CallImportSchema.organization_id == organization_id, - CallImportSchema.workspace_id == workspace_id, - ) - .first() - ) - if not schema: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Call import schema not found in the active workspace.", - ) - if not list(schema.parameters): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Selected schema has no parameters defined.", - ) - return schema - - -def _validate_direct_url_import_ready( - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, Any], -) -> None: - """Ensure direct-URL import has a mapped recording_url column.""" - rec_url_param = next( - ( - p - for p in parameters - if p.type == CallImportParameterType.RECORDING_URL.value - ), - None, - ) - if rec_url_param is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Direct URL import requires a schema parameter of type " - "'recording_url'." - ), - ) - mapped_header = (parameter_mapping or {}).get(rec_url_param.name) - if not (mapped_header or "").strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Direct URL import requires the 'recording_url' parameter to " - "be mapped to a source column." - ), - ) - - -def _validate_exotel_import_ready( - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, Any], -) -> None: - """Ensure Exotel credentialed import has a mapped recording_url column.""" - rec_url_param = next( - ( - p - for p in parameters - if p.type == CallImportParameterType.RECORDING_URL.value - ), - None, - ) - if rec_url_param is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Exotel import requires a schema parameter of type " - "'recording_url'." - ), - ) - mapped_header = (parameter_mapping or {}).get(rec_url_param.name) - if not (mapped_header or "").strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Exotel import requires the 'recording_url' parameter to " - "be mapped to a source column." - ), - ) - - -def _resolve_telephony_integration( - db: Session, - organization_id: UUID, - telephony_integration_id: UUID, - provider: str, -) -> TelephonyIntegration: - """Fetch + validate a telephony credential against the requested provider.""" - integration = ( - db.query(TelephonyIntegration) - .filter( - TelephonyIntegration.id == telephony_integration_id, - TelephonyIntegration.organization_id == organization_id, - ) - .first() - ) - if not integration: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Telephony credential not found for this organization.", - ) - if (integration.provider or "").lower() != provider.lower(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Selected credential is for provider '{integration.provider}', " - f"but request specified '{provider}'." - ), - ) - if not integration.is_active: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Selected telephony credential is inactive.", - ) - return integration - - -def _clean_parameter_mapping( - mapping_payload: Any, - parameters: List[CallImportSchemaParameter], - schema_name: str, -) -> Dict[str, str]: - """Trim values and drop empties; reject unknown parameter names. - - Accepts an already-decoded value (dict-shaped) so the same helper - works for the JSON-form upload path and the JSON-body PATCH path. - """ - if not isinstance(mapping_payload, dict) or not all( - isinstance(k, str) and (v is None or isinstance(v, str)) - for k, v in mapping_payload.items() - ): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "parameter_mapping must be an object of " - "{parameter_name: csv_header}." - ), - ) - - valid_param_names = {p.name for p in parameters} - cleaned: Dict[str, str] = {} - for raw_name, raw_header in mapping_payload.items(): - if raw_name not in valid_param_names: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"parameter_mapping references unknown parameter " - f"'{raw_name}' on schema '{schema_name}'." - ), - ) - header = (raw_header or "").strip() - if header: - cleaned[raw_name] = header - return cleaned - - -def _clean_skipped_columns(skipped_payload: Any) -> List[str]: - """Dedupe (case-insensitively) and drop blanks; preserve original casing.""" - if not isinstance(skipped_payload, list) or not all( - isinstance(item, str) for item in skipped_payload - ): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="skipped_columns must be a list of header strings.", - ) - cleaned: List[str] = [] - seen: set[str] = set() - for item in skipped_payload: - norm = _normalize_header(item) - if not norm or norm in seen: - continue - seen.add(norm) - cleaned.append(item) - return cleaned - - -def _parse_source_file( - file_bytes: bytes, - fmt: str, - sheet_name: Optional[str], - parameters: List[CallImportSchemaParameter], - cleaned_mapping: Dict[str, str], - cleaned_skipped: List[str], -) -> CallImportParseResult: - """Run the format-appropriate parser against a buffer of file bytes.""" - if fmt == "csv": - return _parse_csv(file_bytes, parameters, cleaned_mapping, cleaned_skipped) - return _parse_xlsx( - file_bytes, sheet_name, parameters, cleaned_mapping, cleaned_skipped - ) - - -def _materialize_rows( - db: Session, - call_import: CallImport, - parsed_rows: List[Dict[str, Any]], - organization_id: UUID, -) -> List[CallImportRow]: - """Insert one ``CallImportRow`` per parsed row, returning the new models.""" - row_models: List[CallImportRow] = [] - for idx, row in enumerate(parsed_rows): - # Stamp ``transcript_source='csv'`` when the upload actually - # provided a transcript so the UI badge ("From CSV") works from - # day one. Blank cells stay NULL so the row reads as "no - # production transcript yet". - csv_transcript = row["transcript"] - row_model = CallImportRow( - call_import_id=call_import.id, - organization_id=organization_id, - workspace_id=call_import.workspace_id, - row_index=idx, - conversation_id=row["conversation_id"], - recording_date=( - _parse_recording_date_cell(row["recording_date"]) - if row.get("recording_date") - else None - ), - recording_url=row["recording_url"], - transcript=csv_transcript, - transcript_source=( - "csv" if csv_transcript and csv_transcript.strip() else None - ), - raw_columns=row["parameter_values"] or None, - status=CallImportRowStatus.PENDING, - ) - db.add(row_model) - row_models.append(row_model) - return row_models - - -def _enqueue_row_tasks( - db: Session, - call_import: CallImport, - row_models: List[CallImportRow], -) -> None: - """Schedule fair round-robin dispatch for pending import rows.""" - del db, call_import, row_models - from app.workers.concurrency.fair_import_dispatch import ( - schedule_fair_import_dispatch, - ) - - schedule_fair_import_dispatch(max_workspace_turns=999) - - -def _ensure_blob_storage_enabled() -> None: - """Hard-fail UPLOAD if cloud blob storage isn't configured (no local fallback).""" - from app.services.storage.s3_service import s3_service - - if not s3_service.is_enabled(): - err = ( - s3_service.get_status_message() - or "Cloud blob storage is not enabled or not configured" - ) - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail=( - "Call uploads require cloud blob storage so the file can be " - f"persisted between stages: {err}" - ), - ) - - -def _validate_sheet_choice( - fmt: str, - sheet_name: Optional[str], - available_sheets: Optional[List[Dict[str, Any]]], -) -> Optional[str]: - """Normalize / validate ``sheet_name`` against the persisted snapshot. - - Returns the canonical sheet name (matching the workbook's casing) - so downstream parsing addresses the right worksheet. - """ - if fmt == "csv": - if sheet_name and sheet_name.strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is not applicable to CSV uploads.", - ) - return None - - cleaned = (sheet_name or "").strip() or None - if cleaned is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is required when the source is an Excel workbook.", - ) - - if not available_sheets: - # Nothing to validate against (e.g. legacy batch without snapshot); - # let downstream parsing error out instead of silently importing. - return cleaned - - target = cleaned.strip().lower() - for entry in available_sheets: - name = entry.get("name") if isinstance(entry, dict) else None - if isinstance(name, str) and name.strip().lower() == target: - return name - sheet_names = [ - entry.get("name") for entry in available_sheets if isinstance(entry, dict) - ] - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Sheet '{cleaned}' not found in the staged file. " - f"Available sheets: {sheet_names}" - ), - ) - - -def _tag_response_payload(tags: Optional[List[CallImportTag]]) -> List[Dict[str, Any]]: - """Shape a CallImport's tag relationship for the upload response.""" - return [ - { - "id": tag.id, - "name": tag.name, - "color": tag.color, - "created_at": tag.created_at, - "updated_at": tag.updated_at, - } - for tag in (tags or []) - ] - - -@router.post( - "/preview", - response_model=CallImportPreviewResponse, - operation_id="previewCallImportFile", -) -async def preview_call_import_file( - file: UploadFile = File(...), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportPreviewResponse: - """Inspect an uploaded CSV / Excel file and return its sheets + headers. - - Drives the column-mapping UI without forcing the frontend to parse - CSV / xlsx itself — keeps client and server in lockstep on quoted - fields, encodings, and Excel cell coercion. CSVs return a single - synthetic sheet named after the filename; Excel workbooks return one - entry per worksheet (in workbook order). - """ - del api_key, organization_id, workspace_id, db # auth only - - fmt = _file_format(file.filename) - if fmt is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Unsupported file format. Allowed extensions: " - f"{', '.join(ALLOWED_EXTENSIONS)}." - ), - ) - - file_bytes = await file.read() - if len(file_bytes) > MAX_UPLOAD_BYTES: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", - ) - - if fmt == "csv": - sheets = _csv_preview_sheets(file_bytes, file.filename) - else: - sheets = _xlsx_preview_sheets(file_bytes) - - return CallImportPreviewResponse(format=fmt, sheets=sheets) - - -@router.post( - "", - response_model=CallImportResponse, - status_code=status.HTTP_201_CREATED, - operation_id="createCallImport", -) -async def create_call_import( - file: UploadFile = File( - ..., - description="CSV / Excel file to stage. Persisted to S3 between stages.", - ), - dataset: str = Form( - ..., - description=( - "Required free-text dataset label. Collected up-front so the " - "batch is filterable from the moment it lands." - ), - ), - tag_ids: Optional[List[UUID]] = Form( - None, - description="Optional list of CallImportTag ids to attach to the new batch.", - ), - schema_id: Optional[UUID] = Form( - None, - description=( - "Optional schema pre-pick. The user can still change it during " - "the MAP stage; provided here only so the detail page can pre-" - "select the schema dropdown." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportResponse: - """UPLOAD stage of the staged call-import flow. - - Persists the source file to S3 and creates a ``CallImport`` row with - ``status='uploaded'``. No mapping, no provider, no rows yet — the - user moves through MAP and IMPORT as separate idempotent steps. - - Dataset is collected here (rather than at IMPORT) so the batch is - filterable from the moment it appears in the list view. - """ - del api_key - - fmt = _file_format(file.filename) - if fmt is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Unsupported file format. Allowed extensions: " - f"{', '.join(ALLOWED_EXTENSIONS)}." - ), - ) - - normalized_dataset = _normalize_dataset(dataset) - if not normalized_dataset: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="dataset is required and must be a non-empty string.", - ) - - file_bytes = await file.read() - if len(file_bytes) > MAX_UPLOAD_BYTES: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", - ) - - # Parse-now so we (a) reject garbage uploads up-front instead of - # later in the MAP step, and (b) capture the sheets snapshot the - # MAP UI needs without having to re-fetch the file from S3. - sheets = _build_available_sheets(file_bytes, fmt, file.filename) - - # Optional schema pre-pick: validated only if supplied (the user is - # allowed to set it for the first time during MAP). - if schema_id is not None: - _resolve_schema(db, organization_id, workspace_id, schema_id) - - tag_rows = _resolve_tags(db, organization_id, tag_ids) - - _ensure_blob_storage_enabled() - - # Pre-generate the id so we can compute a deterministic S3 key - # before the row is persisted, keeping ``source_s3_key`` consistent - # with the prefix sweep used at delete-time. - import uuid as _uuid - - call_import_id = _uuid.uuid4() - s3_key = _source_s3_key(organization_id, call_import_id, fmt) - content_type = _source_content_type(fmt) - - from app.services.storage.s3_service import s3_service, StorageError - - try: - s3_service.upload_file_by_key(file_bytes, s3_key, content_type=content_type) - except StorageError as exc: - logger.exception( - "Failed to upload source file to S3 for new call import {}", - call_import_id, - ) - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail=f"Failed to persist upload to S3: {exc}", - ) - - call_import = CallImport( - id=call_import_id, - organization_id=organization_id, - workspace_id=workspace_id, - # Provider + credential aren't known until the IMPORT stage; leave - # them NULL so the staged-vs-legacy distinction is visible at a - # glance from the DB. - provider=None, - telephony_integration_id=None, - original_filename=file.filename, - sheet_name=None, - dataset=normalized_dataset, - schema_id=schema_id, - parameter_mapping={}, - skipped_columns=[], - column_mapping={}, - extra_columns=[], - custom_column_mapping={}, - source_s3_key=s3_key, - source_format=fmt, - source_size_bytes=len(file_bytes), - source_content_type=content_type, - available_sheets=[sheet.model_dump() for sheet in sheets], - total_rows=0, - completed_rows=0, - failed_rows=0, - status=CallImportStatus.UPLOADED, - ) - if tag_rows: - call_import.tags = tag_rows - - db.add(call_import) - try: - db.commit() - except Exception: - db.rollback() - # Best-effort cleanup of the uploaded S3 object so a failed - # commit doesn't leak storage. - try: - s3_service.delete_file_by_key(s3_key) - except Exception as cleanup_exc: # noqa: BLE001 - logger.warning( - "Failed to clean up orphaned S3 object {} after DB rollback: {}", - s3_key, - cleanup_exc, - ) - raise - - db.refresh(call_import) - return _serialize_call_import(db, call_import) - - -@router.patch( - "/{call_import_id}/mapping", - response_model=CallImportResponse, - operation_id="updateCallImportMapping", -) -async def update_call_import_mapping( - call_import_id: UUID, - payload: CallImportMappingUpdate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportResponse: - """MAP stage of the staged call-import flow. - - Validates ``parameter_mapping`` + ``skipped_columns`` against the - sheet headers captured at UPLOAD time and persists them on the - batch. Idempotent: callers may submit this multiple times while - the batch is in ``uploaded`` or ``mapped`` state. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if call_import.status not in ( - CallImportStatus.UPLOADED, - CallImportStatus.MAPPED, - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Cannot edit mapping on a batch in status " - f"'{call_import.status.value}'. Mapping can only be edited " - "before the IMPORT stage." - ), - ) - - if not call_import.source_s3_key or not call_import.source_format: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This batch was not uploaded through the staged flow and " - "cannot have its mapping edited." - ), - ) - - schema = _resolve_schema( - db, organization_id, workspace_id, payload.schema_id - ) - parameters = list(schema.parameters) - - canonical_sheet = _validate_sheet_choice( - call_import.source_format, - payload.sheet_name, - call_import.available_sheets, - ) - - # Pull the headers for the selected sheet straight out of the - # snapshot so we don't have to re-download the file from S3 just to - # validate the mapping. - headers: List[str] = [] - if call_import.available_sheets: - if canonical_sheet is None: - # CSV: single synthetic sheet. - entry = call_import.available_sheets[0] - headers = list(entry.get("headers") or []) - else: - for entry in call_import.available_sheets: - if not isinstance(entry, dict): - continue - name = entry.get("name") - if isinstance(name, str) and name == canonical_sheet: - headers = list(entry.get("headers") or []) - break - - cleaned_mapping = _clean_parameter_mapping( - payload.parameter_mapping, parameters, schema.name - ) - cleaned_skipped = _clean_skipped_columns(payload.skipped_columns) - - # Run the same per-column validation as the parse path so the user - # gets an immediate 400 if a required parameter is left unmapped or - # a header is neither mapped nor skipped — without needing to read - # the file. ``validate_only`` skips the row loop (and the empty-rows - # guard) since the row data lives in S3, not in this request. - if headers: - _apply_schema_mapping( - headers, - iter(()), - parameters, - cleaned_mapping, - cleaned_skipped, - source_label=( - f"Sheet '{canonical_sheet}'" - if canonical_sheet is not None - else "CSV" - ), - validate_only=True, - ) - - call_import.schema_id = schema.id - call_import.parameter_mapping = dict(cleaned_mapping) - call_import.skipped_columns = list(cleaned_skipped) - call_import.sheet_name = canonical_sheet - call_import.status = CallImportStatus.MAPPED - db.commit() - db.refresh(call_import) - return _serialize_call_import(db, call_import) - - -@router.post( - "/{call_import_id}/import", - response_model=CallImportUploadResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="startCallImport", -) -async def start_call_import( - call_import_id: UUID, - payload: CallImportStartRequest, - background_tasks: BackgroundTasks, - legacy: bool = Query( - False, - description=( - "Deprecated escape hatch for import-only processing. " - "New batches should use Run Evaluation instead." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportUploadResponse: - """Deprecated IMPORT stage — use Run Evaluation for new batches. - - Recording fetch is part of the unified evaluation pipeline. This - endpoint remains available only with ``?legacy=true`` for backward - compatibility. - """ - del api_key, background_tasks - - if not legacy: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "Standalone import is deprecated. Use Run Evaluation — " - "recording fetch is part of the evaluation pipeline. " - "Append ?legacy=true to use the import-only path." - ), - ) - - from sqlalchemy.orm import selectinload as _selectinload - - call_import = ( - db.query(CallImport) - .options(_selectinload(CallImport.tags)) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if call_import.status != CallImportStatus.MAPPED: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Cannot start import for a batch in status " - f"'{call_import.status.value}'. Map the columns first." - ), - ) - - if not call_import.source_s3_key or not call_import.source_format: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This batch has no staged source file and cannot be imported " - "through the staged flow." - ), - ) - - if not call_import.schema_id: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Cannot start import without a mapped schema.", - ) - - schema = _resolve_schema( - db, organization_id, workspace_id, call_import.schema_id - ) - parameters = list(schema.parameters) - - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - if (integration.provider or "").lower() == "exotel": - _validate_exotel_import_ready( - parameters, dict(call_import.parameter_mapping or {}) - ) - else: - _validate_direct_url_import_ready( - parameters, dict(call_import.parameter_mapping or {}) - ) - integration = None - - _ensure_blob_storage_enabled() - - if integration is not None: - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - else: - call_import.provider = None - call_import.telephony_integration_id = None - - call_import.total_rows = 0 - call_import.completed_rows = 0 - call_import.failed_rows = 0 - call_import.error_message = None - call_import.status = CallImportStatus.PROCESSING - db.commit() - db.refresh(call_import) - - from app.workers.tasks.call_import_bulk_ops import ( - materialize_call_import_rows_task, - ) - - materialize_call_import_rows_task.delay( - str(call_import_id), - str(organization_id), - str(workspace_id), - schedule_import_dispatch=True, - ) - - return CallImportUploadResponse( - id=call_import.id, - total_rows=0, - status=call_import.status, - dataset=call_import.dataset, - tags=_tag_response_payload(call_import.tags), - message=( - "Import accepted. Rows are being materialized in the background; " - "recordings will be fetched asynchronously." - ), - ) - - -@router.post( - "/upload", - response_model=CallImportUploadResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="uploadCallImportCsv", - deprecated=True, -) -async def upload_call_import_csv( - background_tasks: BackgroundTasks, - file: UploadFile = File(...), - provider: Optional[str] = Form( - None, - description=( - "Telephony provider key (e.g. 'exotel', 'plivo'). Must match the " - "selected telephony_integration_id's provider. Omit together " - "with telephony_integration_id for direct-URL import." - ), - ), - telephony_integration_id: Optional[UUID] = Form( - None, - description=( - "Specific TelephonyIntegration credential row to use when " - "downloading recordings for this batch. Omit together with " - "provider for direct-URL import." - ), - ), - schema_id: UUID = Form( - ..., - description=( - "Reusable Input Parameter schema this upload is mapped against. " - "Must belong to the active workspace." - ), - ), - parameter_mapping: str = Form( - ..., - description=( - "JSON-encoded ``{schema_parameter_name: source_header}`` map " - "covering every required schema parameter. Optional parameters " - "may be omitted or set to an empty string." - ), - ), - skipped_columns: Optional[str] = Form( - None, - description=( - "JSON-encoded list of source header strings the uploader has " - "explicitly skipped. Every header in the file must either be " - "mapped or appear here; otherwise the upload is rejected so a " - "forgotten column never silently drops." - ), - ), - dataset: Optional[str] = Form( - None, - description=( - "Optional free-text dataset label for high-level segregation. " - "Empty strings are stored as NULL." - ), - ), - tag_ids: Optional[List[UUID]] = Form( - None, - description="Optional list of CallImportTag ids to attach to the new batch.", - ), - sheet_name: Optional[str] = Form( - None, - description=( - "Worksheet to import when the file is an Excel workbook " - "(.xlsx / .xlsm). REQUIRED for Excel uploads. Ignored for CSV " - "uploads (rejected with 400 if non-empty so typos surface " - "instead of silently importing the wrong source)." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportUploadResponse: - """Legacy one-shot upload kept for backward compatibility. - - DEPRECATED: prefer the staged flow - (``POST /`` → ``PATCH /{id}/mapping`` → ``POST /{id}/import``) so - each step is idempotent and resumable. This endpoint runs all three - stages inline in a single transaction so existing scripts / - integrations keep working unchanged. - """ - del api_key - - fmt = _file_format(file.filename) - if fmt is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Unsupported file format. Allowed extensions: " - f"{', '.join(ALLOWED_EXTENSIONS)}." - ), - ) - - sheet_name_clean = (sheet_name or "").strip() or None - if fmt == "csv" and sheet_name_clean is not None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is not applicable to CSV uploads.", - ) - if fmt == "xlsx" and sheet_name_clean is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is required when uploading an Excel workbook.", - ) - - file_bytes = await file.read() - if len(file_bytes) > MAX_UPLOAD_BYTES: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", - ) - - schema = _resolve_schema(db, organization_id, workspace_id, schema_id) - parameters = list(schema.parameters) - - mapping_payload = _parse_json_form_field( - "parameter_mapping", parameter_mapping, {} - ) - cleaned_mapping = _clean_parameter_mapping( - mapping_payload, parameters, schema.name - ) - - skipped_payload = _parse_json_form_field("skipped_columns", skipped_columns, []) - cleaned_skipped = _clean_skipped_columns(skipped_payload) - - has_provider = bool((provider or "").strip()) - has_integration = telephony_integration_id is not None - if has_provider != has_integration: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "provider and telephony_integration_id must both be provided " - "or both omitted for direct-URL import." - ), - ) - - if telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, organization_id, telephony_integration_id, provider or "" - ) - if (integration.provider or "").lower() == "exotel": - _validate_exotel_import_ready(parameters, cleaned_mapping) - else: - _validate_direct_url_import_ready(parameters, cleaned_mapping) - integration = None - - parsed_rows = _parse_source_file( - file_bytes, fmt, sheet_name_clean, parameters, cleaned_mapping, cleaned_skipped - ) - _raise_if_no_importable_rows(parsed_rows, source_label=fmt) - - tag_rows = _resolve_tags(db, organization_id, tag_ids) - - call_import = CallImport( - organization_id=organization_id, - workspace_id=workspace_id, - provider=integration.provider if integration is not None else None, - telephony_integration_id=integration.id if integration is not None else None, - original_filename=file.filename, - sheet_name=sheet_name_clean, - dataset=_normalize_dataset(dataset), - schema_id=schema.id, - parameter_mapping=dict(cleaned_mapping), - skipped_columns=list(cleaned_skipped), - # Legacy columns are left empty on new uploads; the detail page - # falls back to ``parameter_mapping`` when ``schema_id`` is set. - column_mapping={}, - extra_columns=[], - custom_column_mapping={}, - total_rows=len(parsed_rows.rows), - completed_rows=0, - failed_rows=0, - status=CallImportStatus.PENDING, - source_row_skips=parse_skips_to_json(parsed_rows.skipped), - ) - if tag_rows: - call_import.tags = tag_rows - db.add(call_import) - db.flush() # populate call_import.id - if integration is None: - # The model's historical Python default is "exotel"; direct-URL - # imports intentionally have no telephony provider. - call_import.provider = None - - row_models = _materialize_rows( - db, call_import, parsed_rows.rows, organization_id - ) - - call_import.status = CallImportStatus.PROCESSING - db.commit() - db.refresh(call_import) - - background_tasks.add_task( - record_call_import_batch_created, - organization_id, - call_import.id, - workspace_id=workspace_id, - total_rows=call_import.total_rows, - source="csv", - provider=call_import.provider, - ) - - _enqueue_row_tasks(db, call_import, row_models) - - return CallImportUploadResponse( - id=call_import.id, - total_rows=call_import.total_rows, - status=call_import.status, - dataset=call_import.dataset, - tags=_tag_response_payload(call_import.tags), - message=( - f"Accepted {call_import.total_rows} rows for import. " - "Recordings will be fetched asynchronously." - ), - ) - - -@router.post( - "/audio-upload", - response_model=CallImportUploadResponse, - status_code=status.HTTP_201_CREATED, - operation_id="uploadCallImportAudio", -) -async def upload_call_import_audio( - background_tasks: BackgroundTasks, - files: List[UploadFile] = File( - ..., - description="One or more manual call recording audio files.", - ), - dataset: str = Form( - ..., - description="Required free-text dataset label for the manual upload batch.", - ), - tag_ids: Optional[List[UUID]] = Form( - None, - description="Optional list of CallImportTag ids to attach to the new batch.", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportUploadResponse: - """Persist manually uploaded recordings as completed CallImport rows. - - The rows skip the provider-download worker entirely because the audio - bytes are already in hand. From this point onward they behave exactly - like completed CSV-import rows: playback reads ``recording_s3_key`` and - the existing diarisation/evaluation endpoints can operate on them. - """ - - normalized_dataset = _normalize_dataset(dataset) - if not normalized_dataset: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="dataset is required and must be a non-empty string.", - ) - if not files: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="At least one audio file is required.", - ) - - _ensure_blob_storage_enabled() - tag_rows = _resolve_tags(db, organization_id, tag_ids) - - max_bytes = int(settings.MAX_FILE_SIZE_MB) * 1024 * 1024 - prepared: List[Dict[str, Any]] = [] - conversation_counts: Dict[str, int] = {} - - for idx, upload in enumerate(files): - filename = upload.filename or f"recording-{idx + 1}" - ext = _audio_extension(filename) - if not ext: - allowed = ", ".join(settings.ALLOWED_AUDIO_FORMATS) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unsupported audio file '{filename}'. Allowed formats: {allowed}.", - ) - - contents = await upload.read() - if not contents: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Audio file '{filename}' is empty.", - ) - if len(contents) > max_bytes: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=( - f"Audio file '{filename}' exceeds " - f"{settings.MAX_FILE_SIZE_MB} MB." - ), - ) - - base_conversation_id = _sanitize_conversation_id(_filename_stem(filename)) - conversation_id = _dedupe_conversation_id( - base_conversation_id, - conversation_counts, - ) - prepared.append( - { - "filename": filename, - "extension": ext, - "content_type": _audio_content_type(ext, upload.content_type), - "contents": contents, - "conversation_id": conversation_id, - } - ) - - original_filename = ( - prepared[0]["filename"] - if len(prepared) == 1 - else f"{len(prepared)} manual recordings" - ) - total_size = sum(len(item["contents"]) for item in prepared) - uploaded_keys: List[str] = [] - - from app.services.storage.s3_service import s3_service - - call_import = CallImport( - organization_id=organization_id, - workspace_id=workspace_id, - provider=None, - telephony_integration_id=None, - original_filename=original_filename, - source_format="audio", - source_size_bytes=total_size, - source_content_type="audio/*", - dataset=normalized_dataset, - total_rows=len(prepared), - completed_rows=len(prepared), - failed_rows=0, - status=CallImportStatus.COMPLETED, - ) - if tag_rows: - call_import.tags = tag_rows - - try: - db.add(call_import) - db.flush() - # The model's historical Python default is "exotel"; manual uploads - # intentionally have no telephony provider. - call_import.provider = None - - row_mappings: List[Dict[str, Any]] = [] - for idx, item in enumerate(prepared): - row_id = uuid4() - key = _audio_s3_key( - organization_id, - call_import.id, - row_id, - item["extension"], - ) - s3_service.upload_file_by_key( - item["contents"], - key, - content_type=item["content_type"], - ) - uploaded_keys.append(key) - - row_mappings.append( - { - "id": row_id, - "call_import_id": call_import.id, - "organization_id": organization_id, - "workspace_id": workspace_id, - "row_index": idx, - "conversation_id": item["conversation_id"], - "recording_url": None, - "transcript": None, - "transcript_source": None, - "raw_columns": {"conversation_id": item["conversation_id"]}, - "status": CallImportRowStatus.COMPLETED, - "recording_s3_key": key, - "recording_content_type": item["content_type"], - "recording_size_bytes": len(item["contents"]), - } - ) - - if is_sharding_enabled(): - from app.db_sharding.row_ops import ( - bulk_insert_mappings_on_shards, - register_shard_slices, - ) - - bulk_insert_mappings_on_shards(db, call_import.id, row_mappings) - register_shard_slices(db, call_import.id, len(row_mappings)) - else: - for mapping in row_mappings: - db.add(CallImportRow(**mapping)) - - db.commit() - except Exception as exc: - db.rollback() - if uploaded_keys and s3_service.is_enabled(): - try: - s3_service.delete_keys(uploaded_keys) - except Exception: - logger.exception( - "Failed to clean up manual audio upload keys after error" - ) - logger.exception("Failed to persist manual call recording upload") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to upload manual recordings: {exc}", - ) from exc - - db.refresh(call_import) - background_tasks.add_task( - record_call_import_batch_created, - organization_id, - call_import.id, - workspace_id=workspace_id, - total_rows=call_import.total_rows, - source="audio", - provider=None, - ) - return CallImportUploadResponse( - id=call_import.id, - total_rows=call_import.total_rows, - status=call_import.status, - dataset=call_import.dataset, - tags=_tag_response_payload(call_import.tags), - message=( - f"Uploaded {call_import.total_rows} manual recording" - f"{'' if call_import.total_rows == 1 else 's'}." - ), - ) - - -@router.get( - "", - response_model=CallImportListResponse, - operation_id="listCallImports", -) -async def list_call_imports( - page: int = Query(1, ge=1), - page_size: int = Query(20, ge=1, le=100), - status_filter: Optional[CallImportStatus] = Query(None, alias="status"), - dataset: Optional[str] = Query( - None, - description=( - "Filter by exact dataset string (case-insensitive). Pass the " - "literal value '__none__' to filter to imports with no dataset." - ), - ), - tag_id: Optional[List[UUID]] = Query( - None, - description="Filter to imports tagged with ALL of the given tag ids.", - ), - source_format: Optional[str] = Query( - None, - description=( - "Filter by source format. Use 'audio' for manual recordings or " - "'__non_audio__' for CSV/Excel/legacy imports." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportListResponse: - """List call-import batches for the active workspace, newest first. - - Scoped to (organization_id, workspace_id) so users only see imports - for the workspace they're currently in. Supports a high-level - ``dataset`` filter (powers the segregation dropdown at the top of - the imports page) plus an AND-style multi-tag filter via repeated - ``tag_id`` parameters. - """ - - query = ( - db.query(CallImport) - .filter( - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - ) - ) - if status_filter is not None: - query = query.filter(CallImport.status == status_filter) - - source_filter = (source_format or "").strip().lower() - if source_filter == "__non_audio__": - query = query.filter( - or_(CallImport.source_format.is_(None), CallImport.source_format != "audio") - ) - elif source_filter: - query = query.filter(func.lower(CallImport.source_format) == source_filter) - - if dataset is not None: - if dataset == "__none__": - query = query.filter(CallImport.dataset.is_(None)) - elif dataset.strip(): - query = query.filter( - func.lower(CallImport.dataset) == dataset.strip().lower() - ) - - if tag_id: - from app.models.database import CallImportTagAssignment - - for single_tag_id in tag_id: - sub = ( - db.query(CallImportTagAssignment.call_import_id) - .filter(CallImportTagAssignment.tag_id == single_tag_id) - .subquery() - ) - query = query.filter(CallImport.id.in_(sub)) - - total = query.count() - items = ( - query.order_by(desc(CallImport.created_at)) - .offset((page - 1) * page_size) - .limit(page_size) - .all() - ) - - return CallImportListResponse( - items=[_serialize_call_import(db, item) for item in items], - total=total, - page=page, - page_size=page_size, - ) - - -@router.get( - "/dispatch-diagnostics", - response_model=CallImportDispatchDiagnosticsResponse, - operation_id="getCallImportDispatchDiagnostics", - dependencies=[Depends(require_admin)], -) -async def get_call_import_dispatch_diagnostics( - workspace_id: Optional[UUID] = Query( - None, - description=( - "Optional workspace filter. When omitted, returns every workspace " - "in the organization with active eval dispatch state." - ), - ), - include_idle_workspaces: bool = Query( - False, - description=( - "When true, include org workspaces with zero pending rows and " - "zero in-flight slots." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportDispatchDiagnosticsResponse: - """Live eval slot usage and fair-dispatch state for operators. - - Org admins use this to diagnose cross-workspace starvation (e.g. one - workspace's 10k run blocking another's pending eval rows) by inspecting - Redis in-flight counters, pending dispatch rows, and scheduler cursors. - """ - del api_key - payload = build_call_import_dispatch_diagnostics( - db, - organization_id, - workspace_id=workspace_id, - include_idle_workspaces=include_idle_workspaces, - ) - return CallImportDispatchDiagnosticsResponse.model_validate(payload) - - -@router.get( - "/datasets", - response_model=List[str], - operation_id="listCallImportDatasets", -) -async def list_call_import_datasets( - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> List[str]: - """Return the distinct, non-null dataset labels in use for the active - workspace. - - Scoped per-workspace so each workspace's Dataset dropdown only shows - its own segregation labels. - """ - rows = ( - db.query(CallImport.dataset) - .filter( - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - CallImport.dataset.isnot(None), - CallImport.dataset != "", - ) - .distinct() - .order_by(CallImport.dataset.asc()) - .all() - ) - return [row[0] for row in rows if row[0]] - - -@router.get( - "/diarisation-prompt-default", - response_model=CallImportDiarisationPromptDefaultResponse, - operation_id="getCallImportDiarisationPromptDefault", -) -async def get_call_import_diarisation_prompt_default( - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), -) -> CallImportDiarisationPromptDefaultResponse: - """Return the canonical LLM diariser prompt. - - The Transcribe / Run Evaluation modals call this on open so they - can pre-fill the prompt textarea. Returning the constant from the - backend (rather than hard-coding it in the frontend) keeps the - fallback used by the worker and the placeholder shown in the UI - in lock-step — operators always see the *actual* default they'd - get if they leave the field blank. - - Registered before ``GET /{call_import_id}`` so the static path is - not mistaken for a UUID import id (which would 422). - """ - del api_key, organization_id - from app.workers.tasks.helpers.llm_diarisation import ( - DEFAULT_DIARIZATION_PROMPT, - ) - - return CallImportDiarisationPromptDefaultResponse( - prompt=DEFAULT_DIARIZATION_PROMPT - ) - - -@router.patch( - "/{call_import_id}", - response_model=CallImportResponse, - operation_id="updateCallImport", -) -async def update_call_import( - call_import_id: UUID, - payload: CallImportUpdate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportResponse: - """Edit dataset / tag assignments (and schema, pre-import) on a batch. - - ``dataset = ""`` clears the label; ``tag_ids = []`` removes all tag - assignments. Fields omitted from the body are left untouched. - - ``schema_id`` is only honoured while the batch is in - ``uploaded`` / ``mapped`` state — once rows have been materialised - the schema is locked. Changing the schema resets any persisted - mapping (the user must re-MAP) and rewinds status to ``uploaded``. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - body = payload.model_dump(exclude_unset=True) - if "dataset" in body: - call_import.dataset = _normalize_dataset(body["dataset"]) - - if "tag_ids" in body: - tag_ids = body["tag_ids"] or [] - call_import.tags = _resolve_tags(db, organization_id, tag_ids) - - if "schema_id" in body and body["schema_id"] is not None: - if call_import.status not in ( - CallImportStatus.UPLOADED, - CallImportStatus.MAPPED, - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Cannot reassign schema on a batch in status " - f"'{call_import.status.value}'." - ), - ) - new_schema = _resolve_schema( - db, organization_id, workspace_id, body["schema_id"] - ) - if call_import.schema_id != new_schema.id: - # Switching schemas invalidates the persisted mapping — - # parameter names won't line up with the new schema, so - # reset to UPLOADED and force a fresh MAP. - call_import.schema_id = new_schema.id - call_import.parameter_mapping = {} - call_import.skipped_columns = [] - call_import.sheet_name = None - call_import.status = CallImportStatus.UPLOADED - - db.commit() - db.refresh(call_import) - return _serialize_call_import(db, call_import) - - -@router.get( - "/{call_import_id}", - response_model=CallImportDetailResponse, - operation_id="getCallImportDetail", -) -async def get_call_import_detail( - call_import_id: UUID, - row_limit: int = Query(500, ge=0, le=5000), - row_offset: int = Query(0, ge=0), - q: Optional[str] = Query( - None, - description=( - "Optional case-insensitive substring filter on " - "``conversation_id``. When set, ``filtered_total_rows`` in " - "the response reflects the post-filter row count so the UI " - "can paginate against the filtered slice." - ), - ), - diarised_status: Optional[str] = Query( - None, - description=( - "Optional filter on ``CallImportRow.diarised_transcript_status``. " - "Accepts one of ``pending``, ``running``, ``completed``, " - "``failed``. When set, ``filtered_total_rows`` reflects the " - "post-filter row count (combined with the ``q`` filter when " - "both are supplied) so the UI can paginate against the same " - "slice it's displaying." - ), - pattern="^(pending|running|completed|failed)$", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportDetailResponse: - """Fetch a single import batch with a slice of its rows. - - ``row_limit=0`` is intentionally allowed so callers that only need the - batch metadata (e.g. the evaluation-detail page rendering the parent's - column mapping) can skip the rows payload entirely. - """ - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if call_import.status == CallImportStatus.PROCESSING: - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - - prior_status = call_import.status - rollup_call_import_batch_status(db, call_import) - if call_import.status != prior_status: - db.commit() - db.refresh(call_import) - - search_term = (q or "").strip() - diarised_status_filter = (diarised_status or "").strip() or None - filtered_total_rows: Optional[int] = None - has_row_filters = bool(search_term or diarised_status_filter) - - if has_row_filters: - if is_sharding_enabled(): - from app.db_sharding.scatter_gather import count_call_import_rows_filtered - - filtered_total_rows = count_call_import_rows_filtered( - db, - call_import.id, - search_term=search_term, - diarised_status_filter=diarised_status_filter, - ) - else: - rows_query = db.query(CallImportRow).filter( - CallImportRow.call_import_id == call_import.id - ) - if search_term: - rows_query = rows_query.filter( - CallImportRow.conversation_id.ilike(f"%{search_term}%") - ) - if diarised_status_filter: - rows_query = rows_query.filter( - CallImportRow.diarised_transcript_status == diarised_status_filter - ) - filtered_total_rows = rows_query.count() - - if row_limit == 0: - rows: List[CallImportRow] = [] - elif is_sharding_enabled(): - from app.db_sharding.scatter_gather import ( - fetch_call_import_rows_filtered_page, - fetch_call_import_rows_page, - ) - - if has_row_filters: - rows = fetch_call_import_rows_filtered_page( - db, - call_import.id, - search_term=search_term, - diarised_status_filter=diarised_status_filter, - offset=row_offset, - limit=row_limit, - ) - else: - rows = fetch_call_import_rows_page( - db, - call_import.id, - offset=row_offset, - limit=row_limit, - ) - else: - rows_query = db.query(CallImportRow).filter( - CallImportRow.call_import_id == call_import.id - ) - if search_term: - rows_query = rows_query.filter( - CallImportRow.conversation_id.ilike(f"%{search_term}%") - ) - if diarised_status_filter: - rows_query = rows_query.filter( - CallImportRow.diarised_transcript_status == diarised_status_filter - ) - rows = ( - rows_query.order_by(CallImportRow.row_index) - .offset(row_offset) - .limit(row_limit) - .all() - ) - - # Batch-wide diarisation status aggregate. One ``GROUP BY`` query - # across the whole batch — much cheaper than paging through every - # row to recount on the client and lets the UI render a - # transcribe/diarise progress bar without a separate roundtrip. - if is_sharding_enabled(): - from app.db_sharding.scatter_gather import aggregate_diarised_transcript_counts - - diarised_status_counts = aggregate_diarised_transcript_counts( - db, call_import.id - ) - else: - diarised_status_counts: Dict[str, int] = {} - for status_value, count in ( - db.query(CallImportRow.diarised_transcript_status, func.count()) - .filter(CallImportRow.call_import_id == call_import.id) - .group_by(CallImportRow.diarised_transcript_status) - .all() - ): - if isinstance(status_value, str): - diarised_status_counts[status_value] = int(count or 0) - - detail = CallImportDetailResponse.model_validate( - _serialize_call_import(db, call_import).model_dump() - ) - detail.rows = [CallImportRowResponse.model_validate(r) for r in rows] - detail.filtered_total_rows = filtered_total_rows - detail.diarised_pending_rows = diarised_status_counts.get("pending", 0) - detail.diarised_running_rows = diarised_status_counts.get("running", 0) - detail.diarised_completed_rows = diarised_status_counts.get("completed", 0) - detail.diarised_failed_rows = diarised_status_counts.get("failed", 0) - return detail - - -@router.get( - "/{call_import_id}/row-ids", - response_model=CallImportRowIdsResponse, - operation_id="listCallImportRowIds", -) -async def list_call_import_row_ids( - call_import_id: UUID, - q: Optional[str] = Query( - None, - description=( - "Optional case-insensitive substring filter on " - "``conversation_id``. Same semantics as the detail endpoint." - ), - ), - diarised_status: Optional[str] = Query( - None, - description=( - "Optional filter on ``CallImportRow.diarised_transcript_status``. " - "Accepts ``pending`` / ``running`` / ``completed`` / ``failed``." - ), - pattern="^(pending|running|completed|failed)$", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportRowIdsResponse: - """Return every matching ``CallImportRow.id`` for cross-page bulk select. - - Lightweight companion to ``GET /{call_import_id}`` — the detail - endpoint caps ``row_limit`` at 5000 and ships the entire row body - on each page, so harvesting ids that way is wasteful when the - user just wants to bulk-delete or bulk-transcribe everything that - matches the current filters. This endpoint applies the same ``q`` - and ``diarised_status`` filters and returns only the ids. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - search_term = (q or "").strip() - status_filter = (diarised_status or "").strip() or None - - if is_sharding_enabled(): - from app.db_sharding.scatter_gather import list_call_import_row_ids_filtered - - ids = list_call_import_row_ids_filtered( - db, - call_import.id, - search_term=search_term, - diarised_status_filter=status_filter, - ) - return CallImportRowIdsResponse(ids=ids, total=len(ids)) - - rows_query = db.query(CallImportRow.id).filter( - CallImportRow.call_import_id == call_import.id - ) - if search_term: - rows_query = rows_query.filter( - CallImportRow.conversation_id.ilike(f"%{search_term}%") - ) - if status_filter: - rows_query = rows_query.filter( - CallImportRow.diarised_transcript_status == status_filter - ) - - ids = [ - row_id - for (row_id,) in rows_query.order_by(CallImportRow.row_index).all() - ] - return CallImportRowIdsResponse(ids=ids, total=len(ids)) - - -def _revoke_pending_tasks(rows: List[CallImportRow]) -> None: - """Best-effort revoke of in-flight Celery tasks for the given rows. - - Failures are logged and swallowed — Celery's control plane is async and - best-effort by design, and we always do an idempotent S3 cleanup - afterwards so a missed revoke can't leak storage. - """ - task_ids = [ - r.celery_task_id - for r in rows - if r.celery_task_id - and r.status in (CallImportRowStatus.PENDING, CallImportRowStatus.PROCESSING) - ] - if not task_ids: - return - - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(task_ids, terminate=False) - logger.info("Revoked {} pending call-import tasks", len(task_ids)) - except Exception as exc: # noqa: BLE001 - logger.warning("Failed to revoke pending call-import tasks: {}", exc) - - -def _delete_s3_objects( - organization_id: UUID, - call_import_id: UUID, - rows: List[CallImportRow], -) -> tuple[int, int]: - """Delete every recording associated with ``rows`` plus a prefix sweep. - - The prefix sweep also cleans up the staged source file written at - UPLOAD time (``…/call_imports/{id}/source.{csv,xlsx}``) — both the - per-row recording keys and the source artefact share the same - organization-scoped prefix, so a single sweep covers them all. - - Returns ``(deleted_count, error_count)``. Never raises — callers proceed - with the DB delete regardless; orphans, if any, can be cleaned up by - re-running the same delete (it's idempotent). - """ - from app.services.storage.s3_service import s3_service - - if not s3_service.is_enabled(): - return 0, 0 - - keys = [r.recording_s3_key for r in rows if r.recording_s3_key] - deleted = 0 - errors = 0 - - if keys: - try: - d, errs = s3_service.delete_keys(keys) - deleted += d - errors += len(errs) - if errs: - logger.warning( - "S3 bulk-delete reported {} errors for call_import {}", - len(errs), - call_import_id, - ) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Bulk S3 delete failed for call_import {}: {}", call_import_id, exc - ) - errors += len(keys) - - # Belt-and-braces sweep: catch anything that landed under the import's - # prefix but never made it into a row's recording_s3_key (narrow - # window where the S3 upload succeeded but the DB commit didn't). - sweep_prefix = ( - f"{s3_service.prefix}organizations/{organization_id}/" - f"call_imports/{call_import_id}/" - ) - try: - d, errs = s3_service.delete_keys_by_prefix(sweep_prefix) - deleted += d - errors += len(errs) - except Exception as exc: # noqa: BLE001 - logger.exception( - "S3 prefix sweep failed for {}: {}", sweep_prefix, exc - ) - - return deleted, errors - - -@router.delete( - "/{call_import_id}", - response_model=CallImportDeleteResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="deleteCallImport", -) -async def delete_call_import( - call_import_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportDeleteResponse: - """Delete a call-import batch asynchronously. - - Flips the batch to ``deleting`` and enqueues background teardown so - large imports (thousands of rows + S3 objects) do not block the API. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - return CallImportDeleteResponse( - id=call_import_id, - status="completed", - ) - - if call_import.status == CallImportStatus.DELETING: - return CallImportDeleteResponse( - id=call_import.id, - status="accepted", - ) - - call_import.status = CallImportStatus.DELETING - call_import.error_message = None - db.commit() - - from app.workers.tasks.call_import_bulk_ops import delete_call_import_task - - delete_call_import_task.delay( - str(call_import_id), - str(organization_id), - ) - - return CallImportDeleteResponse( - id=call_import.id, - status="accepted", - ) - - -def _locate_call_import_row_or_404( - catalog_db: Session, - *, - call_import_id: UUID, - row_id: UUID, - organization_id: UUID, -) -> Tuple[Session, CallImportRow, Optional[Session]]: - """Find a call import row on the correct DB session for mutation. - - When sharding is enabled rows live on shard databases; ``get_db`` only - opens the catalog. Returns ``(row_db, row, extra_catalog_to_close)`` - where ``extra_catalog_to_close`` is the catalog session opened by - :func:`locate_call_import_row` (distinct from the route's catalog - session) and must be closed via :func:`close_row_sessions`. - """ - from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row - - try: - row_db, located_catalog, row, _shard_id = locate_call_import_row(row_id) - except LookupError: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import row not found", - ) from None - if ( - row.call_import_id != call_import_id - or row.organization_id != organization_id - ): - close_row_sessions( - row_db, - located_catalog if located_catalog is not row_db else None, - ) - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import row not found", - ) - extra_catalog = located_catalog if located_catalog is not row_db else None - return row_db, row, extra_catalog - - -@router.delete( - "/{call_import_id}/rows/{row_id}", - status_code=status.HTTP_204_NO_CONTENT, - operation_id="deleteCallImportRow", -) -async def delete_call_import_row( - call_import_id: UUID, - row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Response: - """Delete a single CallImportRow and its S3 recording. - - The parent ``CallImport`` is left in place. After deletion we recompute - its ``total_rows`` / ``completed_rows`` / ``failed_rows`` / ``status`` - so the UI's progress bar stays consistent with reality. - """ - from app.services.storage.s3_service import s3_service - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import.id, - row_id=row_id, - organization_id=organization_id, - ) - try: - _revoke_pending_tasks([row]) - - if row.recording_s3_key and s3_service.is_enabled(): - try: - s3_service.delete_file_by_key(row.recording_s3_key) - except Exception as exc: # noqa: BLE001 — best-effort, DB is source of truth - logger.warning( - "Failed to delete S3 object {} for row {}: {}", - row.recording_s3_key, - row.id, - exc, - ) - - row_db.delete(row) - row_db.commit() - - _recompute_call_import_counters(db, call_import) - db.commit() - finally: - close_row_sessions(row_db, extra_catalog) - - logger.info( - "Deleted call_import_row {} (call_import={}, org={})", - row_id, - call_import.id, - organization_id, - ) - - return Response(status_code=status.HTTP_204_NO_CONTENT) - - -def _recompute_call_import_counters( - db: Session, call_import: CallImport -) -> None: - """Resync ``total/completed/failed_rows`` + status on the parent batch. - - Called after row-level mutations (single delete, bulk delete) so the - UI's progress bar stays consistent with the actual row set. The - rules mirror :func:`delete_call_import_row` so behavior doesn't - diverge between the per-row and bulk paths. - """ - - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - - rollup_call_import_batch_status(db, call_import) - - -@router.post( - "/{call_import_id}/retry-failed", - response_model=CallImportRetryFailedRowsResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="retryFailedCallImportRows", -) -async def retry_failed_call_import_rows( - call_import_id: UUID, - payload: Optional[CallImportRetryFailedRowsRequest] = Body(None), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportRetryFailedRowsResponse: - """Re-enqueue every failed import row in this batch. - - Useful when transient provider issues are resolved and the operator wants - a one-click "try failed downloads again" pass without re-uploading the CSV. - - Pass ``provider`` + ``telephony_integration_id`` (or both omitted for - direct-URL retry) to change how recordings are fetched on this pass. - When the body is omitted entirely, the batch keeps its existing pinned - credentials. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if payload is not None: - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - if (integration.provider or "").lower() == "exotel": - schema = _resolve_schema( - db, - organization_id, - call_import.workspace_id, - call_import.schema_id, - ) - _validate_exotel_import_ready( - list(schema.parameters), - dict(call_import.parameter_mapping or {}), - ) - else: - call_import.provider = None - call_import.telephony_integration_id = None - db.flush() - - failed_rows = ( - db.query(CallImportRow) - .filter( - CallImportRow.call_import_id == call_import.id, - CallImportRow.status == CallImportRowStatus.FAILED, - ) - .order_by(CallImportRow.row_index.asc()) - .all() - ) - if not failed_rows: - return CallImportRetryFailedRowsResponse( - requeued=0, - enqueue_failed=0, - skipped=0, - ) - - from app.workers.concurrency.fair_import_dispatch import ( - schedule_fair_import_dispatch, - ) - - # Reset rows to pending BEFORE enqueue so the UI reflects "retry in - # progress" immediately even if the worker queue is backlogged. - for row in failed_rows: - row.status = CallImportRowStatus.PENDING - row.error_message = None - row.celery_task_id = None - - db.flush() - _recompute_call_import_counters(db, call_import) - db.commit() - - try: - schedule_fair_import_dispatch(max_workspace_turns=999) - requeued = len(failed_rows) - enqueue_failed = 0 - skipped = 0 - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to schedule fair import dispatch for import {}", - call_import.id, - ) - requeued = 0 - enqueue_failed = len(failed_rows) - skipped = 0 - for row in failed_rows: - db.refresh(row) - if row.status != CallImportRowStatus.PENDING: - skipped += 1 - enqueue_failed -= 1 - continue - row.status = CallImportRowStatus.FAILED - row.error_message = f"Failed to enqueue retry: {exc}" - db.flush() - _recompute_call_import_counters(db, call_import) - db.commit() - - return CallImportRetryFailedRowsResponse( - requeued=requeued, - enqueue_failed=enqueue_failed, - skipped=skipped, - ) - - -@router.post( - "/{call_import_id}/rows/bulk-delete", - response_model=CallImportRowBulkDeleteResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="bulkDeleteCallImportRows", -) -async def bulk_delete_call_import_rows( - call_import_id: UUID, - payload: CallImportRowBulkDelete, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportRowBulkDeleteResponse: - """Delete multiple ``CallImportRow`` rows in one request. - - Unknown / cross-tenant row ids are silently skipped — the response - reports how many actually went away so a UI that holds onto stale - ids (e.g. after another tab already deleted a row) doesn't 404 - the entire bulk action. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if not payload.row_ids: - return CallImportRowBulkDeleteResponse(deleted=0, status="completed") - - from app.workers.tasks.call_import_bulk_ops import bulk_delete_call_import_rows_task - - row_id_strs = [str(rid) for rid in payload.row_ids] - - bulk_delete_call_import_rows_task.delay( - str(call_import_id), - str(organization_id), - row_id_strs, - ) - - return CallImportRowBulkDeleteResponse(deleted=0, status="accepted") - - -# --------------------------------------------------------------------------- -# Diarization / transcription endpoints -# --------------------------------------------------------------------------- - - -def _select_rows_for_transcription( - db: Session, - call_import: CallImport, - payload: CallImportTranscribeRequest, - requested_row_ids: Optional[List[UUID]] = None, -) -> tuple[List[CallImportRow], Dict[str, int]]: - """Pick which rows to enqueue for diarisation (delegates to bulk_ops).""" - from app.services.call_imports.bulk_ops import select_rows_for_transcription - - try: - return select_rows_for_transcription( - db, call_import, payload, requested_row_ids=requested_row_ids - ) - except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(exc), - ) from exc - - -@router.post( - "/{call_import_id}/transcribe", - response_model=CallImportTranscribeResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="transcribeCallImport", -) -async def transcribe_call_import( - call_import_id: UUID, - payload: CallImportTranscribeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportTranscribeResponse: - """Fan out diarization tasks for many rows in a single call. - - Returns a summary with how many rows were queued and how many were - skipped (broken down by reason) so the UI can show a meaningful - toast even when nothing actually got enqueued (e.g. "All 12 rows - already have transcripts"). - """ - - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.workers.tasks.call_import_bulk_ops import bulk_diarize_call_import_task - - bulk_diarize_call_import_task.delay( - str(call_import_id), - str(organization_id), - payload.model_dump(mode="json"), - [str(rid) for rid in payload.row_ids] if payload.row_ids else None, - ) - - return CallImportTranscribeResponse( - queued=0, - skipped_rows=0, - skipped_reason_counts={}, - accepted=True, - ) - - -@router.post( - "/{call_import_id}/rows/{row_id}/transcribe", - response_model=CallImportTranscribeResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="transcribeCallImportRow", -) -async def transcribe_call_import_row( - call_import_id: UUID, - row_id: UUID, - payload: CallImportTranscribeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportTranscribeResponse: - """Diarize / transcribe a single row. - - Thin wrapper over the batch endpoint that hard-codes a single - ``row_ids`` filter. Skip counts still surface so the UI can render - "Skipped — transcript present" diagnostics consistently. - """ - - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.services.call_imports.bulk_ops import execute_bulk_diarization - - try: - result = execute_bulk_diarization( - db, - call_import, - payload, - requested_row_ids=[row_id], - ) - except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(exc), - ) from exc - - return CallImportTranscribeResponse( - queued=result.queued, - skipped_rows=result.skipped_rows, - skipped_reason_counts=result.skipped_reason_counts, - ) - - -# --------------------------------------------------------------------------- -# Cancel-in-flight diarisation -# --------------------------------------------------------------------------- -# -# Long-running multimodal LLM diarisation calls (especially LLM-only mode on -# slow audio) can sit in ``pending`` / ``running`` for tens of minutes when an -# upstream provider stalls. Without an abort affordance the operator's only -# recourse is to wait for Celery's ``time_limit`` to fire — which can be -# several minutes — or to manually mutate the DB. These helpers + the two -# endpoints below give the UI a first-class "Stop diarisation" button. -# -# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` helper uses -# ``terminate=False`` because it's called from delete-flow paths where the -# task may simply not get to run (a worker pulls it off the queue and drops -# it). For a user-initiated cancel we want SIGTERM to interrupt the worker -# mid-LLM call so the audio HTTP request actually aborts. ``terminate=True`` -# routes SIGTERM to the executing process; ``signal="SIGTERM"`` is the -# default but we spell it out so the intent is obvious to reviewers. - -# Sentinel error message stamped on cancelled rows. Read by the transcribe -# worker's finaliser (see ``app/workers/tasks/transcribe_call_import_row.py``) -# to detect a row that was cancelled mid-flight and AVOID overwriting it -# with whatever partial result the worker had managed to compute before the -# SIGTERM landed. -CANCELLED_BY_USER_ERROR: str = "Diarisation cancelled by user" - - -def _cancellable_diarisation_states() -> Tuple[str, ...]: - """States that a diarisation row can be cancelled from. - - Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` - state only needs one edit. - """ - return ("pending", "running") - - -def _revoke_diarisation_task(row: CallImportRow) -> None: - """Best-effort revoke of a single row's diarisation Celery task. - - Always swallows control-plane exceptions — Celery's control bus is - inherently best-effort and a missed revoke is not catastrophic - because the DB row is already flipped to ``failed`` by the caller - before this runs (so the UI immediately reflects the cancel; if - the task happens to finish anyway, the worker's finaliser skips - over the row via :data:`CANCELLED_BY_USER_ERROR`). - """ - task_id = (row.celery_task_id or "").strip() - if not task_id: - return - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke( - task_id, terminate=True, signal="SIGTERM" - ) - logger.info( - "Revoked diarisation task {} for call-import row {}", - task_id, - row.id, - ) - except Exception as exc: # noqa: BLE001 — revoke is best-effort - logger.warning( - "Failed to revoke diarisation task {} for row {}: {}", - task_id, - row.id, - exc, - ) - - -def _apply_diarisation_cancel(rows: List[CallImportRow]) -> Tuple[int, int]: - """Cancel diarisation on every cancellable row in ``rows``. - - Returns ``(cancelled, skipped)`` so the caller can build a typed - response without re-querying the DB. The caller is responsible for - ``db.commit()`` after this returns — we deliberately don't commit - here so a batch endpoint can flush all rows in one transaction. - """ - cancellable_states = _cancellable_diarisation_states() - cancelled = 0 - skipped = 0 - for row in rows: - if (row.diarised_transcript_status or "").lower() not in cancellable_states: - skipped += 1 - continue - # Flip the row state BEFORE we revoke so the UI's next poll - # already shows the cancel, even if Celery's control plane is - # slow to ack. - row.diarised_transcript_status = "failed" - row.diarised_transcript_error = CANCELLED_BY_USER_ERROR - _revoke_diarisation_task(row) - # Drop the task id so a follow-up retry (or a stale poll) can't - # accidentally re-revoke or get confused. - row.celery_task_id = None - cancelled += 1 - return cancelled, skipped - - -@router.post( - "/{call_import_id}/rows/{row_id}/cancel-diarisation", - response_model=CallImportRowResponse, - status_code=status.HTTP_200_OK, - operation_id="cancelCallImportRowDiarisation", -) -async def cancel_call_import_row_diarisation( - call_import_id: UUID, - row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportRowResponse: - """Abort an in-flight (or queued) diarisation for a single row. - - Idempotent: calling on a row that's already terminal (``completed`` - / ``failed`` / ``idle``) returns the row unchanged with a 200, so - the UI can fire this from a "Stop" button without having to - pre-check the state. - - Race notes: - - * The row's ``diarised_transcript_status`` is flipped to ``failed`` - with :data:`CANCELLED_BY_USER_ERROR` BEFORE the Celery revoke, - so the polling UI sees the cancel immediately. - * If the worker happens to finish between our DB flip and the - SIGTERM landing, its finaliser will detect the cancelled - sentinel on the row and skip its own status / score writes - (see :mod:`app.workers.tasks.transcribe_call_import_row`). - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import_id, - row_id=row_id, - organization_id=organization_id, - ) - try: - _apply_diarisation_cancel([row]) - row_db.commit() - row_db.refresh(row) - return CallImportRowResponse.model_validate(row) - finally: - close_row_sessions(row_db, extra_catalog) - - -@router.post( - "/{call_import_id}/cancel-diarisation", - response_model=CallImportCancelDiarisationResponse, - status_code=status.HTTP_200_OK, - operation_id="cancelCallImportDiarisation", -) -async def cancel_call_import_diarisation( - call_import_id: UUID, - payload: Optional[CallImportCancelDiarisationRequest] = None, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportCancelDiarisationResponse: - """Abort in-flight diarisation for many rows in a single call. - - Default body (no ``row_ids``) cancels every row in this import - whose ``diarised_transcript_status`` is ``pending`` or - ``running`` — the "stop everything" button. Pass ``row_ids`` to - scope the cancel to the rows the operator has selected. - - Returns ``(cancelled, skipped)`` so the UI can render a tight - toast ("Cancelled 3 rows · 1 skipped (already completed)"). - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - base_query = db.query(CallImportRow).filter( - CallImportRow.call_import_id == call_import_id - ) - - requested_ids = ( - payload.row_ids if payload and payload.row_ids is not None else None - ) - if requested_ids is not None: - if not requested_ids: - # Empty list is "no rows requested" — treat as a no-op - # 200 rather than 400 so the UI can pass through an empty - # selection without a special-case. - return CallImportCancelDiarisationResponse(cancelled=0, skipped=0) - rows = base_query.filter(CallImportRow.id.in_(requested_ids)).all() - found_ids = {r.id for r in rows} - # Treat requested-but-not-found ids as ``skipped`` so the UI's - # numbers reconcile (a stale selection that includes deleted - # rows shouldn't 404 the whole call). - missing = [rid for rid in requested_ids if rid not in found_ids] - skipped_missing = len(missing) - else: - # Implicit "cancel every cancellable row in this import" path. - rows = base_query.filter( - CallImportRow.diarised_transcript_status.in_( - list(_cancellable_diarisation_states()) - ) - ).all() - skipped_missing = 0 - - cancelled, skipped = _apply_diarisation_cancel(rows) - db.commit() - return CallImportCancelDiarisationResponse( - cancelled=cancelled, - skipped=skipped + skipped_missing, - ) - - -def _render_diarised_segments_text( - segments: Optional[List[Dict[str, Any]]], - *, - swap: bool = False, -) -> str: - """Render ``CallImportRow.diarised_segments`` as ``: `` lines. - - Mirrors the worker's ``_render_turns_as_text`` (kept duplicated so - the route doesn't need to import a Celery task module just to - rebuild the rendered transcript). Only ``agent`` and ``user`` are - swapped — multi-party calls keep their ``speaker_N`` labels through - a swap so we don't silently collapse a third speaker into the user - side. - """ - if not segments: - return "" - out: List[str] = [] - for turn in segments: - if not isinstance(turn, dict): - continue - speaker = (turn.get("speaker") or "").strip() - text = (turn.get("text") or "").strip() - if not speaker or not text: - continue - if swap: - if speaker == "agent": - speaker = "user" - elif speaker == "user": - speaker = "agent" - out.append(f"{speaker}: {text}") - return "\n".join(out) - - -@router.post( - "/{call_import_id}/rows/{row_id}/diarised-speaker-swap", - response_model=CallImportRowResponse, - operation_id="toggleCallImportRowSpeakerSwap", -) -async def toggle_call_import_row_speaker_swap( - call_import_id: UUID, - row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportRowResponse: - """Flip the user <-> agent mapping on a diarised row. - - The worker's "first speaker is the agent" heuristic is right most of - the time but does fail on inbound recordings where the customer - greets first, on recordings where the agent stays silent for the - intro, etc. Rather than rerun the (expensive) STT + pyannote - pipeline for those cases, we let reviewers flip the mapping in - place: the structured ``diarised_segments`` are the source of truth - and we re-render the plain-text ``diarised_transcript`` from them - with the swap applied. The next CSV export will then show the - corrected labels. - - Returns the updated row so the frontend can refresh without an - extra round-trip. - """ - - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import_id, - row_id=row_id, - organization_id=organization_id, - ) - try: - segments = ( - row.diarised_segments if isinstance(row.diarised_segments, list) else None - ) - if not segments: - # Without structured turns the swap toggle would have nothing to - # re-render — surface a clear error rather than silently - # flipping a flag the UI never read. - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This row has no structured diarised segments to swap. " - "Re-run diarisation to generate per-speaker turns first." - ), - ) - - new_swap = not bool(row.diarised_speaker_swap) - row.diarised_speaker_swap = new_swap - row.diarised_transcript = ( - _render_diarised_segments_text(segments, swap=new_swap) or None - ) - row_db.commit() - row_db.refresh(row) - return CallImportRowResponse.model_validate(row) - finally: - close_row_sessions(row_db, extra_catalog) - - -# --------------------------------------------------------------------------- -# Cross-run insights for the import detail page -# --------------------------------------------------------------------------- - - -@router.get( - "/{call_import_id}/insights", - response_model=CallImportInsightsResponse, - operation_id="getCallImportInsights", -) -async def get_call_import_insights( - call_import_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportInsightsResponse: - """Aggregate signals across every evaluation run on this import. - - Powers the Insights tab on the call-import detail page: returns - per-metric "latest run" summaries plus a trend series of mean values - across runs so the UI can render a small line chart per metric. Also - bundles transcript coverage stats since those are the cheapest - pre-eval health-check (e.g. "30 of 50 rows still missing - transcripts"). - """ - - del api_key - - from app.models.database import ( - CallImportEvaluation, - CallImportEvaluationRow, - Metric, - ) - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - rows = ( - db.query(CallImportRow) - .filter(CallImportRow.call_import_id == call_import_id) - .all() - ) - # A row "has a transcript" if EITHER the production (CSV) or the - # diarised (worker) column is populated — the insights tile reports - # the union so users see total coverage regardless of which source - # produced the value. - rows_with_transcript = sum( - 1 - for r in rows - if (r.transcript or "").strip() - or (r.diarised_transcript or "").strip() - ) - rows_without_transcript = len(rows) - rows_with_transcript - source_counts: Dict[str, int] = {} - for r in rows: - has_production = bool((r.transcript or "").strip()) - has_diarised = bool((r.diarised_transcript or "").strip()) - if has_production: - key = r.transcript_source or "csv" - source_counts[key] = source_counts.get(key, 0) + 1 - if has_diarised: - source_counts["diarised"] = source_counts.get("diarised", 0) + 1 - - evaluations = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .order_by(CallImportEvaluation.created_at.asc()) - .all() - ) - - # Defer heavy lifting to the aggregation helper so this endpoint and - # the per-run aggregate endpoint share the exact same metric - # bucketing math (no chance of "trend" disagreeing with "latest" on - # the same data set). - from app.api.v1.routes.call_import_evaluations import ( - _compute_metric_aggregates, - ) - - metric_history: Dict[str, List[CallImportInsightsRunPoint]] = {} - metric_meta: Dict[str, Metric] = {} - metric_latest: Dict[str, CallImportMetricAggregate] = {} - - for evaluation in evaluations: - eval_rows = ( - db.query(CallImportEvaluationRow) - .filter(CallImportEvaluationRow.evaluation_id == evaluation.id) - .all() - ) - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - for agg in aggregates: - if agg.metric_id not in metric_meta: - # ``agg.metric_id`` is normally a UUID string, but the - # aggregator also emits ids that surface in row scores - # without a matching ``Metric`` row (e.g. a metric the - # user deleted mid-run, or LLM-discovered slugs). Those - # are not valid UUIDs, so coerce defensively and skip - # the metric registry lookup when the cast fails — the - # ``meta is None`` branch below already handles the - # display via the values stored on ``agg`` itself. - try: - metric_uuid = UUID(agg.metric_id) - except (ValueError, AttributeError, TypeError): - metric_uuid = None - if metric_uuid is not None: - metric_obj = ( - db.query(Metric) - .filter( - Metric.id == metric_uuid, - Metric.organization_id == organization_id, - ) - .first() - ) - if metric_obj is not None: - metric_meta[agg.metric_id] = metric_obj - history = metric_history.setdefault(agg.metric_id, []) - history.append( - CallImportInsightsRunPoint( - evaluation_id=evaluation.id, - name=evaluation.name, - created_at=evaluation.created_at, - mean=agg.mean, - completed_rows=agg.count, - ) - ) - metric_latest[agg.metric_id] = agg - - metrics_payload: List[CallImportInsightsMetric] = [] - for metric_id, latest in metric_latest.items(): - meta = metric_meta.get(metric_id) - metrics_payload.append( - CallImportInsightsMetric( - metric_id=metric_id, - metric_name=(meta.name if meta else latest.metric_name), - metric_type=(meta.metric_type if meta else latest.metric_type), - latest=latest, - trend=metric_history.get(metric_id, []), - ) - ) - - return CallImportInsightsResponse( - call_import_id=call_import_id, - total_rows=len(rows), - rows_with_transcript=rows_with_transcript, - rows_without_transcript=rows_without_transcript, - transcript_source_counts=source_counts, - evaluation_count=len(evaluations), - metrics=metrics_payload, - ) - - -from app.core.auth.capabilities import CALLS_DELETE, CALLS_IMPORT, CALLS_VIEW -from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities - -apply_workspace_route_capabilities( - router, - view_capability=CALLS_VIEW, - manage_capability=CALLS_IMPORT, - delete_capability=CALLS_DELETE, -) +"""CSV-driven call import routes. + +Users upload a CSV plus a per-batch column mapping (CSV header -> system +field). The backend persists a CallImport batch + one CallImportRow per +line, then fans the rows out to the Celery ``imports`` queue where each +row is downloaded using the telephony credential pinned on the batch. +Exotel credentialed imports require a ``recording_url`` on every row; +direct-URL imports (no credential) also require a mapped recording URL. +""" + +from __future__ import annotations + +import csv +import io +import json +import re +from dataclasses import dataclass, field +from datetime import date, datetime, time, timedelta +from typing import Any, Dict, Iterable, List, Optional, Tuple +from uuid import UUID, uuid4 + +from fastapi import APIRouter, Body, BackgroundTasks, Depends, File, Form, HTTPException, Query, Response, UploadFile, status +from loguru import logger +from sqlalchemy import desc, func, or_ +from sqlalchemy.orm import Session + +from app.config import settings +from app.core.auth import Principal, get_principal +from app.core.auth.rbac import require_admin +from app.database import get_db +from app.db_sharding.sessions import is_sharding_enabled +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.billing.flexprice_service import record_call_import_batch_created +from app.services.call_imports.audit import ( + actor_emails_for_call_import, + emails_for_user_ids, + stamp_call_import_actor, + user_ids_from_call_imports, +) +from app.services.call_imports.dispatch_diagnostics import ( + build_call_import_dispatch_diagnostics, +) +from app.models.database import ( + CallImport, + CallImportRow, + CallImportSchema, + CallImportSchemaParameter, + CallImportTag, + TelephonyIntegration, +) +from app.models.enums import ( + CallImportParameterType, + CallImportRowStatus, + CallImportStatus, +) +from app.models.schemas import ( + CallImportCancelDiarisationRequest, + CallImportCancelDiarisationResponse, + CallImportDetailResponse, + CallImportDeleteResponse, + CallImportDiarisationPromptDefaultResponse, + CallImportDispatchDiagnosticsResponse, + CallImportInsightsMetric, + CallImportInsightsResponse, + CallImportInsightsRunPoint, + CallImportListResponse, + CallImportMappingUpdate, + CallImportMetricAggregate, + CallImportPreviewResponse, + CallImportPreviewSheet, + CallImportRetryFailedRowsRequest, + CallImportRetryFailedRowsResponse, + CallImportResponse, + CallImportRowIdsResponse, + CallImportRowBulkDelete, + CallImportRowBulkDeleteResponse, + CallImportRowResponse, + CallImportStartRequest, + CallImportTranscribeRequest, + CallImportTranscribeResponse, + CallImportUpdate, + CallImportUploadResponse, +) + + +router = APIRouter( + prefix="/call-imports", + tags=["Call Imports"], + dependencies=[Depends(require_enterprise_feature("call_imports"))], +) + + +@dataclass(frozen=True) +class CallImportParseSkip: + """One source row excluded during CSV/Excel parse (identity / recording URL).""" + + source_row: int + reason: str + message: str + + +@dataclass +class CallImportParseResult: + rows: List[Dict[str, Any]] = field(default_factory=list) + skipped: List[CallImportParseSkip] = field(default_factory=list) + + +def parse_skips_to_json(skips: List[CallImportParseSkip]) -> List[Dict[str, Any]]: + """Persistable JSON shape for ``CallImport.source_row_skips``.""" + return [ + { + "source_row": item.source_row, + "reason": item.reason, + "message": item.message, + } + for item in skips + ] + + +def _normalize_dataset(raw: Optional[str]) -> Optional[str]: + """Trim and treat empty strings as 'no dataset' (NULL).""" + if raw is None: + return None + cleaned = raw.strip() + return cleaned or None + + +def _serialize_call_import( + db: Session, + call_import: CallImport, + *, + user_emails: Optional[Dict[UUID, str]] = None, +) -> CallImportResponse: + """Catalog parent fields; counters come from SQL rollup (not Redis merge).""" + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + from app.services.call_imports.progress_counters import ( + clear_import_progress_redis, + read_import_progress, + ) + + redis_completed, redis_failed = read_import_progress(call_import.id) + if ( + redis_completed + or redis_failed + or int(call_import.completed_rows or 0) > int(call_import.total_rows or 0) + or int(call_import.failed_rows or 0) > int(call_import.total_rows or 0) + ): + rollup_call_import_batch_status(db, call_import) + db.flush() + + clear_import_progress_redis(call_import.id) + db.refresh(call_import) + total = int(call_import.total_rows or 0) + completed = min(int(call_import.completed_rows or 0), total) if total else int( + call_import.completed_rows or 0 + ) + failed = min(int(call_import.failed_rows or 0), total) if total else int( + call_import.failed_rows or 0 + ) + if user_emails is None: + user_emails = emails_for_user_ids( + db, user_ids_from_call_imports([call_import]) + ) + created_email, updated_email = actor_emails_for_call_import( + call_import, user_emails + ) + base = CallImportResponse.model_validate(call_import) + return base.model_copy( + update={ + "completed_rows": completed, + "failed_rows": failed, + "created_by_email": created_email, + "last_updated_by_email": updated_email, + } + ) + + +def _resolve_tags( + db: Session, organization_id: UUID, tag_ids: Optional[List[UUID]] +) -> List[CallImportTag]: + """Look up tag rows by id, scoped to the organization. + + Raises HTTPException(400) if any id is unknown for the org. + """ + if not tag_ids: + return [] + rows = ( + db.query(CallImportTag) + .filter( + CallImportTag.organization_id == organization_id, + CallImportTag.id.in_(tag_ids), + ) + .all() + ) + found_ids = {row.id for row in rows} + missing = [str(tag_id) for tag_id in tag_ids if tag_id not in found_ids] + if missing: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unknown call_import_tag id(s): {missing}", + ) + return rows + + +MAX_UPLOAD_BYTES = 15 * 1024 * 1024 # 15 MB upload cap (CSV or Excel) + +# File extensions accepted by the upload + preview endpoints. Keep in +# lockstep with the frontend ``accept`` attribute on the file picker. +CSV_EXTENSIONS = (".csv",) +XLSX_EXTENSIONS = (".xlsx", ".xlsm") +ALLOWED_EXTENSIONS = CSV_EXTENSIONS + XLSX_EXTENSIONS + +AUDIO_CONTENT_TYPES = { + "wav": "audio/wav", + "mp3": "audio/mpeg", + "flac": "audio/flac", + "m4a": "audio/mp4", +} + + +def _file_format(filename: Optional[str]) -> Optional[str]: + """Classify ``filename`` as ``'csv'`` / ``'xlsx'`` or ``None`` if unsupported.""" + if not filename: + return None + name = filename.lower() + if name.endswith(CSV_EXTENSIONS): + return "csv" + if name.endswith(XLSX_EXTENSIONS): + return "xlsx" + return None + + +def _audio_extension(filename: Optional[str]) -> Optional[str]: + """Return the validated lower-case extension for a manual recording.""" + if not filename or "." not in filename: + return None + ext = filename.rsplit(".", 1)[-1].lower().strip() + allowed = {fmt.lower().lstrip(".") for fmt in settings.ALLOWED_AUDIO_FORMATS} + return ext if ext in allowed else None + + +def _audio_content_type(ext: str, upload_content_type: Optional[str]) -> str: + """Prefer the browser-supplied audio content type, with a safe fallback.""" + supplied = (upload_content_type or "").strip() + if supplied and supplied != "application/octet-stream": + return supplied + return AUDIO_CONTENT_TYPES.get(ext.lower(), "application/octet-stream") + + +def _audio_s3_key( + organization_id: UUID, call_import_id: UUID, row_id: UUID, ext: str +) -> str: + """Build the canonical S3 key for a manually uploaded recording.""" + from app.services.storage.s3_service import s3_service + + return ( + f"{s3_service.prefix}organizations/{organization_id}/call_imports/" + f"{call_import_id}/{row_id}.{ext}" + ) + + +def _filename_stem(filename: Optional[str]) -> str: + """Extract a cross-platform filename stem from an UploadFile name.""" + raw = (filename or "").strip() + basename = re.split(r"[\\/]", raw)[-1] if raw else "" + if "." in basename: + basename = basename.rsplit(".", 1)[0] + return basename.strip() + + +def _sanitize_conversation_id(raw: str) -> str: + """Turn a filename stem into a stable conversation_id.""" + cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", raw.strip()) + cleaned = re.sub(r"_+", "_", cleaned).strip("._-") + return (cleaned or "recording")[:255] + + +def _dedupe_conversation_id( + base: str, counts: Dict[str, int] +) -> str: + """Make conversation ids unique within one manual upload batch.""" + count = counts.get(base, 0) + 1 + counts[base] = count + if count == 1: + return base + suffix = f"-{count}" + return f"{base[: 255 - len(suffix)]}{suffix}" + + +def _normalize_header(name: str) -> str: + return (name or "").strip().lower() + + +def _header_lookup(fieldnames: List[str]) -> Dict[str, str]: + """Map normalized header -> original header for case-insensitive lookup.""" + return {_normalize_header(h): h for h in fieldnames or []} + + +def _resolve_mapped_header( + mapping_value: Optional[str], header_lookup: Dict[str, str] +) -> Optional[str]: + """Translate a user-supplied CSV header into the actual column key. + + The frontend sends headers exactly as they appear in the source file, + but we still normalize on the server so trailing whitespace / casing + doesn't break matching. Returns the canonical fieldname or ``None`` + if not present in the file. + """ + if not mapping_value: + return None + return header_lookup.get(_normalize_header(mapping_value)) + + +def _xlsx_cell_to_str(value: Any) -> str: + """Coerce an openpyxl cell value to the string the rest of the + pipeline expects. + + openpyxl returns native Python types (int, float, datetime, bool, + None). The CSV path always works with strings, so we mirror that: + integers stringify cleanly (no ``.0`` suffix on whole-number floats), + datetimes use ISO-8601, booleans use SQL-style ``TRUE`` / ``FALSE``. + """ + if value is None: + return "" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + if value.is_integer(): + return str(int(value)) + return str(value) + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, date): + return value.isoformat() + if isinstance(value, time): + return value.isoformat() + if isinstance(value, timedelta): + return str(value) + return str(value) + + +def _parse_recording_date_cell(cell: str) -> date: + """Parse day-first dates with one/two digit day-month parts.""" + match = re.fullmatch(r"\s*(\d{1,2})[/-](\d{1,2})[/-](\d{4})\s*", cell) + if match: + day, month, year = (int(part) for part in match.groups()) + return date(year, month, day) + + # Native Excel date cells arrive from ``_xlsx_cell_to_str`` as ISO + # datetimes (e.g. ``2026-01-04T00:00:00``). Accept that resolved date, + # while keeping plain ISO dates rejected for hand-entered text/CSV cells. + if "T" in cell: + return datetime.fromisoformat(cell.replace("Z", "+00:00")).date() + + raise ValueError("expected D/M/YYYY or D-M-YYYY") + + +def _coerce_parameter_value( + raw: str, + param_type: CallImportParameterType, + *, + row_idx: int, + param_name: str, +) -> Any: + """Validate + coerce a single CSV cell against its declared type. + + Returns the typed Python value to surface in ``raw_columns``. Empty + strings are returned as ``None`` regardless of the parameter type so + optional cells stay null end-to-end. Coercion failures raise a + 400 with a row-anchored message. + """ + cell = (raw or "").strip() + if not cell: + return None + + if param_type == CallImportParameterType.CONVERSATION_ID: + return cell + if param_type == CallImportParameterType.RECORDING_URL: + # Recording URLs are exercised by the worker (which downloads + # them); we only do a light "starts with http" check here so a + # paste-error surfaces immediately at upload time. + lower = cell.lower() + if not (lower.startswith("http://") or lower.startswith("https://")): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + "valid recording URL (must start with http:// or https://)." + ), + ) + return cell + if param_type == CallImportParameterType.RECORDING_DATE: + try: + parsed_date = _parse_recording_date_cell(cell) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid recording date ({cell!r}); expected day-first " + "D/M/YYYY or D-M-YYYY." + ), + ) + return parsed_date.strftime("%d/%m/%Y") + if param_type == CallImportParameterType.TRANSCRIPT: + return cell + if param_type == CallImportParameterType.TEXT: + return cell + if param_type == CallImportParameterType.NUMBER: + try: + value = float(cell) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid number ({cell!r})." + ), + ) + if value.is_integer(): + return int(value) + return value + if param_type == CallImportParameterType.BOOLEAN: + truthy = {"true", "yes", "y", "1", "t"} + falsy = {"false", "no", "n", "0", "f"} + norm = cell.lower() + if norm in truthy: + return True + if norm in falsy: + return False + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid boolean ({cell!r})." + ), + ) + if param_type == CallImportParameterType.DATETIME: + try: + parsed = datetime.fromisoformat(cell.replace("Z", "+00:00")) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid ISO-8601 date/time ({cell!r})." + ), + ) + return parsed.isoformat() + if param_type == CallImportParameterType.URL: + lower = cell.lower() + if not (lower.startswith("http://") or lower.startswith("https://")): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + "valid URL (must start with http:// or https://)." + ), + ) + return cell + # Unknown types: store as text and let the next migration catch up. + return cell + + +def _recording_url_cell_is_valid_http(raw: str) -> bool: + cell = (raw or "").strip() + if not cell: + return False + lower = cell.lower() + return lower.startswith("http://") or lower.startswith("https://") + + +def _parameter_is_required(param: CallImportSchemaParameter) -> bool: + """Return whether a schema parameter must be mapped on every upload.""" + if param.is_required: + return True + try: + param_type = CallImportParameterType(param.type) + except ValueError: + return False + return param_type in ( + CallImportParameterType.CONVERSATION_ID, + CallImportParameterType.RECORDING_URL, + ) + + +def _apply_schema_mapping( + fieldnames: List[str], + rows_iter: Iterable[Dict[str, str]], + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, str], + skipped_columns: List[str], + *, + source_label: str = "CSV", + validate_only: bool = False, +) -> CallImportParseResult: + """Schema-driven row projection: parameter -> CSV header -> typed value. + + Validates that every required schema parameter is mapped to a CSV + header that actually exists in the file, and that every CSV header + is either mapped to a parameter or explicitly listed in + ``skipped_columns``. Returns one dict per non-empty data row with: + + * ``conversation_id`` (str, mandatory) + * ``recording_date`` (Optional[str], DD/MM/YYYY date) + * ``recording_url`` (Optional[str]) + * ``transcript`` (Optional[str]) + * ``parameter_values`` (Dict[str, Any]) of typed values keyed by + parameter name (drives ``raw_columns`` so the export can + reproduce the source). + + ``validate_only=True`` runs the header / mapping / skipped-column + checks (every check that doesn't need to read row data) and then + returns an empty list — used by the MAP stage to validate a + mapping payload against the cached sheet snapshot without + re-fetching the source bytes from S3. + """ + header_lookup = _header_lookup(list(fieldnames)) + + # 1. Look up the conversation_id parameter so we can address it + # directly while building each row. + conv_param = next( + (p for p in parameters if p.type == CallImportParameterType.CONVERSATION_ID), + None, + ) + if conv_param is None: + # The schema invariant should have caught this on create/update, + # but a defensive 400 here keeps us safe against hand-rolled + # API callers that bypassed validation. + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Selected schema is missing the mandatory conversation_id parameter.", + ) + # 2. Resolve every mapped parameter to a canonical fieldname. + # Required parameters MUST resolve; optional ones may resolve to + # None if the user left them blank (no mapping). + canonical_by_param: Dict[str, Optional[str]] = {} + recording_date_param_name: Optional[str] = None + rec_url_param_name: Optional[str] = None + transcript_param_name: Optional[str] = None + for param in parameters: + mapped_header = parameter_mapping.get(param.name) + canonical = ( + _resolve_mapped_header(mapped_header, header_lookup) + if mapped_header + else None + ) + if _parameter_is_required(param) and canonical is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"{source_label} does not contain the column " + f"'{mapped_header or ''}' mapped to required parameter " + f"'{param.name}'." + ), + ) + canonical_by_param[param.name] = canonical + if param.type == CallImportParameterType.RECORDING_DATE: + recording_date_param_name = param.name + elif param.type == CallImportParameterType.RECORDING_URL: + rec_url_param_name = param.name + elif param.type == CallImportParameterType.TRANSCRIPT: + transcript_param_name = param.name + + # 3. Every CSV column must either be mapped to a parameter or + # explicitly skipped. Catches "I forgot to skip the email + # column" gracefully instead of dropping data silently. + mapped_canonicals = {c for c in canonical_by_param.values() if c} + skipped_canonicals = { + _resolve_mapped_header(h, header_lookup) + for h in skipped_columns + } + skipped_canonicals.discard(None) + unhandled = [ + h + for h in fieldnames + if h not in mapped_canonicals and h not in skipped_canonicals + ] + if unhandled: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"{source_label} columns must either be mapped to a schema " + f"parameter or explicitly skipped. Unhandled: {unhandled}." + ), + ) + + conv_canonical = canonical_by_param[conv_param.name] + rec_canonical = ( + canonical_by_param.get(rec_url_param_name) + if rec_url_param_name + else None + ) + recording_date_canonical = ( + canonical_by_param.get(recording_date_param_name) + if recording_date_param_name + else None + ) + transcript_canonical = ( + canonical_by_param.get(transcript_param_name) + if transcript_param_name + else None + ) + + if validate_only: + # MAP-stage validation: every header check above has already + # run; the row loop only matters at IMPORT time. Skip it (and + # the "no data rows" guard at the bottom of the function) so + # the caller gets a clean pass when the mapping is shaped right. + return CallImportParseResult() + + parsed: List[Dict[str, Any]] = [] + skipped: List[CallImportParseSkip] = [] + for idx, row in enumerate(rows_iter): + # Drop fully-blank lines - matches the legacy parser behavior so + # trailing-newline edge cases don't fail an otherwise-good upload. + non_blank = any( + (row.get(c) or "").strip() + for c in mapped_canonicals + if c + ) + if not non_blank: + continue + + source_row = idx + 1 + conv_value = (row.get(conv_canonical) or "").strip() if conv_canonical else "" + if not conv_value: + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="missing_conversation_id", + message=( + f"Row {source_row} is missing the '{conv_param.name}' " + "(conversation_id) value." + ), + ) + ) + continue + + if rec_canonical and rec_url_param_name: + rec_param = next( + (p for p in parameters if p.name == rec_url_param_name), + None, + ) + if rec_param is not None and _parameter_is_required(rec_param): + rec_raw = (row.get(rec_canonical) or "").strip() + if not rec_raw: + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="missing_recording_url", + message=( + f"Row {source_row} is missing the required " + f"'{rec_url_param_name}' value." + ), + ) + ) + continue + if not _recording_url_cell_is_valid_http(rec_raw): + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="invalid_recording_url", + message=( + f"Row {source_row}: value for " + f"'{rec_url_param_name}' is not a valid recording " + "URL (must start with http:// or https://)." + ), + ) + ) + continue + + # Materialize every mapped parameter into the per-row snapshot, + # running per-type coercion so a bad cell aborts the upload + # rather than silently storing garbage. + parameter_values: Dict[str, Any] = {} + row_skipped = False + for param in parameters: + canonical = canonical_by_param[param.name] + if canonical is None: + continue + try: + param_type = CallImportParameterType(param.type) + except ValueError: + param_type = CallImportParameterType.TEXT + coerced = _coerce_parameter_value( + row.get(canonical) or "", + param_type, + row_idx=idx, + param_name=param.name, + ) + if _parameter_is_required(param) and coerced is None: + if param_type == CallImportParameterType.RECORDING_URL: + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="missing_recording_url", + message=( + f"Row {source_row} is missing the required " + f"'{param.name}' value." + ), + ) + ) + row_skipped = True + break + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {source_row} is missing the required " + f"'{param.name}' value." + ), + ) + parameter_values[param.name] = coerced + if row_skipped: + continue + + rec_value = ( + (row.get(rec_canonical) or "").strip() if rec_canonical else "" + ) + transcript_value = ( + (row.get(transcript_canonical) or "").strip() + if transcript_canonical + else "" + ) + recording_date_value = ( + parameter_values.get(recording_date_param_name) + if recording_date_param_name + else None + ) + + parsed.append( + { + "conversation_id": conv_value, + "recording_date": recording_date_value, + "recording_url": rec_value or None, + "transcript": transcript_value or None, + "parameter_values": parameter_values, + } + ) + + if not parsed and not skipped: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"{source_label} did not contain any data rows.", + ) + + return CallImportParseResult(rows=parsed, skipped=skipped) + + +def _raise_if_no_importable_rows( + result: CallImportParseResult, *, source_label: str = "CSV" +) -> None: + """Sync upload / API callers fail fast when every data row was skipped.""" + if result.rows: + return + if result.skipped: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"No importable rows. {len(result.skipped)} row(s) skipped due " + "to missing or invalid conversation ID or recording URL." + ), + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"{source_label} did not contain any data rows.", + ) + + +def _parse_csv( + file_bytes: bytes, + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, str], + skipped_columns: List[str], +) -> CallImportParseResult: + """Parse a CSV file using the resolved schema parameters.""" + if not file_bytes: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Uploaded CSV is empty.", + ) + + try: + text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) + except UnicodeDecodeError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV must be UTF-8 encoded.", + ) + + reader = csv.DictReader(text_stream) + if not reader.fieldnames: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV is missing a header row.", + ) + + return _apply_schema_mapping( + list(reader.fieldnames), + reader, + parameters, + parameter_mapping, + skipped_columns, + source_label="CSV", + ) + + +def _open_xlsx_workbook(file_bytes: bytes): + """Open an xlsx/xlsm workbook from in-memory bytes (read-only stream). + + Imports openpyxl lazily so the module loads even in environments that + haven't installed the optional dep yet (e.g. lightweight tooling + images). Surfaces a clean 400 if openpyxl is missing or the file is + not a valid Office Open XML workbook. + """ + if not file_bytes: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Uploaded Excel file is empty.", + ) + try: + from openpyxl import load_workbook # type: ignore + from openpyxl.utils.exceptions import InvalidFileException # type: ignore + except ImportError as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=( + "Excel uploads require the 'openpyxl' package which is " + "not installed in this environment." + ), + ) from exc + + try: + return load_workbook( + io.BytesIO(file_bytes), + read_only=True, + data_only=True, + ) + except InvalidFileException as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"File is not a valid .xlsx workbook: {exc}", + ) from exc + except Exception as exc: # zipfile.BadZipFile etc. + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Could not open Excel workbook: {exc}", + ) from exc + + +def _xlsx_sheet_headers_and_rows( + worksheet, +) -> Tuple[List[str], List[Dict[str, str]]]: + """Read row 1 as headers and the rest as dicts of stringified cells. + + Empty trailing header cells are dropped. Duplicate headers preserve + the first occurrence (matches ``csv.DictReader`` behavior, which + silently drops duplicates). + """ + iterator = worksheet.iter_rows(values_only=True) + try: + header_row = next(iterator) + except StopIteration: + return [], [] + + headers: List[str] = [] + seen: set[str] = set() + for cell in header_row: + name = _xlsx_cell_to_str(cell).strip() + if not name: + # Stop at the first blank header — treats trailing empty + # columns as not part of the table (matches typical Excel + # workbook conventions). + break + norm = name.lower() + if norm in seen: + continue + seen.add(norm) + headers.append(name) + + rows: List[Dict[str, str]] = [] + for row in iterator: + if row is None: + continue + # Pad / truncate to the header length so dict construction is + # stable even when a row has fewer / extra cells than the header. + cells = list(row[: len(headers)]) + if len(cells) < len(headers): + cells.extend([None] * (len(headers) - len(cells))) + if not any(_xlsx_cell_to_str(c).strip() for c in cells): + # Skip fully-blank rows (openpyxl read_only routinely yields + # trailing empties when the worksheet's used range exceeds + # the actual data). + continue + rows.append( + { + header: _xlsx_cell_to_str(value) + for header, value in zip(headers, cells) + } + ) + + return headers, rows + + +def _parse_xlsx( + file_bytes: bytes, + sheet_name: Optional[str], + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, str], + skipped_columns: List[str], +) -> CallImportParseResult: + """Parse a single worksheet from an xlsx/xlsm workbook. + + ``sheet_name`` must match one of the workbook's sheets (case + insensitive whitespace-trimmed match). Returns the same shape as + :func:`_parse_csv` so the upload handler can persist either format + through the same code path. + """ + if not sheet_name or not sheet_name.strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is required when uploading an Excel workbook.", + ) + + workbook = _open_xlsx_workbook(file_bytes) + try: + sheet_names = list(workbook.sheetnames) + target_norm = sheet_name.strip().lower() + match = next( + (s for s in sheet_names if s.strip().lower() == target_norm), + None, + ) + if match is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Sheet '{sheet_name}' not found in workbook. " + f"Available sheets: {sheet_names}" + ), + ) + worksheet = workbook[match] + headers, rows = _xlsx_sheet_headers_and_rows(worksheet) + finally: + workbook.close() + + if not headers: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Sheet '{sheet_name}' is missing a header row.", + ) + + return _apply_schema_mapping( + headers, + rows, + parameters, + parameter_mapping, + skipped_columns, + source_label=f"Sheet '{sheet_name}'", + ) + + +def _csv_preview_sheets( + file_bytes: bytes, filename: Optional[str] +) -> List[CallImportPreviewSheet]: + """Build the synthetic single-sheet preview entry for a CSV upload.""" + if not file_bytes: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Uploaded CSV is empty.", + ) + try: + text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) + except UnicodeDecodeError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV must be UTF-8 encoded.", + ) + reader = csv.DictReader(text_stream) + if not reader.fieldnames: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV is missing a header row.", + ) + headers = list(reader.fieldnames) + row_count = 0 + for row in reader: + # Match the parse-time skip: ignore fully blank rows so the + # count the user sees lines up with what /upload will ingest. + if any((v or "").strip() for v in row.values()): + row_count += 1 + + sheet_label = (filename or "sheet1").rsplit("/", 1)[-1] or "sheet1" + return [ + CallImportPreviewSheet( + name=sheet_label, + headers=headers, + row_count=row_count, + ) + ] + + +def _xlsx_preview_sheets(file_bytes: bytes) -> List[CallImportPreviewSheet]: + """List every worksheet in the workbook with its headers and row count.""" + workbook = _open_xlsx_workbook(file_bytes) + sheets: List[CallImportPreviewSheet] = [] + try: + for name in workbook.sheetnames: + worksheet = workbook[name] + headers, rows = _xlsx_sheet_headers_and_rows(worksheet) + sheets.append( + CallImportPreviewSheet( + name=name, + headers=headers, + row_count=len(rows), + ) + ) + finally: + workbook.close() + return sheets + + +def _parse_json_form_field(name: str, raw: Optional[str], default): + """Decode a JSON-encoded form field with a friendly 400 on bad JSON.""" + if raw is None or raw == "": + return default + try: + return json.loads(raw) + except json.JSONDecodeError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"{name} must be valid JSON: {exc}", + ) + + +# --------------------------------------------------------------------------- +# Shared helpers used by the staged endpoints (UPLOAD / MAP / IMPORT) and the +# legacy one-shot ``POST /upload`` shim. Extracted here so each stage and the +# back-compat path operate on the exact same validation + persistence code. +# --------------------------------------------------------------------------- + + +def _source_content_type(fmt: str) -> str: + """Return the canonical ``Content-Type`` for a parsed file format.""" + if fmt == "csv": + return "text/csv" + if fmt == "xlsx": + return ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ) + return "application/octet-stream" + + +def _source_s3_key( + organization_id: UUID, call_import_id: UUID, fmt: str +) -> str: + """Build the canonical S3 key for an upload's source file. + + Mirrors the per-row recording key convention used by + ``process_call_import_row`` so a single prefix sweep on delete still + cleans up both the source artefact and every fetched recording. + """ + from app.services.storage.s3_service import s3_service + + ext = "xlsx" if fmt == "xlsx" else "csv" + return ( + f"{s3_service.prefix}organizations/{organization_id}/call_imports/" + f"{call_import_id}/source.{ext}" + ) + + +def _build_available_sheets( + file_bytes: bytes, fmt: str, filename: Optional[str] +) -> List[CallImportPreviewSheet]: + """Snapshot of sheets + headers cached on the batch at UPLOAD time.""" + if fmt == "csv": + return _csv_preview_sheets(file_bytes, filename) + return _xlsx_preview_sheets(file_bytes) + + +def _resolve_schema( + db: Session, + organization_id: UUID, + workspace_id: UUID, + schema_id: UUID, +) -> CallImportSchema: + """Fetch + validate a schema row in the active workspace. + + Eager-loads ``parameters`` so callers can iterate without re-querying. + """ + from sqlalchemy.orm import selectinload as _selectinload + + schema = ( + db.query(CallImportSchema) + .options(_selectinload(CallImportSchema.parameters)) + .filter( + CallImportSchema.id == schema_id, + CallImportSchema.organization_id == organization_id, + CallImportSchema.workspace_id == workspace_id, + ) + .first() + ) + if not schema: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Call import schema not found in the active workspace.", + ) + if not list(schema.parameters): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Selected schema has no parameters defined.", + ) + return schema + + +def _validate_direct_url_import_ready( + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, Any], +) -> None: + """Ensure direct-URL import has a mapped recording_url column.""" + rec_url_param = next( + ( + p + for p in parameters + if p.type == CallImportParameterType.RECORDING_URL.value + ), + None, + ) + if rec_url_param is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Direct URL import requires a schema parameter of type " + "'recording_url'." + ), + ) + mapped_header = (parameter_mapping or {}).get(rec_url_param.name) + if not (mapped_header or "").strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Direct URL import requires the 'recording_url' parameter to " + "be mapped to a source column." + ), + ) + + +def _validate_exotel_import_ready( + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, Any], +) -> None: + """Ensure Exotel credentialed import has a mapped recording_url column.""" + rec_url_param = next( + ( + p + for p in parameters + if p.type == CallImportParameterType.RECORDING_URL.value + ), + None, + ) + if rec_url_param is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Exotel import requires a schema parameter of type " + "'recording_url'." + ), + ) + mapped_header = (parameter_mapping or {}).get(rec_url_param.name) + if not (mapped_header or "").strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Exotel import requires the 'recording_url' parameter to " + "be mapped to a source column." + ), + ) + + +def _resolve_telephony_integration( + db: Session, + organization_id: UUID, + telephony_integration_id: UUID, + provider: str, +) -> TelephonyIntegration: + """Fetch + validate a telephony credential against the requested provider.""" + integration = ( + db.query(TelephonyIntegration) + .filter( + TelephonyIntegration.id == telephony_integration_id, + TelephonyIntegration.organization_id == organization_id, + ) + .first() + ) + if not integration: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Telephony credential not found for this organization.", + ) + if (integration.provider or "").lower() != provider.lower(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Selected credential is for provider '{integration.provider}', " + f"but request specified '{provider}'." + ), + ) + if not integration.is_active: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Selected telephony credential is inactive.", + ) + return integration + + +def _clean_parameter_mapping( + mapping_payload: Any, + parameters: List[CallImportSchemaParameter], + schema_name: str, +) -> Dict[str, str]: + """Trim values and drop empties; reject unknown parameter names. + + Accepts an already-decoded value (dict-shaped) so the same helper + works for the JSON-form upload path and the JSON-body PATCH path. + """ + if not isinstance(mapping_payload, dict) or not all( + isinstance(k, str) and (v is None or isinstance(v, str)) + for k, v in mapping_payload.items() + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "parameter_mapping must be an object of " + "{parameter_name: csv_header}." + ), + ) + + valid_param_names = {p.name for p in parameters} + cleaned: Dict[str, str] = {} + for raw_name, raw_header in mapping_payload.items(): + if raw_name not in valid_param_names: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"parameter_mapping references unknown parameter " + f"'{raw_name}' on schema '{schema_name}'." + ), + ) + header = (raw_header or "").strip() + if header: + cleaned[raw_name] = header + return cleaned + + +def _clean_skipped_columns(skipped_payload: Any) -> List[str]: + """Dedupe (case-insensitively) and drop blanks; preserve original casing.""" + if not isinstance(skipped_payload, list) or not all( + isinstance(item, str) for item in skipped_payload + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="skipped_columns must be a list of header strings.", + ) + cleaned: List[str] = [] + seen: set[str] = set() + for item in skipped_payload: + norm = _normalize_header(item) + if not norm or norm in seen: + continue + seen.add(norm) + cleaned.append(item) + return cleaned + + +def _parse_source_file( + file_bytes: bytes, + fmt: str, + sheet_name: Optional[str], + parameters: List[CallImportSchemaParameter], + cleaned_mapping: Dict[str, str], + cleaned_skipped: List[str], +) -> CallImportParseResult: + """Run the format-appropriate parser against a buffer of file bytes.""" + if fmt == "csv": + return _parse_csv(file_bytes, parameters, cleaned_mapping, cleaned_skipped) + return _parse_xlsx( + file_bytes, sheet_name, parameters, cleaned_mapping, cleaned_skipped + ) + + +def _materialize_rows( + db: Session, + call_import: CallImport, + parsed_rows: List[Dict[str, Any]], + organization_id: UUID, +) -> List[CallImportRow]: + """Insert one ``CallImportRow`` per parsed row, returning the new models.""" + row_models: List[CallImportRow] = [] + for idx, row in enumerate(parsed_rows): + # Stamp ``transcript_source='csv'`` when the upload actually + # provided a transcript so the UI badge ("From CSV") works from + # day one. Blank cells stay NULL so the row reads as "no + # production transcript yet". + csv_transcript = row["transcript"] + row_model = CallImportRow( + call_import_id=call_import.id, + organization_id=organization_id, + workspace_id=call_import.workspace_id, + row_index=idx, + conversation_id=row["conversation_id"], + recording_date=( + _parse_recording_date_cell(row["recording_date"]) + if row.get("recording_date") + else None + ), + recording_url=row["recording_url"], + transcript=csv_transcript, + transcript_source=( + "csv" if csv_transcript and csv_transcript.strip() else None + ), + raw_columns=row["parameter_values"] or None, + status=CallImportRowStatus.PENDING, + ) + db.add(row_model) + row_models.append(row_model) + return row_models + + +def _enqueue_row_tasks( + db: Session, + call_import: CallImport, + row_models: List[CallImportRow], +) -> None: + """Schedule fair round-robin dispatch for pending import rows.""" + del db, call_import, row_models + from app.workers.concurrency.fair_import_dispatch import ( + schedule_fair_import_dispatch, + ) + + schedule_fair_import_dispatch(max_workspace_turns=999) + + +def _ensure_blob_storage_enabled() -> None: + """Hard-fail UPLOAD if cloud blob storage isn't configured (no local fallback).""" + from app.services.storage.s3_service import s3_service + + if not s3_service.is_enabled(): + err = ( + s3_service.get_status_message() + or "Cloud blob storage is not enabled or not configured" + ) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=( + "Call uploads require cloud blob storage so the file can be " + f"persisted between stages: {err}" + ), + ) + + +def _validate_sheet_choice( + fmt: str, + sheet_name: Optional[str], + available_sheets: Optional[List[Dict[str, Any]]], +) -> Optional[str]: + """Normalize / validate ``sheet_name`` against the persisted snapshot. + + Returns the canonical sheet name (matching the workbook's casing) + so downstream parsing addresses the right worksheet. + """ + if fmt == "csv": + if sheet_name and sheet_name.strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is not applicable to CSV uploads.", + ) + return None + + cleaned = (sheet_name or "").strip() or None + if cleaned is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is required when the source is an Excel workbook.", + ) + + if not available_sheets: + # Nothing to validate against (e.g. legacy batch without snapshot); + # let downstream parsing error out instead of silently importing. + return cleaned + + target = cleaned.strip().lower() + for entry in available_sheets: + name = entry.get("name") if isinstance(entry, dict) else None + if isinstance(name, str) and name.strip().lower() == target: + return name + sheet_names = [ + entry.get("name") for entry in available_sheets if isinstance(entry, dict) + ] + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Sheet '{cleaned}' not found in the staged file. " + f"Available sheets: {sheet_names}" + ), + ) + + +def _tag_response_payload(tags: Optional[List[CallImportTag]]) -> List[Dict[str, Any]]: + """Shape a CallImport's tag relationship for the upload response.""" + return [ + { + "id": tag.id, + "name": tag.name, + "color": tag.color, + "created_at": tag.created_at, + "updated_at": tag.updated_at, + } + for tag in (tags or []) + ] + + +@router.post( + "/preview", + response_model=CallImportPreviewResponse, + operation_id="previewCallImportFile", +) +async def preview_call_import_file( + file: UploadFile = File(...), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> CallImportPreviewResponse: + """Inspect an uploaded CSV / Excel file and return its sheets + headers. + + Drives the column-mapping UI without forcing the frontend to parse + CSV / xlsx itself — keeps client and server in lockstep on quoted + fields, encodings, and Excel cell coercion. CSVs return a single + synthetic sheet named after the filename; Excel workbooks return one + entry per worksheet (in workbook order). + """ + del api_key, organization_id, workspace_id, db # auth only + + fmt = _file_format(file.filename) + if fmt is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Unsupported file format. Allowed extensions: " + f"{', '.join(ALLOWED_EXTENSIONS)}." + ), + ) + + file_bytes = await file.read() + if len(file_bytes) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", + ) + + if fmt == "csv": + sheets = _csv_preview_sheets(file_bytes, file.filename) + else: + sheets = _xlsx_preview_sheets(file_bytes) + + return CallImportPreviewResponse(format=fmt, sheets=sheets) + + +@router.post( + "", + response_model=CallImportResponse, + status_code=status.HTTP_201_CREATED, + operation_id="createCallImport", +) +async def create_call_import( + file: UploadFile = File( + ..., + description="CSV / Excel file to stage. Persisted to S3 between stages.", + ), + dataset: str = Form( + ..., + description=( + "Required free-text dataset label. Collected up-front so the " + "batch is filterable from the moment it lands." + ), + ), + tag_ids: Optional[List[UUID]] = Form( + None, + description="Optional list of CallImportTag ids to attach to the new batch.", + ), + schema_id: Optional[UUID] = Form( + None, + description=( + "Optional schema pre-pick. The user can still change it during " + "the MAP stage; provided here only so the detail page can pre-" + "select the schema dropdown." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportResponse: + """UPLOAD stage of the staged call-import flow. + + Persists the source file to S3 and creates a ``CallImport`` row with + ``status='uploaded'``. No mapping, no provider, no rows yet — the + user moves through MAP and IMPORT as separate idempotent steps. + + Dataset is collected here (rather than at IMPORT) so the batch is + filterable from the moment it appears in the list view. + """ + del api_key + + fmt = _file_format(file.filename) + if fmt is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Unsupported file format. Allowed extensions: " + f"{', '.join(ALLOWED_EXTENSIONS)}." + ), + ) + + normalized_dataset = _normalize_dataset(dataset) + if not normalized_dataset: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="dataset is required and must be a non-empty string.", + ) + + file_bytes = await file.read() + if len(file_bytes) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", + ) + + # Parse-now so we (a) reject garbage uploads up-front instead of + # later in the MAP step, and (b) capture the sheets snapshot the + # MAP UI needs without having to re-fetch the file from S3. + sheets = _build_available_sheets(file_bytes, fmt, file.filename) + + # Optional schema pre-pick: validated only if supplied (the user is + # allowed to set it for the first time during MAP). + if schema_id is not None: + _resolve_schema(db, organization_id, workspace_id, schema_id) + + tag_rows = _resolve_tags(db, organization_id, tag_ids) + + _ensure_blob_storage_enabled() + + # Pre-generate the id so we can compute a deterministic S3 key + # before the row is persisted, keeping ``source_s3_key`` consistent + # with the prefix sweep used at delete-time. + import uuid as _uuid + + call_import_id = _uuid.uuid4() + s3_key = _source_s3_key(organization_id, call_import_id, fmt) + content_type = _source_content_type(fmt) + + from app.services.storage.s3_service import s3_service, StorageError + + try: + s3_service.upload_file_by_key(file_bytes, s3_key, content_type=content_type) + except StorageError as exc: + logger.exception( + "Failed to upload source file to S3 for new call import {}", + call_import_id, + ) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=f"Failed to persist upload to S3: {exc}", + ) + + call_import = CallImport( + id=call_import_id, + organization_id=organization_id, + workspace_id=workspace_id, + # Provider + credential aren't known until the IMPORT stage; leave + # them NULL so the staged-vs-legacy distinction is visible at a + # glance from the DB. + provider=None, + telephony_integration_id=None, + original_filename=file.filename, + sheet_name=None, + dataset=normalized_dataset, + schema_id=schema_id, + parameter_mapping={}, + skipped_columns=[], + column_mapping={}, + extra_columns=[], + custom_column_mapping={}, + source_s3_key=s3_key, + source_format=fmt, + source_size_bytes=len(file_bytes), + source_content_type=content_type, + available_sheets=[sheet.model_dump() for sheet in sheets], + total_rows=0, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.UPLOADED, + ) + if tag_rows: + call_import.tags = tag_rows + + stamp_call_import_actor(call_import, principal, creating=True) + db.add(call_import) + try: + db.commit() + except Exception: + db.rollback() + # Best-effort cleanup of the uploaded S3 object so a failed + # commit doesn't leak storage. + try: + s3_service.delete_file_by_key(s3_key) + except Exception as cleanup_exc: # noqa: BLE001 + logger.warning( + "Failed to clean up orphaned S3 object {} after DB rollback: {}", + s3_key, + cleanup_exc, + ) + raise + + db.refresh(call_import) + return _serialize_call_import(db, call_import) + + +@router.patch( + "/{call_import_id}/mapping", + response_model=CallImportResponse, + operation_id="updateCallImportMapping", +) +async def update_call_import_mapping( + call_import_id: UUID, + payload: CallImportMappingUpdate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportResponse: + """MAP stage of the staged call-import flow. + + Validates ``parameter_mapping`` + ``skipped_columns`` against the + sheet headers captured at UPLOAD time and persists them on the + batch. Idempotent: callers may submit this multiple times while + the batch is in ``uploaded`` or ``mapped`` state. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if call_import.status not in ( + CallImportStatus.UPLOADED, + CallImportStatus.MAPPED, + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot edit mapping on a batch in status " + f"'{call_import.status.value}'. Mapping can only be edited " + "before the IMPORT stage." + ), + ) + + if not call_import.source_s3_key or not call_import.source_format: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This batch was not uploaded through the staged flow and " + "cannot have its mapping edited." + ), + ) + + schema = _resolve_schema( + db, organization_id, workspace_id, payload.schema_id + ) + parameters = list(schema.parameters) + + canonical_sheet = _validate_sheet_choice( + call_import.source_format, + payload.sheet_name, + call_import.available_sheets, + ) + + # Pull the headers for the selected sheet straight out of the + # snapshot so we don't have to re-download the file from S3 just to + # validate the mapping. + headers: List[str] = [] + if call_import.available_sheets: + if canonical_sheet is None: + # CSV: single synthetic sheet. + entry = call_import.available_sheets[0] + headers = list(entry.get("headers") or []) + else: + for entry in call_import.available_sheets: + if not isinstance(entry, dict): + continue + name = entry.get("name") + if isinstance(name, str) and name == canonical_sheet: + headers = list(entry.get("headers") or []) + break + + cleaned_mapping = _clean_parameter_mapping( + payload.parameter_mapping, parameters, schema.name + ) + cleaned_skipped = _clean_skipped_columns(payload.skipped_columns) + + # Run the same per-column validation as the parse path so the user + # gets an immediate 400 if a required parameter is left unmapped or + # a header is neither mapped nor skipped — without needing to read + # the file. ``validate_only`` skips the row loop (and the empty-rows + # guard) since the row data lives in S3, not in this request. + if headers: + _apply_schema_mapping( + headers, + iter(()), + parameters, + cleaned_mapping, + cleaned_skipped, + source_label=( + f"Sheet '{canonical_sheet}'" + if canonical_sheet is not None + else "CSV" + ), + validate_only=True, + ) + + call_import.schema_id = schema.id + call_import.parameter_mapping = dict(cleaned_mapping) + call_import.skipped_columns = list(cleaned_skipped) + call_import.sheet_name = canonical_sheet + call_import.status = CallImportStatus.MAPPED + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + return _serialize_call_import(db, call_import) + + +@router.post( + "/{call_import_id}/import", + response_model=CallImportUploadResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="startCallImport", +) +async def start_call_import( + call_import_id: UUID, + payload: CallImportStartRequest, + background_tasks: BackgroundTasks, + legacy: bool = Query( + False, + description=( + "Deprecated escape hatch for import-only processing. " + "New batches should use Run Evaluation instead." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportUploadResponse: + """Deprecated IMPORT stage — use Run Evaluation for new batches. + + Recording fetch is part of the unified evaluation pipeline. This + endpoint remains available only with ``?legacy=true`` for backward + compatibility. + """ + del api_key, background_tasks + + if not legacy: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "Standalone import is deprecated. Use Run Evaluation — " + "recording fetch is part of the evaluation pipeline. " + "Append ?legacy=true to use the import-only path." + ), + ) + + from sqlalchemy.orm import selectinload as _selectinload + + call_import = ( + db.query(CallImport) + .options(_selectinload(CallImport.tags)) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if call_import.status != CallImportStatus.MAPPED: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot start import for a batch in status " + f"'{call_import.status.value}'. Map the columns first." + ), + ) + + if not call_import.source_s3_key or not call_import.source_format: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This batch has no staged source file and cannot be imported " + "through the staged flow." + ), + ) + + if not call_import.schema_id: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Cannot start import without a mapped schema.", + ) + + schema = _resolve_schema( + db, organization_id, workspace_id, call_import.schema_id + ) + parameters = list(schema.parameters) + + if payload.telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + if (integration.provider or "").lower() == "exotel": + _validate_exotel_import_ready( + parameters, dict(call_import.parameter_mapping or {}) + ) + else: + _validate_direct_url_import_ready( + parameters, dict(call_import.parameter_mapping or {}) + ) + integration = None + + _ensure_blob_storage_enabled() + + if integration is not None: + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + else: + call_import.provider = None + call_import.telephony_integration_id = None + + call_import.total_rows = 0 + call_import.completed_rows = 0 + call_import.failed_rows = 0 + call_import.error_message = None + call_import.status = CallImportStatus.PROCESSING + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + + from app.workers.tasks.call_import_bulk_ops import ( + materialize_call_import_rows_task, + ) + + materialize_call_import_rows_task.delay( + str(call_import_id), + str(organization_id), + str(workspace_id), + schedule_import_dispatch=True, + ) + + return CallImportUploadResponse( + id=call_import.id, + total_rows=0, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + "Import accepted. Rows are being materialized in the background; " + "recordings will be fetched asynchronously." + ), + ) + + +@router.post( + "/upload", + response_model=CallImportUploadResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="uploadCallImportCsv", + deprecated=True, +) +async def upload_call_import_csv( + background_tasks: BackgroundTasks, + file: UploadFile = File(...), + provider: Optional[str] = Form( + None, + description=( + "Telephony provider key (e.g. 'exotel', 'plivo'). Must match the " + "selected telephony_integration_id's provider. Omit together " + "with telephony_integration_id for direct-URL import." + ), + ), + telephony_integration_id: Optional[UUID] = Form( + None, + description=( + "Specific TelephonyIntegration credential row to use when " + "downloading recordings for this batch. Omit together with " + "provider for direct-URL import." + ), + ), + schema_id: UUID = Form( + ..., + description=( + "Reusable Input Parameter schema this upload is mapped against. " + "Must belong to the active workspace." + ), + ), + parameter_mapping: str = Form( + ..., + description=( + "JSON-encoded ``{schema_parameter_name: source_header}`` map " + "covering every required schema parameter. Optional parameters " + "may be omitted or set to an empty string." + ), + ), + skipped_columns: Optional[str] = Form( + None, + description=( + "JSON-encoded list of source header strings the uploader has " + "explicitly skipped. Every header in the file must either be " + "mapped or appear here; otherwise the upload is rejected so a " + "forgotten column never silently drops." + ), + ), + dataset: Optional[str] = Form( + None, + description=( + "Optional free-text dataset label for high-level segregation. " + "Empty strings are stored as NULL." + ), + ), + tag_ids: Optional[List[UUID]] = Form( + None, + description="Optional list of CallImportTag ids to attach to the new batch.", + ), + sheet_name: Optional[str] = Form( + None, + description=( + "Worksheet to import when the file is an Excel workbook " + "(.xlsx / .xlsm). REQUIRED for Excel uploads. Ignored for CSV " + "uploads (rejected with 400 if non-empty so typos surface " + "instead of silently importing the wrong source)." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportUploadResponse: + """Legacy one-shot upload kept for backward compatibility. + + DEPRECATED: prefer the staged flow + (``POST /`` → ``PATCH /{id}/mapping`` → ``POST /{id}/import``) so + each step is idempotent and resumable. This endpoint runs all three + stages inline in a single transaction so existing scripts / + integrations keep working unchanged. + """ + del api_key + + fmt = _file_format(file.filename) + if fmt is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Unsupported file format. Allowed extensions: " + f"{', '.join(ALLOWED_EXTENSIONS)}." + ), + ) + + sheet_name_clean = (sheet_name or "").strip() or None + if fmt == "csv" and sheet_name_clean is not None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is not applicable to CSV uploads.", + ) + if fmt == "xlsx" and sheet_name_clean is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is required when uploading an Excel workbook.", + ) + + file_bytes = await file.read() + if len(file_bytes) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", + ) + + schema = _resolve_schema(db, organization_id, workspace_id, schema_id) + parameters = list(schema.parameters) + + mapping_payload = _parse_json_form_field( + "parameter_mapping", parameter_mapping, {} + ) + cleaned_mapping = _clean_parameter_mapping( + mapping_payload, parameters, schema.name + ) + + skipped_payload = _parse_json_form_field("skipped_columns", skipped_columns, []) + cleaned_skipped = _clean_skipped_columns(skipped_payload) + + has_provider = bool((provider or "").strip()) + has_integration = telephony_integration_id is not None + if has_provider != has_integration: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "provider and telephony_integration_id must both be provided " + "or both omitted for direct-URL import." + ), + ) + + if telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, organization_id, telephony_integration_id, provider or "" + ) + if (integration.provider or "").lower() == "exotel": + _validate_exotel_import_ready(parameters, cleaned_mapping) + else: + _validate_direct_url_import_ready(parameters, cleaned_mapping) + integration = None + + parsed_rows = _parse_source_file( + file_bytes, fmt, sheet_name_clean, parameters, cleaned_mapping, cleaned_skipped + ) + _raise_if_no_importable_rows(parsed_rows, source_label=fmt) + + tag_rows = _resolve_tags(db, organization_id, tag_ids) + + call_import = CallImport( + organization_id=organization_id, + workspace_id=workspace_id, + provider=integration.provider if integration is not None else None, + telephony_integration_id=integration.id if integration is not None else None, + original_filename=file.filename, + sheet_name=sheet_name_clean, + dataset=_normalize_dataset(dataset), + schema_id=schema.id, + parameter_mapping=dict(cleaned_mapping), + skipped_columns=list(cleaned_skipped), + # Legacy columns are left empty on new uploads; the detail page + # falls back to ``parameter_mapping`` when ``schema_id`` is set. + column_mapping={}, + extra_columns=[], + custom_column_mapping={}, + total_rows=len(parsed_rows.rows), + completed_rows=0, + failed_rows=0, + status=CallImportStatus.PENDING, + source_row_skips=parse_skips_to_json(parsed_rows.skipped), + ) + if tag_rows: + call_import.tags = tag_rows + stamp_call_import_actor(call_import, principal, creating=True) + db.add(call_import) + db.flush() # populate call_import.id + if integration is None: + # The model's historical Python default is "exotel"; direct-URL + # imports intentionally have no telephony provider. + call_import.provider = None + + row_models = _materialize_rows( + db, call_import, parsed_rows.rows, organization_id + ) + + call_import.status = CallImportStatus.PROCESSING + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + + background_tasks.add_task( + record_call_import_batch_created, + organization_id, + call_import.id, + workspace_id=workspace_id, + total_rows=call_import.total_rows, + source="csv", + provider=call_import.provider, + ) + + _enqueue_row_tasks(db, call_import, row_models) + + return CallImportUploadResponse( + id=call_import.id, + total_rows=call_import.total_rows, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + f"Accepted {call_import.total_rows} rows for import. " + "Recordings will be fetched asynchronously." + ), + ) + + +@router.post( + "/audio-upload", + response_model=CallImportUploadResponse, + status_code=status.HTTP_201_CREATED, + operation_id="uploadCallImportAudio", +) +async def upload_call_import_audio( + background_tasks: BackgroundTasks, + files: List[UploadFile] = File( + ..., + description="One or more manual call recording audio files.", + ), + dataset: str = Form( + ..., + description="Required free-text dataset label for the manual upload batch.", + ), + tag_ids: Optional[List[UUID]] = Form( + None, + description="Optional list of CallImportTag ids to attach to the new batch.", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportUploadResponse: + """Persist manually uploaded recordings as completed CallImport rows. + + The rows skip the provider-download worker entirely because the audio + bytes are already in hand. From this point onward they behave exactly + like completed CSV-import rows: playback reads ``recording_s3_key`` and + the existing diarisation/evaluation endpoints can operate on them. + """ + + normalized_dataset = _normalize_dataset(dataset) + if not normalized_dataset: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="dataset is required and must be a non-empty string.", + ) + if not files: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="At least one audio file is required.", + ) + + _ensure_blob_storage_enabled() + tag_rows = _resolve_tags(db, organization_id, tag_ids) + + max_bytes = int(settings.MAX_FILE_SIZE_MB) * 1024 * 1024 + prepared: List[Dict[str, Any]] = [] + conversation_counts: Dict[str, int] = {} + + for idx, upload in enumerate(files): + filename = upload.filename or f"recording-{idx + 1}" + ext = _audio_extension(filename) + if not ext: + allowed = ", ".join(settings.ALLOWED_AUDIO_FORMATS) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unsupported audio file '{filename}'. Allowed formats: {allowed}.", + ) + + contents = await upload.read() + if not contents: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Audio file '{filename}' is empty.", + ) + if len(contents) > max_bytes: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=( + f"Audio file '{filename}' exceeds " + f"{settings.MAX_FILE_SIZE_MB} MB." + ), + ) + + base_conversation_id = _sanitize_conversation_id(_filename_stem(filename)) + conversation_id = _dedupe_conversation_id( + base_conversation_id, + conversation_counts, + ) + prepared.append( + { + "filename": filename, + "extension": ext, + "content_type": _audio_content_type(ext, upload.content_type), + "contents": contents, + "conversation_id": conversation_id, + } + ) + + original_filename = ( + prepared[0]["filename"] + if len(prepared) == 1 + else f"{len(prepared)} manual recordings" + ) + total_size = sum(len(item["contents"]) for item in prepared) + uploaded_keys: List[str] = [] + + from app.services.storage.s3_service import s3_service + + call_import = CallImport( + organization_id=organization_id, + workspace_id=workspace_id, + provider=None, + telephony_integration_id=None, + original_filename=original_filename, + source_format="audio", + source_size_bytes=total_size, + source_content_type="audio/*", + dataset=normalized_dataset, + total_rows=len(prepared), + completed_rows=len(prepared), + failed_rows=0, + status=CallImportStatus.COMPLETED, + ) + if tag_rows: + call_import.tags = tag_rows + + stamp_call_import_actor(call_import, principal, creating=True) + try: + db.add(call_import) + db.flush() + # The model's historical Python default is "exotel"; manual uploads + # intentionally have no telephony provider. + call_import.provider = None + + row_mappings: List[Dict[str, Any]] = [] + for idx, item in enumerate(prepared): + row_id = uuid4() + key = _audio_s3_key( + organization_id, + call_import.id, + row_id, + item["extension"], + ) + s3_service.upload_file_by_key( + item["contents"], + key, + content_type=item["content_type"], + ) + uploaded_keys.append(key) + + row_mappings.append( + { + "id": row_id, + "call_import_id": call_import.id, + "organization_id": organization_id, + "workspace_id": workspace_id, + "row_index": idx, + "conversation_id": item["conversation_id"], + "recording_url": None, + "transcript": None, + "transcript_source": None, + "raw_columns": {"conversation_id": item["conversation_id"]}, + "status": CallImportRowStatus.COMPLETED, + "recording_s3_key": key, + "recording_content_type": item["content_type"], + "recording_size_bytes": len(item["contents"]), + } + ) + + if is_sharding_enabled(): + from app.db_sharding.row_ops import ( + bulk_insert_mappings_on_shards, + register_shard_slices, + ) + + bulk_insert_mappings_on_shards(db, call_import.id, row_mappings) + register_shard_slices(db, call_import.id, len(row_mappings)) + else: + for mapping in row_mappings: + db.add(CallImportRow(**mapping)) + + db.commit() + except Exception as exc: + db.rollback() + if uploaded_keys and s3_service.is_enabled(): + try: + s3_service.delete_keys(uploaded_keys) + except Exception: + logger.exception( + "Failed to clean up manual audio upload keys after error" + ) + logger.exception("Failed to persist manual call recording upload") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to upload manual recordings: {exc}", + ) from exc + + db.refresh(call_import) + background_tasks.add_task( + record_call_import_batch_created, + organization_id, + call_import.id, + workspace_id=workspace_id, + total_rows=call_import.total_rows, + source="audio", + provider=None, + ) + return CallImportUploadResponse( + id=call_import.id, + total_rows=call_import.total_rows, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + f"Uploaded {call_import.total_rows} manual recording" + f"{'' if call_import.total_rows == 1 else 's'}." + ), + ) + + +@router.get( + "", + response_model=CallImportListResponse, + operation_id="listCallImports", +) +async def list_call_imports( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + status_filter: Optional[CallImportStatus] = Query(None, alias="status"), + dataset: Optional[str] = Query( + None, + description=( + "Filter by exact dataset string (case-insensitive). Pass the " + "literal value '__none__' to filter to imports with no dataset." + ), + ), + tag_id: Optional[List[UUID]] = Query( + None, + description="Filter to imports tagged with ALL of the given tag ids.", + ), + source_format: Optional[str] = Query( + None, + description=( + "Filter by source format. Use 'audio' for manual recordings or " + "'__non_audio__' for CSV/Excel/legacy imports." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> CallImportListResponse: + """List call-import batches for the active workspace, newest first. + + Scoped to (organization_id, workspace_id) so users only see imports + for the workspace they're currently in. Supports a high-level + ``dataset`` filter (powers the segregation dropdown at the top of + the imports page) plus an AND-style multi-tag filter via repeated + ``tag_id`` parameters. + """ + + query = ( + db.query(CallImport) + .filter( + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + ) + if status_filter is not None: + query = query.filter(CallImport.status == status_filter) + + source_filter = (source_format or "").strip().lower() + if source_filter == "__non_audio__": + query = query.filter( + or_(CallImport.source_format.is_(None), CallImport.source_format != "audio") + ) + elif source_filter: + query = query.filter(func.lower(CallImport.source_format) == source_filter) + + if dataset is not None: + if dataset == "__none__": + query = query.filter(CallImport.dataset.is_(None)) + elif dataset.strip(): + query = query.filter( + func.lower(CallImport.dataset) == dataset.strip().lower() + ) + + if tag_id: + from app.models.database import CallImportTagAssignment + + for single_tag_id in tag_id: + sub = ( + db.query(CallImportTagAssignment.call_import_id) + .filter(CallImportTagAssignment.tag_id == single_tag_id) + .subquery() + ) + query = query.filter(CallImport.id.in_(sub)) + + total = query.count() + items = ( + query.order_by(desc(CallImport.created_at)) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + + email_map = emails_for_user_ids(db, user_ids_from_call_imports(items)) + return CallImportListResponse( + items=[ + _serialize_call_import(db, item, user_emails=email_map) + for item in items + ], + total=total, + page=page, + page_size=page_size, + ) + + +@router.get( + "/dispatch-diagnostics", + response_model=CallImportDispatchDiagnosticsResponse, + operation_id="getCallImportDispatchDiagnostics", + dependencies=[Depends(require_admin)], +) +async def get_call_import_dispatch_diagnostics( + workspace_id: Optional[UUID] = Query( + None, + description=( + "Optional workspace filter. When omitted, returns every workspace " + "in the organization with active eval dispatch state." + ), + ), + include_idle_workspaces: bool = Query( + False, + description=( + "When true, include org workspaces with zero pending rows and " + "zero in-flight slots." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportDispatchDiagnosticsResponse: + """Live eval slot usage and fair-dispatch state for operators. + + Org admins use this to diagnose cross-workspace starvation (e.g. one + workspace's 10k run blocking another's pending eval rows) by inspecting + Redis in-flight counters, pending dispatch rows, and scheduler cursors. + """ + del api_key + payload = build_call_import_dispatch_diagnostics( + db, + organization_id, + workspace_id=workspace_id, + include_idle_workspaces=include_idle_workspaces, + ) + return CallImportDispatchDiagnosticsResponse.model_validate(payload) + + +@router.get( + "/datasets", + response_model=List[str], + operation_id="listCallImportDatasets", +) +async def list_call_import_datasets( + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> List[str]: + """Return the distinct, non-null dataset labels in use for the active + workspace. + + Scoped per-workspace so each workspace's Dataset dropdown only shows + its own segregation labels. + """ + rows = ( + db.query(CallImport.dataset) + .filter( + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + CallImport.dataset.isnot(None), + CallImport.dataset != "", + ) + .distinct() + .order_by(CallImport.dataset.asc()) + .all() + ) + return [row[0] for row in rows if row[0]] + + +@router.get( + "/diarisation-prompt-default", + response_model=CallImportDiarisationPromptDefaultResponse, + operation_id="getCallImportDiarisationPromptDefault", +) +async def get_call_import_diarisation_prompt_default( + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), +) -> CallImportDiarisationPromptDefaultResponse: + """Return the canonical LLM diariser prompt. + + The Transcribe / Run Evaluation modals call this on open so they + can pre-fill the prompt textarea. Returning the constant from the + backend (rather than hard-coding it in the frontend) keeps the + fallback used by the worker and the placeholder shown in the UI + in lock-step — operators always see the *actual* default they'd + get if they leave the field blank. + + Registered before ``GET /{call_import_id}`` so the static path is + not mistaken for a UUID import id (which would 422). + """ + del api_key, organization_id + from app.workers.tasks.helpers.llm_diarisation import ( + DEFAULT_DIARIZATION_PROMPT, + ) + + return CallImportDiarisationPromptDefaultResponse( + prompt=DEFAULT_DIARIZATION_PROMPT + ) + + +@router.patch( + "/{call_import_id}", + response_model=CallImportResponse, + operation_id="updateCallImport", +) +async def update_call_import( + call_import_id: UUID, + payload: CallImportUpdate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportResponse: + """Edit dataset / tag assignments (and schema, pre-import) on a batch. + + ``dataset = ""`` clears the label; ``tag_ids = []`` removes all tag + assignments. Fields omitted from the body are left untouched. + + ``schema_id`` is only honoured while the batch is in + ``uploaded`` / ``mapped`` state — once rows have been materialised + the schema is locked. Changing the schema resets any persisted + mapping (the user must re-MAP) and rewinds status to ``uploaded``. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + body = payload.model_dump(exclude_unset=True) + if "dataset" in body: + call_import.dataset = _normalize_dataset(body["dataset"]) + + if "tag_ids" in body: + tag_ids = body["tag_ids"] or [] + call_import.tags = _resolve_tags(db, organization_id, tag_ids) + + if "schema_id" in body and body["schema_id"] is not None: + if call_import.status not in ( + CallImportStatus.UPLOADED, + CallImportStatus.MAPPED, + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot reassign schema on a batch in status " + f"'{call_import.status.value}'." + ), + ) + new_schema = _resolve_schema( + db, organization_id, workspace_id, body["schema_id"] + ) + if call_import.schema_id != new_schema.id: + # Switching schemas invalidates the persisted mapping — + # parameter names won't line up with the new schema, so + # reset to UPLOADED and force a fresh MAP. + call_import.schema_id = new_schema.id + call_import.parameter_mapping = {} + call_import.skipped_columns = [] + call_import.sheet_name = None + call_import.status = CallImportStatus.UPLOADED + + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + return _serialize_call_import(db, call_import) + + +@router.get( + "/{call_import_id}", + response_model=CallImportDetailResponse, + operation_id="getCallImportDetail", +) +async def get_call_import_detail( + call_import_id: UUID, + row_limit: int = Query(500, ge=0, le=5000), + row_offset: int = Query(0, ge=0), + q: Optional[str] = Query( + None, + description=( + "Optional case-insensitive substring filter on " + "``conversation_id``. When set, ``filtered_total_rows`` in " + "the response reflects the post-filter row count so the UI " + "can paginate against the filtered slice." + ), + ), + diarised_status: Optional[str] = Query( + None, + description=( + "Optional filter on ``CallImportRow.diarised_transcript_status``. " + "Accepts one of ``pending``, ``running``, ``completed``, " + "``failed``. When set, ``filtered_total_rows`` reflects the " + "post-filter row count (combined with the ``q`` filter when " + "both are supplied) so the UI can paginate against the same " + "slice it's displaying." + ), + pattern="^(pending|running|completed|failed)$", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportDetailResponse: + """Fetch a single import batch with a slice of its rows. + + ``row_limit=0`` is intentionally allowed so callers that only need the + batch metadata (e.g. the evaluation-detail page rendering the parent's + column mapping) can skip the rows payload entirely. + """ + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if call_import.status == CallImportStatus.PROCESSING: + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + + prior_status = call_import.status + rollup_call_import_batch_status(db, call_import) + if call_import.status != prior_status: + db.commit() + db.refresh(call_import) + + search_term = (q or "").strip() + diarised_status_filter = (diarised_status or "").strip() or None + filtered_total_rows: Optional[int] = None + has_row_filters = bool(search_term or diarised_status_filter) + + if has_row_filters: + if is_sharding_enabled(): + from app.db_sharding.scatter_gather import count_call_import_rows_filtered + + filtered_total_rows = count_call_import_rows_filtered( + db, + call_import.id, + search_term=search_term, + diarised_status_filter=diarised_status_filter, + ) + else: + rows_query = db.query(CallImportRow).filter( + CallImportRow.call_import_id == call_import.id + ) + if search_term: + rows_query = rows_query.filter( + CallImportRow.conversation_id.ilike(f"%{search_term}%") + ) + if diarised_status_filter: + rows_query = rows_query.filter( + CallImportRow.diarised_transcript_status == diarised_status_filter + ) + filtered_total_rows = rows_query.count() + + if row_limit == 0: + rows: List[CallImportRow] = [] + elif is_sharding_enabled(): + from app.db_sharding.scatter_gather import ( + fetch_call_import_rows_filtered_page, + fetch_call_import_rows_page, + ) + + if has_row_filters: + rows = fetch_call_import_rows_filtered_page( + db, + call_import.id, + search_term=search_term, + diarised_status_filter=diarised_status_filter, + offset=row_offset, + limit=row_limit, + ) + else: + rows = fetch_call_import_rows_page( + db, + call_import.id, + offset=row_offset, + limit=row_limit, + ) + else: + rows_query = db.query(CallImportRow).filter( + CallImportRow.call_import_id == call_import.id + ) + if search_term: + rows_query = rows_query.filter( + CallImportRow.conversation_id.ilike(f"%{search_term}%") + ) + if diarised_status_filter: + rows_query = rows_query.filter( + CallImportRow.diarised_transcript_status == diarised_status_filter + ) + rows = ( + rows_query.order_by(CallImportRow.row_index) + .offset(row_offset) + .limit(row_limit) + .all() + ) + + # Batch-wide diarisation status aggregate. One ``GROUP BY`` query + # across the whole batch — much cheaper than paging through every + # row to recount on the client and lets the UI render a + # transcribe/diarise progress bar without a separate roundtrip. + if is_sharding_enabled(): + from app.db_sharding.scatter_gather import aggregate_diarised_transcript_counts + + diarised_status_counts = aggregate_diarised_transcript_counts( + db, call_import.id + ) + else: + diarised_status_counts: Dict[str, int] = {} + for status_value, count in ( + db.query(CallImportRow.diarised_transcript_status, func.count()) + .filter(CallImportRow.call_import_id == call_import.id) + .group_by(CallImportRow.diarised_transcript_status) + .all() + ): + if isinstance(status_value, str): + diarised_status_counts[status_value] = int(count or 0) + + detail = CallImportDetailResponse.model_validate( + _serialize_call_import(db, call_import).model_dump() + ) + detail.rows = [CallImportRowResponse.model_validate(r) for r in rows] + detail.filtered_total_rows = filtered_total_rows + detail.diarised_pending_rows = diarised_status_counts.get("pending", 0) + detail.diarised_running_rows = diarised_status_counts.get("running", 0) + detail.diarised_completed_rows = diarised_status_counts.get("completed", 0) + detail.diarised_failed_rows = diarised_status_counts.get("failed", 0) + return detail + + +@router.get( + "/{call_import_id}/row-ids", + response_model=CallImportRowIdsResponse, + operation_id="listCallImportRowIds", +) +async def list_call_import_row_ids( + call_import_id: UUID, + q: Optional[str] = Query( + None, + description=( + "Optional case-insensitive substring filter on " + "``conversation_id``. Same semantics as the detail endpoint." + ), + ), + diarised_status: Optional[str] = Query( + None, + description=( + "Optional filter on ``CallImportRow.diarised_transcript_status``. " + "Accepts ``pending`` / ``running`` / ``completed`` / ``failed``." + ), + pattern="^(pending|running|completed|failed)$", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportRowIdsResponse: + """Return every matching ``CallImportRow.id`` for cross-page bulk select. + + Lightweight companion to ``GET /{call_import_id}`` — the detail + endpoint caps ``row_limit`` at 5000 and ships the entire row body + on each page, so harvesting ids that way is wasteful when the + user just wants to bulk-delete or bulk-transcribe everything that + matches the current filters. This endpoint applies the same ``q`` + and ``diarised_status`` filters and returns only the ids. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + search_term = (q or "").strip() + status_filter = (diarised_status or "").strip() or None + + if is_sharding_enabled(): + from app.db_sharding.scatter_gather import list_call_import_row_ids_filtered + + ids = list_call_import_row_ids_filtered( + db, + call_import.id, + search_term=search_term, + diarised_status_filter=status_filter, + ) + return CallImportRowIdsResponse(ids=ids, total=len(ids)) + + rows_query = db.query(CallImportRow.id).filter( + CallImportRow.call_import_id == call_import.id + ) + if search_term: + rows_query = rows_query.filter( + CallImportRow.conversation_id.ilike(f"%{search_term}%") + ) + if status_filter: + rows_query = rows_query.filter( + CallImportRow.diarised_transcript_status == status_filter + ) + + ids = [ + row_id + for (row_id,) in rows_query.order_by(CallImportRow.row_index).all() + ] + return CallImportRowIdsResponse(ids=ids, total=len(ids)) + + +def _revoke_pending_tasks(rows: List[CallImportRow]) -> None: + """Best-effort revoke of in-flight Celery tasks for the given rows. + + Failures are logged and swallowed — Celery's control plane is async and + best-effort by design, and we always do an idempotent S3 cleanup + afterwards so a missed revoke can't leak storage. + """ + task_ids = [ + r.celery_task_id + for r in rows + if r.celery_task_id + and r.status in (CallImportRowStatus.PENDING, CallImportRowStatus.PROCESSING) + ] + if not task_ids: + return + + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(task_ids, terminate=False) + logger.info("Revoked {} pending call-import tasks", len(task_ids)) + except Exception as exc: # noqa: BLE001 + logger.warning("Failed to revoke pending call-import tasks: {}", exc) + + +def _delete_s3_objects( + organization_id: UUID, + call_import_id: UUID, + rows: List[CallImportRow], +) -> tuple[int, int]: + """Delete every recording associated with ``rows`` plus a prefix sweep. + + The prefix sweep also cleans up the staged source file written at + UPLOAD time (``…/call_imports/{id}/source.{csv,xlsx}``) — both the + per-row recording keys and the source artefact share the same + organization-scoped prefix, so a single sweep covers them all. + + Returns ``(deleted_count, error_count)``. Never raises — callers proceed + with the DB delete regardless; orphans, if any, can be cleaned up by + re-running the same delete (it's idempotent). + """ + from app.services.storage.s3_service import s3_service + + if not s3_service.is_enabled(): + return 0, 0 + + keys = [r.recording_s3_key for r in rows if r.recording_s3_key] + deleted = 0 + errors = 0 + + if keys: + try: + d, errs = s3_service.delete_keys(keys) + deleted += d + errors += len(errs) + if errs: + logger.warning( + "S3 bulk-delete reported {} errors for call_import {}", + len(errs), + call_import_id, + ) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Bulk S3 delete failed for call_import {}: {}", call_import_id, exc + ) + errors += len(keys) + + # Belt-and-braces sweep: catch anything that landed under the import's + # prefix but never made it into a row's recording_s3_key (narrow + # window where the S3 upload succeeded but the DB commit didn't). + sweep_prefix = ( + f"{s3_service.prefix}organizations/{organization_id}/" + f"call_imports/{call_import_id}/" + ) + try: + d, errs = s3_service.delete_keys_by_prefix(sweep_prefix) + deleted += d + errors += len(errs) + except Exception as exc: # noqa: BLE001 + logger.exception( + "S3 prefix sweep failed for {}: {}", sweep_prefix, exc + ) + + return deleted, errors + + +@router.delete( + "/{call_import_id}", + response_model=CallImportDeleteResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="deleteCallImport", +) +async def delete_call_import( + call_import_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportDeleteResponse: + """Delete a call-import batch asynchronously. + + Flips the batch to ``deleting`` and enqueues background teardown so + large imports (thousands of rows + S3 objects) do not block the API. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + return CallImportDeleteResponse( + id=call_import_id, + status="completed", + ) + + if call_import.status == CallImportStatus.DELETING: + return CallImportDeleteResponse( + id=call_import.id, + status="accepted", + ) + + call_import.status = CallImportStatus.DELETING + call_import.error_message = None + stamp_call_import_actor(call_import, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import delete_call_import_task + + delete_call_import_task.delay( + str(call_import_id), + str(organization_id), + ) + + return CallImportDeleteResponse( + id=call_import.id, + status="accepted", + ) + + +def _locate_call_import_row_or_404( + catalog_db: Session, + *, + call_import_id: UUID, + row_id: UUID, + organization_id: UUID, +) -> Tuple[Session, CallImportRow, Optional[Session]]: + """Find a call import row on the correct DB session for mutation. + + When sharding is enabled rows live on shard databases; ``get_db`` only + opens the catalog. Returns ``(row_db, row, extra_catalog_to_close)`` + where ``extra_catalog_to_close`` is the catalog session opened by + :func:`locate_call_import_row` (distinct from the route's catalog + session) and must be closed via :func:`close_row_sessions`. + """ + from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row + + try: + row_db, located_catalog, row, _shard_id = locate_call_import_row(row_id) + except LookupError: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import row not found", + ) from None + if ( + row.call_import_id != call_import_id + or row.organization_id != organization_id + ): + close_row_sessions( + row_db, + located_catalog if located_catalog is not row_db else None, + ) + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import row not found", + ) + extra_catalog = located_catalog if located_catalog is not row_db else None + return row_db, row, extra_catalog + + +@router.delete( + "/{call_import_id}/rows/{row_id}", + status_code=status.HTTP_204_NO_CONTENT, + operation_id="deleteCallImportRow", +) +async def delete_call_import_row( + call_import_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Response: + """Delete a single CallImportRow and its S3 recording. + + The parent ``CallImport`` is left in place. After deletion we recompute + its ``total_rows`` / ``completed_rows`` / ``failed_rows`` / ``status`` + so the UI's progress bar stays consistent with reality. + """ + from app.services.storage.s3_service import s3_service + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.db_sharding.row_ops import close_row_sessions + + row_db, row, extra_catalog = _locate_call_import_row_or_404( + db, + call_import_id=call_import.id, + row_id=row_id, + organization_id=organization_id, + ) + try: + _revoke_pending_tasks([row]) + + if row.recording_s3_key and s3_service.is_enabled(): + try: + s3_service.delete_file_by_key(row.recording_s3_key) + except Exception as exc: # noqa: BLE001 — best-effort, DB is source of truth + logger.warning( + "Failed to delete S3 object {} for row {}: {}", + row.recording_s3_key, + row.id, + exc, + ) + + row_db.delete(row) + row_db.commit() + + _recompute_call_import_counters(db, call_import) + stamp_call_import_actor(call_import, principal) + db.commit() + finally: + close_row_sessions(row_db, extra_catalog) + + logger.info( + "Deleted call_import_row {} (call_import={}, org={})", + row_id, + call_import.id, + organization_id, + ) + + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +def _recompute_call_import_counters( + db: Session, call_import: CallImport +) -> None: + """Resync ``total/completed/failed_rows`` + status on the parent batch. + + Called after row-level mutations (single delete, bulk delete) so the + UI's progress bar stays consistent with the actual row set. The + rules mirror :func:`delete_call_import_row` so behavior doesn't + diverge between the per-row and bulk paths. + """ + + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + + rollup_call_import_batch_status(db, call_import) + + +@router.post( + "/{call_import_id}/retry-failed", + response_model=CallImportRetryFailedRowsResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="retryFailedCallImportRows", +) +async def retry_failed_call_import_rows( + call_import_id: UUID, + payload: Optional[CallImportRetryFailedRowsRequest] = Body(None), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRetryFailedRowsResponse: + """Re-enqueue every failed import row in this batch. + + Useful when transient provider issues are resolved and the operator wants + a one-click "try failed downloads again" pass without re-uploading the CSV. + + Pass ``provider`` + ``telephony_integration_id`` (or both omitted for + direct-URL retry) to change how recordings are fetched on this pass. + When the body is omitted entirely, the batch keeps its existing pinned + credentials. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if payload is not None: + if payload.telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + if (integration.provider or "").lower() == "exotel": + schema = _resolve_schema( + db, + organization_id, + call_import.workspace_id, + call_import.schema_id, + ) + _validate_exotel_import_ready( + list(schema.parameters), + dict(call_import.parameter_mapping or {}), + ) + else: + call_import.provider = None + call_import.telephony_integration_id = None + db.flush() + + failed_rows = ( + db.query(CallImportRow) + .filter( + CallImportRow.call_import_id == call_import.id, + CallImportRow.status == CallImportRowStatus.FAILED, + ) + .order_by(CallImportRow.row_index.asc()) + .all() + ) + if not failed_rows: + return CallImportRetryFailedRowsResponse( + requeued=0, + enqueue_failed=0, + skipped=0, + ) + + from app.workers.concurrency.fair_import_dispatch import ( + schedule_fair_import_dispatch, + ) + + # Reset rows to pending BEFORE enqueue so the UI reflects "retry in + # progress" immediately even if the worker queue is backlogged. + for row in failed_rows: + row.status = CallImportRowStatus.PENDING + row.error_message = None + row.celery_task_id = None + + db.flush() + _recompute_call_import_counters(db, call_import) + stamp_call_import_actor(call_import, principal) + db.commit() + + try: + schedule_fair_import_dispatch(max_workspace_turns=999) + requeued = len(failed_rows) + enqueue_failed = 0 + skipped = 0 + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to schedule fair import dispatch for import {}", + call_import.id, + ) + requeued = 0 + enqueue_failed = len(failed_rows) + skipped = 0 + for row in failed_rows: + db.refresh(row) + if row.status != CallImportRowStatus.PENDING: + skipped += 1 + enqueue_failed -= 1 + continue + row.status = CallImportRowStatus.FAILED + row.error_message = f"Failed to enqueue retry: {exc}" + db.flush() + _recompute_call_import_counters(db, call_import) + db.commit() + + return CallImportRetryFailedRowsResponse( + requeued=requeued, + enqueue_failed=enqueue_failed, + skipped=skipped, + ) + + +@router.post( + "/{call_import_id}/rows/bulk-delete", + response_model=CallImportRowBulkDeleteResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="bulkDeleteCallImportRows", +) +async def bulk_delete_call_import_rows( + call_import_id: UUID, + payload: CallImportRowBulkDelete, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRowBulkDeleteResponse: + """Delete multiple ``CallImportRow`` rows in one request. + + Unknown / cross-tenant row ids are silently skipped — the response + reports how many actually went away so a UI that holds onto stale + ids (e.g. after another tab already deleted a row) doesn't 404 + the entire bulk action. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if not payload.row_ids: + return CallImportRowBulkDeleteResponse(deleted=0, status="completed") + + from app.workers.tasks.call_import_bulk_ops import bulk_delete_call_import_rows_task + + row_id_strs = [str(rid) for rid in payload.row_ids] + + stamp_call_import_actor(call_import, principal) + db.commit() + + bulk_delete_call_import_rows_task.delay( + str(call_import_id), + str(organization_id), + row_id_strs, + ) + + return CallImportRowBulkDeleteResponse(deleted=0, status="accepted") + + +# --------------------------------------------------------------------------- +# Diarization / transcription endpoints +# --------------------------------------------------------------------------- + + +def _select_rows_for_transcription( + db: Session, + call_import: CallImport, + payload: CallImportTranscribeRequest, + requested_row_ids: Optional[List[UUID]] = None, +) -> tuple[List[CallImportRow], Dict[str, int]]: + """Pick which rows to enqueue for diarisation (delegates to bulk_ops).""" + from app.services.call_imports.bulk_ops import select_rows_for_transcription + + try: + return select_rows_for_transcription( + db, call_import, payload, requested_row_ids=requested_row_ids + ) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + + +@router.post( + "/{call_import_id}/transcribe", + response_model=CallImportTranscribeResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="transcribeCallImport", +) +async def transcribe_call_import( + call_import_id: UUID, + payload: CallImportTranscribeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportTranscribeResponse: + """Fan out diarization tasks for many rows in a single call. + + Returns a summary with how many rows were queued and how many were + skipped (broken down by reason) so the UI can show a meaningful + toast even when nothing actually got enqueued (e.g. "All 12 rows + already have transcripts"). + """ + + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.workers.tasks.call_import_bulk_ops import bulk_diarize_call_import_task + + stamp_call_import_actor(call_import, principal) + db.commit() + + bulk_diarize_call_import_task.delay( + str(call_import_id), + str(organization_id), + payload.model_dump(mode="json"), + [str(rid) for rid in payload.row_ids] if payload.row_ids else None, + ) + + return CallImportTranscribeResponse( + queued=0, + skipped_rows=0, + skipped_reason_counts={}, + accepted=True, + ) + + +@router.post( + "/{call_import_id}/rows/{row_id}/transcribe", + response_model=CallImportTranscribeResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="transcribeCallImportRow", +) +async def transcribe_call_import_row( + call_import_id: UUID, + row_id: UUID, + payload: CallImportTranscribeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportTranscribeResponse: + """Diarize / transcribe a single row. + + Thin wrapper over the batch endpoint that hard-codes a single + ``row_ids`` filter. Skip counts still surface so the UI can render + "Skipped — transcript present" diagnostics consistently. + """ + + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.services.call_imports.bulk_ops import execute_bulk_diarization + + try: + result = execute_bulk_diarization( + db, + call_import, + payload, + requested_row_ids=[row_id], + ) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + + stamp_call_import_actor(call_import, principal) + db.commit() + + return CallImportTranscribeResponse( + queued=result.queued, + skipped_rows=result.skipped_rows, + skipped_reason_counts=result.skipped_reason_counts, + ) + + +# --------------------------------------------------------------------------- +# Cancel-in-flight diarisation +# --------------------------------------------------------------------------- +# +# Long-running multimodal LLM diarisation calls (especially LLM-only mode on +# slow audio) can sit in ``pending`` / ``running`` for tens of minutes when an +# upstream provider stalls. Without an abort affordance the operator's only +# recourse is to wait for Celery's ``time_limit`` to fire — which can be +# several minutes — or to manually mutate the DB. These helpers + the two +# endpoints below give the UI a first-class "Stop diarisation" button. +# +# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` helper uses +# ``terminate=False`` because it's called from delete-flow paths where the +# task may simply not get to run (a worker pulls it off the queue and drops +# it). For a user-initiated cancel we want SIGTERM to interrupt the worker +# mid-LLM call so the audio HTTP request actually aborts. ``terminate=True`` +# routes SIGTERM to the executing process; ``signal="SIGTERM"`` is the +# default but we spell it out so the intent is obvious to reviewers. + +# Sentinel error message stamped on cancelled rows. Read by the transcribe +# worker's finaliser (see ``app/workers/tasks/transcribe_call_import_row.py``) +# to detect a row that was cancelled mid-flight and AVOID overwriting it +# with whatever partial result the worker had managed to compute before the +# SIGTERM landed. +CANCELLED_BY_USER_ERROR: str = "Diarisation cancelled by user" + + +def _cancellable_diarisation_states() -> Tuple[str, ...]: + """States that a diarisation row can be cancelled from. + + Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` + state only needs one edit. + """ + return ("pending", "running") + + +def _revoke_diarisation_task(row: CallImportRow) -> None: + """Best-effort revoke of a single row's diarisation Celery task. + + Always swallows control-plane exceptions — Celery's control bus is + inherently best-effort and a missed revoke is not catastrophic + because the DB row is already flipped to ``failed`` by the caller + before this runs (so the UI immediately reflects the cancel; if + the task happens to finish anyway, the worker's finaliser skips + over the row via :data:`CANCELLED_BY_USER_ERROR`). + """ + task_id = (row.celery_task_id or "").strip() + if not task_id: + return + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke( + task_id, terminate=True, signal="SIGTERM" + ) + logger.info( + "Revoked diarisation task {} for call-import row {}", + task_id, + row.id, + ) + except Exception as exc: # noqa: BLE001 — revoke is best-effort + logger.warning( + "Failed to revoke diarisation task {} for row {}: {}", + task_id, + row.id, + exc, + ) + + +def _apply_diarisation_cancel(rows: List[CallImportRow]) -> Tuple[int, int]: + """Cancel diarisation on every cancellable row in ``rows``. + + Returns ``(cancelled, skipped)`` so the caller can build a typed + response without re-querying the DB. The caller is responsible for + ``db.commit()`` after this returns — we deliberately don't commit + here so a batch endpoint can flush all rows in one transaction. + """ + cancellable_states = _cancellable_diarisation_states() + cancelled = 0 + skipped = 0 + for row in rows: + if (row.diarised_transcript_status or "").lower() not in cancellable_states: + skipped += 1 + continue + # Flip the row state BEFORE we revoke so the UI's next poll + # already shows the cancel, even if Celery's control plane is + # slow to ack. + row.diarised_transcript_status = "failed" + row.diarised_transcript_error = CANCELLED_BY_USER_ERROR + _revoke_diarisation_task(row) + # Drop the task id so a follow-up retry (or a stale poll) can't + # accidentally re-revoke or get confused. + row.celery_task_id = None + cancelled += 1 + return cancelled, skipped + + +@router.post( + "/{call_import_id}/rows/{row_id}/cancel-diarisation", + response_model=CallImportRowResponse, + status_code=status.HTTP_200_OK, + operation_id="cancelCallImportRowDiarisation", +) +async def cancel_call_import_row_diarisation( + call_import_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRowResponse: + """Abort an in-flight (or queued) diarisation for a single row. + + Idempotent: calling on a row that's already terminal (``completed`` + / ``failed`` / ``idle``) returns the row unchanged with a 200, so + the UI can fire this from a "Stop" button without having to + pre-check the state. + + Race notes: + + * The row's ``diarised_transcript_status`` is flipped to ``failed`` + with :data:`CANCELLED_BY_USER_ERROR` BEFORE the Celery revoke, + so the polling UI sees the cancel immediately. + * If the worker happens to finish between our DB flip and the + SIGTERM landing, its finaliser will detect the cancelled + sentinel on the row and skip its own status / score writes + (see :mod:`app.workers.tasks.transcribe_call_import_row`). + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.db_sharding.row_ops import close_row_sessions + + row_db, row, extra_catalog = _locate_call_import_row_or_404( + db, + call_import_id=call_import_id, + row_id=row_id, + organization_id=organization_id, + ) + try: + _apply_diarisation_cancel([row]) + row_db.commit() + stamp_call_import_actor(call_import, principal) + db.commit() + row_db.refresh(row) + return CallImportRowResponse.model_validate(row) + finally: + close_row_sessions(row_db, extra_catalog) + + +@router.post( + "/{call_import_id}/cancel-diarisation", + response_model=CallImportCancelDiarisationResponse, + status_code=status.HTTP_200_OK, + operation_id="cancelCallImportDiarisation", +) +async def cancel_call_import_diarisation( + call_import_id: UUID, + payload: Optional[CallImportCancelDiarisationRequest] = None, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportCancelDiarisationResponse: + """Abort in-flight diarisation for many rows in a single call. + + Default body (no ``row_ids``) cancels every row in this import + whose ``diarised_transcript_status`` is ``pending`` or + ``running`` — the "stop everything" button. Pass ``row_ids`` to + scope the cancel to the rows the operator has selected. + + Returns ``(cancelled, skipped)`` so the UI can render a tight + toast ("Cancelled 3 rows · 1 skipped (already completed)"). + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + base_query = db.query(CallImportRow).filter( + CallImportRow.call_import_id == call_import_id + ) + + requested_ids = ( + payload.row_ids if payload and payload.row_ids is not None else None + ) + if requested_ids is not None: + if not requested_ids: + # Empty list is "no rows requested" — treat as a no-op + # 200 rather than 400 so the UI can pass through an empty + # selection without a special-case. + return CallImportCancelDiarisationResponse(cancelled=0, skipped=0) + rows = base_query.filter(CallImportRow.id.in_(requested_ids)).all() + found_ids = {r.id for r in rows} + # Treat requested-but-not-found ids as ``skipped`` so the UI's + # numbers reconcile (a stale selection that includes deleted + # rows shouldn't 404 the whole call). + missing = [rid for rid in requested_ids if rid not in found_ids] + skipped_missing = len(missing) + else: + # Implicit "cancel every cancellable row in this import" path. + rows = base_query.filter( + CallImportRow.diarised_transcript_status.in_( + list(_cancellable_diarisation_states()) + ) + ).all() + skipped_missing = 0 + + cancelled, skipped = _apply_diarisation_cancel(rows) + stamp_call_import_actor(call_import, principal) + db.commit() + return CallImportCancelDiarisationResponse( + cancelled=cancelled, + skipped=skipped + skipped_missing, + ) + + +def _render_diarised_segments_text( + segments: Optional[List[Dict[str, Any]]], + *, + swap: bool = False, +) -> str: + """Render ``CallImportRow.diarised_segments`` as ``: `` lines. + + Mirrors the worker's ``_render_turns_as_text`` (kept duplicated so + the route doesn't need to import a Celery task module just to + rebuild the rendered transcript). Only ``agent`` and ``user`` are + swapped — multi-party calls keep their ``speaker_N`` labels through + a swap so we don't silently collapse a third speaker into the user + side. + """ + if not segments: + return "" + out: List[str] = [] + for turn in segments: + if not isinstance(turn, dict): + continue + speaker = (turn.get("speaker") or "").strip() + text = (turn.get("text") or "").strip() + if not speaker or not text: + continue + if swap: + if speaker == "agent": + speaker = "user" + elif speaker == "user": + speaker = "agent" + out.append(f"{speaker}: {text}") + return "\n".join(out) + + +@router.post( + "/{call_import_id}/rows/{row_id}/diarised-speaker-swap", + response_model=CallImportRowResponse, + operation_id="toggleCallImportRowSpeakerSwap", +) +async def toggle_call_import_row_speaker_swap( + call_import_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRowResponse: + """Flip the user <-> agent mapping on a diarised row. + + The worker's "first speaker is the agent" heuristic is right most of + the time but does fail on inbound recordings where the customer + greets first, on recordings where the agent stays silent for the + intro, etc. Rather than rerun the (expensive) STT + pyannote + pipeline for those cases, we let reviewers flip the mapping in + place: the structured ``diarised_segments`` are the source of truth + and we re-render the plain-text ``diarised_transcript`` from them + with the swap applied. The next CSV export will then show the + corrected labels. + + Returns the updated row so the frontend can refresh without an + extra round-trip. + """ + + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.db_sharding.row_ops import close_row_sessions + + row_db, row, extra_catalog = _locate_call_import_row_or_404( + db, + call_import_id=call_import_id, + row_id=row_id, + organization_id=organization_id, + ) + try: + segments = ( + row.diarised_segments if isinstance(row.diarised_segments, list) else None + ) + if not segments: + # Without structured turns the swap toggle would have nothing to + # re-render — surface a clear error rather than silently + # flipping a flag the UI never read. + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This row has no structured diarised segments to swap. " + "Re-run diarisation to generate per-speaker turns first." + ), + ) + + new_swap = not bool(row.diarised_speaker_swap) + row.diarised_speaker_swap = new_swap + row.diarised_transcript = ( + _render_diarised_segments_text(segments, swap=new_swap) or None + ) + row_db.commit() + stamp_call_import_actor(call_import, principal) + db.commit() + row_db.refresh(row) + return CallImportRowResponse.model_validate(row) + finally: + close_row_sessions(row_db, extra_catalog) + + +# --------------------------------------------------------------------------- +# Cross-run insights for the import detail page +# --------------------------------------------------------------------------- + + +@router.get( + "/{call_import_id}/insights", + response_model=CallImportInsightsResponse, + operation_id="getCallImportInsights", +) +async def get_call_import_insights( + call_import_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportInsightsResponse: + """Aggregate signals across every evaluation run on this import. + + Powers the Insights tab on the call-import detail page: returns + per-metric "latest run" summaries plus a trend series of mean values + across runs so the UI can render a small line chart per metric. Also + bundles transcript coverage stats since those are the cheapest + pre-eval health-check (e.g. "30 of 50 rows still missing + transcripts"). + """ + + del api_key + + from app.models.database import ( + CallImportEvaluation, + CallImportEvaluationRow, + Metric, + ) + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + rows = ( + db.query(CallImportRow) + .filter(CallImportRow.call_import_id == call_import_id) + .all() + ) + # A row "has a transcript" if EITHER the production (CSV) or the + # diarised (worker) column is populated — the insights tile reports + # the union so users see total coverage regardless of which source + # produced the value. + rows_with_transcript = sum( + 1 + for r in rows + if (r.transcript or "").strip() + or (r.diarised_transcript or "").strip() + ) + rows_without_transcript = len(rows) - rows_with_transcript + source_counts: Dict[str, int] = {} + for r in rows: + has_production = bool((r.transcript or "").strip()) + has_diarised = bool((r.diarised_transcript or "").strip()) + if has_production: + key = r.transcript_source or "csv" + source_counts[key] = source_counts.get(key, 0) + 1 + if has_diarised: + source_counts["diarised"] = source_counts.get("diarised", 0) + 1 + + evaluations = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .order_by(CallImportEvaluation.created_at.asc()) + .all() + ) + + # Defer heavy lifting to the aggregation helper so this endpoint and + # the per-run aggregate endpoint share the exact same metric + # bucketing math (no chance of "trend" disagreeing with "latest" on + # the same data set). + from app.api.v1.routes.call_import_evaluations import ( + _compute_metric_aggregates, + ) + + metric_history: Dict[str, List[CallImportInsightsRunPoint]] = {} + metric_meta: Dict[str, Metric] = {} + metric_latest: Dict[str, CallImportMetricAggregate] = {} + + for evaluation in evaluations: + eval_rows = ( + db.query(CallImportEvaluationRow) + .filter(CallImportEvaluationRow.evaluation_id == evaluation.id) + .all() + ) + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + for agg in aggregates: + if agg.metric_id not in metric_meta: + # ``agg.metric_id`` is normally a UUID string, but the + # aggregator also emits ids that surface in row scores + # without a matching ``Metric`` row (e.g. a metric the + # user deleted mid-run, or LLM-discovered slugs). Those + # are not valid UUIDs, so coerce defensively and skip + # the metric registry lookup when the cast fails — the + # ``meta is None`` branch below already handles the + # display via the values stored on ``agg`` itself. + try: + metric_uuid = UUID(agg.metric_id) + except (ValueError, AttributeError, TypeError): + metric_uuid = None + if metric_uuid is not None: + metric_obj = ( + db.query(Metric) + .filter( + Metric.id == metric_uuid, + Metric.organization_id == organization_id, + ) + .first() + ) + if metric_obj is not None: + metric_meta[agg.metric_id] = metric_obj + history = metric_history.setdefault(agg.metric_id, []) + history.append( + CallImportInsightsRunPoint( + evaluation_id=evaluation.id, + name=evaluation.name, + created_at=evaluation.created_at, + mean=agg.mean, + completed_rows=agg.count, + ) + ) + metric_latest[agg.metric_id] = agg + + metrics_payload: List[CallImportInsightsMetric] = [] + for metric_id, latest in metric_latest.items(): + meta = metric_meta.get(metric_id) + metrics_payload.append( + CallImportInsightsMetric( + metric_id=metric_id, + metric_name=(meta.name if meta else latest.metric_name), + metric_type=(meta.metric_type if meta else latest.metric_type), + latest=latest, + trend=metric_history.get(metric_id, []), + ) + ) + + return CallImportInsightsResponse( + call_import_id=call_import_id, + total_rows=len(rows), + rows_with_transcript=rows_with_transcript, + rows_without_transcript=rows_without_transcript, + transcript_source_counts=source_counts, + evaluation_count=len(evaluations), + metrics=metrics_payload, + ) + + +from app.core.auth.capabilities import CALLS_DELETE, CALLS_IMPORT, CALLS_VIEW +from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities + +apply_workspace_route_capabilities( + router, + view_capability=CALLS_VIEW, + manage_capability=CALLS_IMPORT, + delete_capability=CALLS_DELETE, +) diff --git a/app/config.py b/app/config.py index 91ecdbb3..0eec6b2a 100644 --- a/app/config.py +++ b/app/config.py @@ -1,6 +1,8 @@ """Configuration management using Pydantic settings.""" import json +import os +import re import yaml from pathlib import Path from typing import Annotated, Any, Dict, List, Optional, Union @@ -107,7 +109,7 @@ class Settings(BaseSettings): RATE_LIMIT_PER_MINUTE: int = 60 # Authentication - AUTH_PROVIDERS: List[str] = ["api_key"] + AUTH_PROVIDERS: Annotated[List[str], NoDecode] = ["api_key"] AUTH_LOCAL_ALLOW_SIGNUP: bool = True AUTH_LOCAL_TOKEN_TTL_MINUTES: int = 15 AUTH_REFRESH_TOKEN_TTL_DAYS: int = 7 @@ -412,6 +414,19 @@ def apply_service_mode(mode: str) -> None: settings.SERVICE_MODE = normalized +_ENV_REF_PATTERN = re.compile(r"^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$") + + +def _expand_env_ref(value: Any) -> Any: + """Replace ``${VAR}`` with ``os.environ[VAR]`` when loading YAML secrets.""" + if not isinstance(value, str): + return value + match = _ENV_REF_PATTERN.match(value.strip()) + if not match: + return value + return os.environ.get(match.group(1), "") + + def load_config_from_file(config_path: str) -> None: """Load configuration from a YAML file and update global settings.""" import yaml @@ -577,9 +592,13 @@ def load_config_from_file(config_path: str) -> None: if "region" in s3_config: settings.S3_REGION = s3_config["region"] if "access_key_id" in s3_config: - settings.S3_ACCESS_KEY_ID = s3_config["access_key_id"] + resolved = _expand_env_ref(s3_config["access_key_id"]) + if resolved: + settings.S3_ACCESS_KEY_ID = resolved if "secret_access_key" in s3_config: - settings.S3_SECRET_ACCESS_KEY = s3_config["secret_access_key"] + resolved = _expand_env_ref(s3_config["secret_access_key"]) + if resolved: + settings.S3_SECRET_ACCESS_KEY = resolved if "endpoint_url" in s3_config: settings.S3_ENDPOINT_URL = s3_config["endpoint_url"] if "prefix" in s3_config: @@ -730,9 +749,13 @@ def load_config_from_file(config_path: str) -> None: if "region" in loki_s3: settings.LOKI_S3_REGION = loki_s3["region"] if "access_key_id" in loki_s3: - settings.LOKI_S3_ACCESS_KEY_ID = loki_s3["access_key_id"] + resolved = _expand_env_ref(loki_s3["access_key_id"]) + if resolved: + settings.LOKI_S3_ACCESS_KEY_ID = resolved if "secret_access_key" in loki_s3: - settings.LOKI_S3_SECRET_ACCESS_KEY = loki_s3["secret_access_key"] + resolved = _expand_env_ref(loki_s3["secret_access_key"]) + if resolved: + settings.LOKI_S3_SECRET_ACCESS_KEY = resolved if "prefix" in loki_s3: settings.LOKI_S3_PREFIX = loki_s3["prefix"] if "plivo" in config_data: diff --git a/app/core/auth/token_revocation.py b/app/core/auth/token_revocation.py index b1e2f0ac..379c20be 100644 --- a/app/core/auth/token_revocation.py +++ b/app/core/auth/token_revocation.py @@ -34,7 +34,7 @@ def revoke_access_jti(jti: str, ttl_seconds: int) -> None: """Blacklist an access token until its natural expiry.""" ttl = max(int(ttl_seconds), 1) try: - _get_redis().setex(f"revoked:jti:{jti}", ttl, "1") + _get_redis().set(f"revoked:jti:{jti}", "1", ex=ttl) except redis.RedisError as exc: logger.warning("Redis unavailable for token revocation; using in-memory fallback: %s", exc) _in_memory_revoked[jti] = time.time() + ttl diff --git a/app/migrations/059_call_import_audit_users.py b/app/migrations/059_call_import_audit_users.py new file mode 100644 index 00000000..41ec06db --- /dev/null +++ b/app/migrations/059_call_import_audit_users.py @@ -0,0 +1,69 @@ +""" +Migration: last_updated_by_user_id on call imports and evaluations. + +Supports surfacing who created / last modified a batch or evaluation run +via FK to users (email resolved at read time). +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Add last_updated_by_user_id to call_imports and call_import_evaluations" +) + + +def _column_exists(db: Session, table: str, column: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table, "column_name": column}, + ).first() + return row is not None + + +def _table_exists(db: Session, table: str) -> bool: + row = db.execute( + text( + "SELECT 1 FROM information_schema.tables WHERE table_name = :t" + ), + {"t": table}, + ).first() + return row is not None + + +def upgrade(db: Session): + for table in ("call_imports", "call_import_evaluations"): + if not _table_exists(db, table): + print(f"{table} does not exist, skipping...") + continue + if _column_exists(db, table, "last_updated_by_user_id"): + print(f"{table}.last_updated_by_user_id already exists, skipping...") + continue + db.execute( + text( + f""" + ALTER TABLE {table} + ADD COLUMN last_updated_by_user_id UUID NULL + REFERENCES users(id) ON DELETE SET NULL + """ + ) + ) + print(f"Added {table}.last_updated_by_user_id") + + +def downgrade(db: Session): + for table in ("call_import_evaluations", "call_imports"): + if not _table_exists(db, table): + continue + if not _column_exists(db, table, "last_updated_by_user_id"): + continue + db.execute( + text(f"ALTER TABLE {table} DROP COLUMN last_updated_by_user_id") + ) + print(f"Dropped {table}.last_updated_by_user_id") diff --git a/app/migrations/060_call_import_evaluation_pdf_reports.py b/app/migrations/060_call_import_evaluation_pdf_reports.py new file mode 100644 index 00000000..8f2c6012 --- /dev/null +++ b/app/migrations/060_call_import_evaluation_pdf_reports.py @@ -0,0 +1,76 @@ +"""Migration: stored PDF reports for call import evaluations.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add call_import_evaluation_pdf_reports for S3-stored evaluation PDFs" + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if _table_exists(db, "call_import_evaluation_pdf_reports"): + print("call_import_evaluation_pdf_reports already exists, skipping") + db.commit() + return + + db.execute( + text( + """ + CREATE TABLE call_import_evaluation_pdf_reports ( + id UUID PRIMARY KEY, + evaluation_id UUID NOT NULL REFERENCES call_import_evaluations(id) ON DELETE CASCADE, + call_import_id UUID NOT NULL REFERENCES call_imports(id) ON DELETE CASCADE, + organization_id UUID NOT NULL REFERENCES organizations(id), + workspace_id UUID NOT NULL REFERENCES workspaces(id), + snapshot_id UUID REFERENCES call_import_evaluation_report_snapshots(id) ON DELETE SET NULL, + vendor_name VARCHAR(120) NOT NULL, + report_type VARCHAR(20) NOT NULL DEFAULT 'external', + filename VARCHAR(255), + s3_key VARCHAR(512), + report_config JSONB NOT NULL DEFAULT '{}'::jsonb, + cache_fingerprint VARCHAR(64), + created_by TEXT, + created_by_user_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_call_import_eval_pdf_reports_eval + ON call_import_evaluation_pdf_reports (evaluation_id, created_at DESC) + """ + ) + ) + db.execute( + text( + """ + CREATE UNIQUE INDEX IF NOT EXISTS + uq_call_import_eval_pdf_reports_eval_cache_fp + ON call_import_evaluation_pdf_reports (evaluation_id, cache_fingerprint) + WHERE cache_fingerprint IS NOT NULL + """ + ) + ) + print("Created call_import_evaluation_pdf_reports") + db.commit() + + +def downgrade(db: Session): + db.execute(text("DROP TABLE IF EXISTS call_import_evaluation_pdf_reports")) + db.commit() diff --git a/app/migrations/061_call_import_eval_pdf_report_cache_index.py b/app/migrations/061_call_import_eval_pdf_report_cache_index.py new file mode 100644 index 00000000..baa579b0 --- /dev/null +++ b/app/migrations/061_call_import_eval_pdf_report_cache_index.py @@ -0,0 +1,137 @@ +"""Migration: PDF report cache fingerprint column rename and lookup index.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Rename config_fingerprint to cache_fingerprint and add unique " + "(evaluation_id, cache_fingerprint) for scale-safe cache lookups" +) + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def _column_exists(db: Session, column_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'call_import_evaluation_pdf_reports' + AND column_name = :column_name + """ + ), + {"column_name": column_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _table_exists(db, "call_import_evaluation_pdf_reports"): + print("call_import_evaluation_pdf_reports missing, skipping 061") + db.commit() + return + + if _column_exists(db, "config_fingerprint") and not _column_exists( + db, "cache_fingerprint" + ): + db.execute( + text( + """ + ALTER TABLE call_import_evaluation_pdf_reports + RENAME COLUMN config_fingerprint TO cache_fingerprint + """ + ) + ) + print("Renamed config_fingerprint -> cache_fingerprint") + + if not _column_exists(db, "cache_fingerprint"): + print("cache_fingerprint column missing, skipping index work") + db.commit() + return + + db.execute( + text( + """ + DELETE FROM call_import_evaluation_pdf_reports stale + USING call_import_evaluation_pdf_reports keep + WHERE stale.evaluation_id = keep.evaluation_id + AND stale.cache_fingerprint = keep.cache_fingerprint + AND stale.cache_fingerprint IS NOT NULL + AND stale.id <> keep.id + AND ( + stale.created_at < keep.created_at + OR ( + stale.created_at = keep.created_at + AND stale.id::text < keep.id::text + ) + ) + """ + ) + ) + + db.execute( + text( + """ + DROP INDEX IF EXISTS ix_call_import_eval_pdf_reports_fingerprint + """ + ) + ) + db.execute( + text( + """ + CREATE UNIQUE INDEX IF NOT EXISTS + uq_call_import_eval_pdf_reports_eval_cache_fp + ON call_import_evaluation_pdf_reports (evaluation_id, cache_fingerprint) + WHERE cache_fingerprint IS NOT NULL + """ + ) + ) + print("Ensured unique (evaluation_id, cache_fingerprint) index") + db.commit() + + +def downgrade(db: Session): + if not _table_exists(db, "call_import_evaluation_pdf_reports"): + db.commit() + return + + db.execute( + text( + """ + DROP INDEX IF EXISTS uq_call_import_eval_pdf_reports_eval_cache_fp + """ + ) + ) + if _column_exists(db, "cache_fingerprint") and not _column_exists( + db, "config_fingerprint" + ): + db.execute( + text( + """ + ALTER TABLE call_import_evaluation_pdf_reports + RENAME COLUMN cache_fingerprint TO config_fingerprint + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_call_import_eval_pdf_reports_fingerprint + ON call_import_evaluation_pdf_reports (config_fingerprint) + """ + ) + ) + db.commit() diff --git a/app/models/database.py b/app/models/database.py index a21a4819..68bcfebc 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -1,2729 +1,2779 @@ -"""SQLAlchemy database models.""" - -from sqlalchemy import ( - BigInteger, - Boolean, - Column, - Date, - DateTime, - DDL, - Enum, - event, - Float, - ForeignKey, - Integer, - JSON, - String, - Text, - UniqueConstraint, - select, - text, -) -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import relationship -from sqlalchemy.sql import func -import uuid -import enum -from app.models.enums import ( - EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, - LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, - IntegrationPlatform, ModelProvider, VoiceBundleType, TestAgentConversationStatus, - MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, - AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, - PromptOptimizationStatus, CallImportStatus, CallImportRowStatus, -) - -def get_enum_values(enum_class): - """Helper to get values from enum class for SQLAlchemy.""" - return [e.value for e in enum_class] - -from app.database import Base - - -# Enums moved to enums.py - - -class Organization(Base): - """Organization model for multi-tenancy.""" - - __tablename__ = "organizations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - name = Column(String(255), nullable=False) - voice_playground_threshold_overrides = Column(JSON, nullable=True) - # AlignEval-style judge alignment thresholds. - # Shape: {"min_labels_to_evaluate": int, "min_labels_to_optimize": int} - # Falls back to system defaults (20 / 50) when null. - judge_alignment_settings = Column(JSON, nullable=True) - # Per-org LLM gateway overrides (enabled, gateway_type, base_url, keys). - llm_gateway_settings = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - api_keys = relationship("APIKey", back_populates="organization") - members = relationship("OrganizationMember", back_populates="organization", cascade="all, delete-orphan") - invitations = relationship("Invitation", back_populates="organization", cascade="all, delete-orphan") - workspaces = relationship( - "Workspace", - back_populates="organization", - cascade="all, delete-orphan", - ) - workspace_roles = relationship( - "WorkspaceRole", - back_populates="organization", - cascade="all, delete-orphan", - ) - - -class Workspace(Base): - """Workspace - in-org isolation boundary for call imports and metrics. - - Every organization has at least one workspace (``is_default = True``, - seeded by migration 033). Users pick an "active workspace" in the UI; - list endpoints filter by it so users only see calls/metrics from the - project they're currently working in. Access is governed by - ``workspace_members`` and org-scoped ``workspace_roles`` (capability - bundles); org admins implicitly access all workspaces. - """ - - __tablename__ = "workspaces" - __table_args__ = ( - UniqueConstraint("organization_id", "slug", name="uq_workspaces_org_slug"), - ) - - # ``server_default`` is required so that raw-SQL INSERTs (e.g. the - # per-org Default seed in migration 033) can omit ``id`` and let the - # database fill it in. Without it, ``create_all`` produces a column - # with NOT NULL but no DEFAULT, and the migration crashes with - # ``null value in column "id"``. - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - slug = Column(String(255), nullable=False) - # At most one default per org. Enforced on Postgres by the partial - # unique index attached via the after_create event below; on - # SQLite (test runs) we rely on the route-level _check_slug_unique - # check + the Default-workspace conftest fixture instead, because - # SQLite doesn't support partial indexes the same way. - is_default = Column(Boolean, nullable=False, default=False, server_default="false") - # Reusable PDF/report branding metadata scoped to this workspace. Images - # live in S3. Shape: {"heading": str|null, "images": [{id, s3_key, - # content_type, filename, size_bytes, updated_at}, ...]}. - report_branding = Column(JSON, nullable=True) - created_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - organization = relationship("Organization", back_populates="workspaces") - members = relationship( - "WorkspaceMember", - back_populates="workspace", - cascade="all, delete-orphan", - ) - - -class WorkspaceRole(Base): - """Org-scoped workspace role (system or custom) as a capability bundle.""" - - __tablename__ = "workspace_roles" - __table_args__ = ( - UniqueConstraint("organization_id", "name", name="uq_workspace_roles_org_name"), - ) - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - capabilities = Column(JSON, nullable=False, default=list) - is_system = Column(Boolean, nullable=False, default=False, server_default="false") - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - organization = relationship("Organization", back_populates="workspace_roles") - members = relationship("WorkspaceMember", back_populates="role") - - -class WorkspaceMember(Base): - """User membership in a workspace with an assigned workspace role.""" - - __tablename__ = "workspace_members" - __table_args__ = ( - UniqueConstraint("workspace_id", "user_id", name="uq_workspace_members_ws_user"), - ) - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - role_id = Column( - UUID(as_uuid=True), - ForeignKey("workspace_roles.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - added_by_user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="SET NULL"), - nullable=True, - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - workspace = relationship("Workspace", back_populates="members") - user = relationship("User", foreign_keys=[user_id]) - role = relationship("WorkspaceRole", back_populates="members") - added_by = relationship("User", foreign_keys=[added_by_user_id]) - - -# Partial unique index: "at most one default workspace per org". This -# is attached as an after_create event (rather than declared in -# ``__table_args__``) because SQLAlchemy's ``Index(..., -# postgresql_where=...)`` silently degrades to a *full* unique index on -# SQLite - which then forbids any second workspace per org and breaks -# the test suite. ``execute_if(dialect="postgresql")`` makes this DDL -# a no-op on SQLite while still emitting it on Postgres (prod, CI). -event.listen( - Workspace.__table__, - "after_create", - DDL( - "CREATE UNIQUE INDEX IF NOT EXISTS uq_workspaces_org_default " - "ON workspaces (organization_id) WHERE is_default" - ).execute_if(dialect="postgresql"), -) - - -class User(Base): - """User model for authentication and profile management.""" - - __tablename__ = "users" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - email = Column(String(255), unique=True, nullable=False, index=True) - name = Column(String(255), nullable=True) - first_name = Column(String(255), nullable=True) - last_name = Column(String(255), nullable=True) - password_hash = Column(String(255), nullable=True) # Nullable for users created via invitation - external_id = Column(String(255), unique=True, nullable=True, index=True) - auth_provider = Column(String(50), nullable=True) - mfa_enabled = Column(Boolean, default=False, nullable=False) - last_login_at = Column(DateTime(timezone=True), nullable=True) - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - organization_memberships = relationship("OrganizationMember", back_populates="user", cascade="all, delete-orphan") - api_keys = relationship("APIKey", back_populates="user") - invitations = relationship("Invitation", back_populates="invited_user", foreign_keys="Invitation.invited_user_id") - refresh_tokens = relationship("RefreshToken", back_populates="user", cascade="all, delete-orphan") - - -class RefreshToken(Base): - """Opaque refresh token for extending local-password sessions.""" - - __tablename__ = "refresh_tokens" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True) - token_hash = Column(String(64), unique=True, nullable=False, index=True) - expires_at = Column(DateTime(timezone=True), nullable=False) - revoked_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - user = relationship("User", back_populates="refresh_tokens") - - -class OrganizationMember(Base): - """Organization membership with role.""" - - __tablename__ = "organization_members" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True) - role = Column(String, nullable=False, default=RoleEnum.READER.value) - - # User preferences for this organization - default_agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) - - joined_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Unique constraint: one membership per user per organization - __table_args__ = ( - UniqueConstraint('organization_id', 'user_id', name='uq_org_user'), - ) - - # Relationships - organization = relationship("Organization", back_populates="members") - user = relationship("User", back_populates="organization_memberships") - default_agent = relationship("Agent", foreign_keys=[default_agent_id]) - - -class Invitation(Base): - """Invitation model for inviting users to organizations.""" - - __tablename__ = "invitations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - invited_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Null if user doesn't exist yet - invited_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) - email = Column(String(255), nullable=False) # Email of invited user - role = Column(String, nullable=False, default=RoleEnum.READER.value) - status = Column(String, nullable=False, default=InvitationStatus.PENDING.value) - - - - token = Column(String(255), unique=True, nullable=False, index=True) # Invitation token - expires_at = Column(DateTime(timezone=True), nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - accepted_at = Column(DateTime(timezone=True), nullable=True) - - # Relationships - organization = relationship("Organization", back_populates="invitations") - invited_user = relationship("User", foreign_keys=[invited_user_id], back_populates="invitations") - invited_by = relationship("User", foreign_keys=[invited_by_id]) - - -class APIKey(Base): - """API Key model for authentication.""" - - __tablename__ = "api_keys" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - key = Column(String(255), unique=True, nullable=False, index=True) - name = Column(String(255), nullable=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Optional: link to user - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - last_used = Column(DateTime(timezone=True), nullable=True) - - # Relationships - organization = relationship("Organization", back_populates="api_keys") - user = relationship("User", back_populates="api_keys") - - -class AudioFile(Base): - """Audio file model.""" - - __tablename__ = "audio_files" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - filename = Column(String(255), nullable=False) - file_path = Column(String(512), nullable=False) - file_size = Column(Integer, nullable=False) # Size in bytes - duration = Column(Float, nullable=True) # Duration in seconds - sample_rate = Column(Integer, nullable=True) - channels = Column(Integer, nullable=True) - format = Column(String(10), nullable=False) # wav, mp3, flac, etc. - uploaded_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Relationships - evaluations = relationship("Evaluation", back_populates="audio_file") - - -class Evaluation(Base): - """Evaluation job model.""" - - __tablename__ = "evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every legacy audio evaluation belongs to a - # workspace within its org. Stamped from the X-Workspace-Id header - # (falling back to the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - audio_id = Column(UUID(as_uuid=True), ForeignKey("audio_files.id"), nullable=False) - reference_text = Column(String, nullable=True) # For WER calculation - evaluation_type = Column(String, nullable=False) - model_name = Column(String(100), nullable=True) - status = Column(String, default=EvaluationStatus.PENDING.value, nullable=False) - - - - metrics_requested = Column(JSON, nullable=True) # List of requested metrics - created_at = Column(DateTime(timezone=True), server_default=func.now()) - started_at = Column(DateTime(timezone=True), nullable=True) - completed_at = Column(DateTime(timezone=True), nullable=True) - error_message = Column(String, nullable=True) - - # Relationships - audio_file = relationship("AudioFile", back_populates="evaluations") - result = relationship("EvaluationResult", back_populates="evaluation", uselist=False) - - -class EvaluationResult(Base): - """Evaluation result model.""" - - __tablename__ = "evaluation_results" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column(UUID(as_uuid=True), ForeignKey("evaluations.id"), nullable=False, unique=True) - # Workspace isolation: mirrors the parent Evaluation's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - transcript = Column(String, nullable=True) - metrics = Column(JSON, nullable=False) # {"wer": 0.05, "latency_ms": 1250, ...} - raw_output = Column(JSON, nullable=True) # Full model output - processing_time = Column(Float, nullable=True) # Processing time in seconds - model_used = Column(String(100), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Relationships - evaluation = relationship("Evaluation", back_populates="result") - - -# ============================================ -# VAIOPS MODELS - Voice AI Ops -# ============================================ - -# Enums moved to enums.py - - -class Agent(Base): - """Test Agent - The voice AI agent being evaluated""" - __tablename__ = "agents" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - agent_id = Column(String(6), unique=True, nullable=True, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every agent belongs to a workspace within its - # org. Stamped from the X-Workspace-Id header (falling back to the - # org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String, nullable=False) - phone_number = Column(String, nullable=True) # Optional, required only for phone_call - language = Column(String, nullable=False, default=LanguageEnum.ENGLISH.value) - description = Column(String) - provider_prompt = Column(Text, nullable=True) - provider_prompt_synced_at = Column(DateTime(timezone=True), nullable=True) - call_type = Column(String, nullable=False, default=CallTypeEnum.OUTBOUND.value) - call_medium = Column(String, nullable=False, default=CallMediumEnum.PHONE_CALL.value) - telephony_phone_number_id = Column( - UUID(as_uuid=True), - ForeignKey("telephony_phone_numbers.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - - - - # Voice configuration - either voice_bundle_id OR ai_provider_id OR voice_ai_integration_id (mutually exclusive) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True, index=True) - ai_provider_id = Column(UUID(as_uuid=True), ForeignKey("aiproviders.id"), nullable=True, index=True) - - # Voice AI agent integration (Retell, Vapi, etc.) - voice_ai_integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id"), nullable=True, index=True) - voice_ai_agent_id = Column(String, nullable=True) # Agent ID from the external provider (Retell/Vapi) - prompt_variables = Column(JSON, nullable=True) - silence_hangup_secs = Column(Integer, nullable=False, server_default="15") - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -class Persona(Base): - """Persona - TTS provider-tied voice identity for testing""" - __tablename__ = "personas" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every persona belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String, nullable=False) - gender = Column(String, nullable=False, default=GenderEnum.NEUTRAL.value) - tts_provider = Column(String(100), nullable=True) - tts_voice_id = Column(String(255), nullable=True) - tts_voice_name = Column(String(255), nullable=True) - is_custom = Column(Boolean, default=False) - description = Column(Text, nullable=True) - tts_config = Column(JSON, nullable=True) - llm_temperature = Column(Float, nullable=True) - llm_max_tokens = Column(Integer, nullable=True) - response_delay_ms = Column(Integer, nullable=True) - max_turns = Column(Integer, nullable=True) - allow_interruptions = Column(Boolean, nullable=True) - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -class Scenario(Base): - """Scenario - The conversation scenario/test case""" - __tablename__ = "scenarios" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every scenario belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) - name = Column(String, nullable=False) - description = Column(String) - required_info = Column(JSON) - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -# Enums moved to enums.py - - -class Integration(Base): - """Integration model for connecting with external voice AI platforms.""" - __tablename__ = "integrations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - platform = Column(String, nullable=False) - - - - name = Column(String, nullable=True) # Optional friendly name - api_key = Column(String, nullable=False) # Encrypted Private API key for the platform - public_key = Column(String, nullable=True) # Optional Public API key (e.g. for Vapi) - is_active = Column(Boolean, default=True, nullable=False) - # Multiple credentials per (org, platform) are allowed. is_default marks - # the row used when a caller does not explicitly select a credential. - # A partial unique index in migration 028 enforces at most one default - # per (org, platform) at the DB level. - is_default = Column(Boolean, default=False, nullable=False) - # inherit | gateway | direct — per-credential LLM routing override - routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated - - -class ManualTranscription(Base): - """Manual transcription model for storing transcriptions from S3 audio files.""" - - __tablename__ = "manual_transcriptions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - name = Column(String(255), nullable=True) # User-friendly name for the transcription - audio_file_key = Column(String(512), nullable=False) # S3 key or file path - transcript = Column(String, nullable=False) # Full transcript text - speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] - stt_model = Column(String(100), nullable=True) # STT model used (e.g., "whisper-1", "google-speech-v2") - stt_provider = Column(String, nullable=True) # Provider used - - - - language = Column(String(10), nullable=True) # Detected or specified language - processing_time = Column(Float, nullable=True) # Processing time in seconds - raw_output = Column(JSON, nullable=True) # Full model output for reference - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class ConversationEvaluation(Base): - """Conversation evaluation model for evaluating manual transcriptions against agent objectives.""" - - __tablename__ = "conversation_evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - transcription_id = Column(UUID(as_uuid=True), ForeignKey("manual_transcriptions.id"), nullable=False, index=True) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) - - # Evaluation results - objective_achieved = Column(Boolean, nullable=False) # Binary: was the conversation objective achieved? - objective_achieved_reason = Column(String, nullable=True) # Explanation for the binary result - additional_metrics = Column(JSON, nullable=True) # Additional evaluation metrics (e.g., professionalism, clarity, etc.) - overall_score = Column(Float, nullable=True) # Overall score (0.0 to 1.0) - - # LLM metadata - llm_provider = Column(Enum(ModelProvider, native_enum=False), nullable=True) - - llm_model = Column(String(100), nullable=True) - llm_response = Column(JSON, nullable=True) # Full LLM response for reference - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class AIProvider(Base): - """AI Provider - Stores API keys for different AI platforms.""" - __tablename__ = "aiproviders" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String, nullable=False) - - - - api_key = Column(String, nullable=False) # Encrypted API key - name = Column(String, nullable=True) # Optional friendly name - # Azure OpenAI resource endpoint (e.g. https://my-resource.openai.azure.com). - # Only used when provider is azure; other providers ignore this column. - endpoint_url = Column(String, nullable=True) - is_active = Column(Boolean, default=True, nullable=False) - # Multiple AIProvider rows per (org, provider) are allowed. is_default - # marks the row resolved when no explicit credential id is selected. - # A partial unique index in migration 028 enforces at most one default. - is_default = Column(Boolean, default=False, nullable=False) - # inherit | gateway | direct — per-credential LLM routing override - routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") - # Bifrost custom model ID used when routing via gateway - gateway_model = Column(String(255), nullable=True) - # inherit | litellm_shim | native_openai — Bifrost API surface override - gateway_interface = Column(String(20), nullable=False, default="inherit", server_default="inherit") - # Optional per-credential Bifrost/gateway base URL override - gateway_base_url = Column(String(512), nullable=True) - # Optional auth header for Bifrost (e.g. x-bf-vk, Authorization, x-api-key) - gateway_auth_header = Column(String(64), nullable=True) - # Env var name whose value is sent as the gateway auth secret - gateway_auth_secret_env = Column(String(128), nullable=True) - # Encrypted inline gateway auth secret (alternative to env var) - gateway_auth_secret = Column(String, nullable=True) - # Arbitrary HTTP headers sent with gateway-routed LiteLLM calls - gateway_extra_headers = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated - - -# Enums moved to enums.py - - -class VoiceBundle(Base): - """VoiceBundle - Composable unit combining STT, LLM, and TTS for voice AI testing, or S2S models.""" - __tablename__ = "voicebundles" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - name = Column(String, nullable=False) - description = Column(String, nullable=True) - - # Bundle type: either STT+LLM+TTS or S2S - # Using String instead of Enum to avoid SQLAlchemy enum conversion issues - # The enum conversion is handled in the Pydantic schemas - bundle_type = Column(String(50), nullable=False, default=VoiceBundleType.STT_LLM_TTS.value) - - # STT Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - stt_provider = Column(String, nullable=True) - # Optional explicit credential row (aiproviders.id or integrations.id). - # When NULL the credential resolver picks the default row for the - # provider. No FK is set because the target table varies by provider. - stt_credential_id = Column(UUID(as_uuid=True), nullable=True) - - stt_model = Column(String, nullable=True) # e.g., "whisper-1", "google-speech-v2" - - # LLM Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - llm_provider = Column(String, nullable=True) - llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - - llm_model = Column(String, nullable=True) # e.g., "gpt-4", "claude-3-opus" - llm_temperature = Column(Float, nullable=True, default=0.7) - llm_max_tokens = Column(Integer, nullable=True) - llm_config = Column(JSON, nullable=True) # Additional LLM configuration (extensible) - - # TTS Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - tts_provider = Column(String, nullable=True) - tts_credential_id = Column(UUID(as_uuid=True), nullable=True) - - tts_model = Column(String, nullable=True) # e.g., "tts-1", "neural-voice" - tts_voice = Column(String, nullable=True) # Voice selection if applicable - tts_config = Column(JSON, nullable=True) # Additional TTS configuration (extensible) - - # S2S Configuration - required for S2S type, optional for STT_LLM_TTS - s2s_provider = Column(String, nullable=True) - s2s_credential_id = Column(UUID(as_uuid=True), nullable=True) - - - - s2s_model = Column(String, nullable=True) # e.g., "gpt-4o-transcribe", speech-to-speech model - s2s_config = Column(JSON, nullable=True) # Additional S2S configuration (extensible) - - # Additional configuration for extensibility - extra_metadata = Column(JSON, nullable=True) # For future extensions (renamed from 'metadata' to avoid SQLAlchemy conflict) - - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -# Enums moved to enums.py - - -class TestAgentConversation(Base): - """Test Agent Conversation - Records conversations between test AI agent and voice AI agent.""" - __tablename__ = "test_agent_conversations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every playground conversation belongs to a - # workspace within its org. Stamped from the X-Workspace-Id header - # (falling back to the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Configuration - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=False) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) - - # Conversation data - status = Column(String, nullable=False, default=TestAgentConversationStatus.INITIALIZING.value) - - - - live_transcription = Column(JSON, nullable=True) # Array of conversation turns with timestamps - conversation_audio_key = Column(String, nullable=True) # S3 key for recorded conversation audio - full_transcript = Column(String, nullable=True) # Full conversation transcript - - # Metadata - started_at = Column(DateTime(timezone=True), server_default=func.now()) - ended_at = Column(DateTime(timezone=True), nullable=True) - duration_seconds = Column(Float, nullable=True) - - # Additional metadata - conversation_metadata = Column(JSON, nullable=True) # Additional conversation metadata - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class EvaluatorSuite(Base): - """Evaluator suite — one agent + one persona + N scenario combinations.""" - - __tablename__ = "evaluator_suites" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - name = Column(String, nullable=True) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) - metric_ids = Column(JSON, nullable=True) - llm_provider = Column(String, nullable=True) - llm_model = Column(String, nullable=True) - llm_config = Column(JSON, nullable=True) - tags = Column(JSON, nullable=True) - default_runs_per_combination = Column(Integer, nullable=False, default=1) - round_robin_index = Column(Integer, nullable=False, default=0) - is_active = Column(Boolean, nullable=False, default=False) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class Evaluator(Base): - """Evaluator - Configuration for testing agents with specific persona and scenario combinations, or custom prompt evaluators.""" - __tablename__ = "evaluators" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluator_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every evaluator belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Display name (required for custom evaluators, optional for standard) - name = Column(String, nullable=True) - - # Parent suite (nullable for legacy/custom evaluators) - suite_id = Column( - UUID(as_uuid=True), - ForeignKey("evaluator_suites.id", ondelete="CASCADE"), - nullable=True, - index=True, - ) - - # Standard evaluator configuration (nullable for custom evaluators) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) - - # Custom evaluator prompt (used instead of agent/persona/scenario) - custom_prompt = Column(Text, nullable=True) - - # Custom evaluator metric selection. When set, the worker filters the - # enabled-org metrics down to only these IDs (list of metric UUID strings). - # Standard evaluators leave this NULL and use all enabled agent metrics. - metric_ids = Column(JSON, nullable=True) - - # LLM configuration for evaluation (overrides hardcoded defaults) - llm_provider = Column(String, nullable=True) # e.g. "openai", "anthropic", "google" - llm_model = Column(String, nullable=True) # e.g. "gpt-4.1", "claude-sonnet-4-20250514" - llm_config = Column(JSON, nullable=True) - - # Tags for categorization - tags = Column(JSON, nullable=True) # Array of tag strings - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -# Enums moved to enums.py - - -class Metric(Base): - """Metric - Configuration for evaluation metrics. - - Supports a 2-level hierarchy via ``parent_metric_id``: a "category" - parent metric (e.g. "Call Outcome") owns N child sub-metric labels - (e.g. "happy_completion", "angry_hangup"). ``selection_mode`` is set - only on parents and controls how the LLM scores children together - (``single_choice`` = pick exactly one; ``multi_label`` = independent - yes/no with logical consistency). Children are always boolean. - """ - __tablename__ = "metrics" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: two-shape column. - # - # * ``workspace_id = `` — workspace-scoped metric. Only - # visible inside that workspace (the default behavior; existing - # rows all look like this). - # * ``workspace_id IS NULL`` — org-shared metric. Surfaces in - # every workspace's listing under this org so users don't have - # to recreate the same metric per workspace. - # - # Children always inherit their parent's ``workspace_id`` (including - # NULL) so a category metric's whole subtree shares one scope; the - # add-child / promote-discovered endpoints enforce this. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=True, - index=True, - ) - - # Basic information - name = Column(String, nullable=False) - description = Column(String, nullable=True) - # Free-form illustrative example used to sharpen the LLM judge's - # rubric. Today this is consumed by child sub-labels of a - # categorization parent metric so each label can carry "what does - # this look like in a transcript?" text alongside the rubric in - # ``description``. The column lives on every Metric row for - # forward-compat: a standalone metric could later surface its own - # example without another migration. - example = Column(Text, nullable=True) - - # Configuration - metric_type = Column(String, nullable=False, default=MetricType.RATING.value) - metric_category = Column( - String(30), - nullable=False, - default=MetricCategory.QUALITY.value, - server_default=MetricCategory.QUALITY.value, - ) - trigger = Column(String, nullable=False, default=MetricTrigger.ALWAYS.value) - metric_origin = Column(String(30), nullable=False, default="default") - supported_surfaces = Column(JSON, nullable=False, default=list) # ["agent", "voice_playground", "blind_test"] - enabled_surfaces = Column(JSON, nullable=False, default=list) # subset of supported_surfaces - custom_data_type = Column(String(30), nullable=True) # "boolean" | "enum" | "number_range" - custom_config = Column(JSON, nullable=True) # enum options / number range config - tags = Column(JSON, nullable=True) # ["tone", "latency", ...] - - # Hierarchy: NULL = standalone or parent. When set, this row is a - # child sub-metric of the referenced parent. ON DELETE CASCADE so - # deleting a category removes its children atomically. - parent_metric_id = Column( - UUID(as_uuid=True), - ForeignKey("metrics.id", ondelete="CASCADE"), - nullable=True, - index=True, - ) - # Set only on parent rows (``parent_metric_id IS NULL``). Either - # ``single_choice`` or ``multi_label``. NULL = legacy / non-hierarchical - # metric (no children). - selection_mode = Column(String(20), nullable=True) - - # When true on a parent metric (any selection_mode), the LLM is - # invited during call-import evaluation to emit additional - # candidate sub-labels beyond the user-defined children. The - # candidates surface in a "Discovered labels" panel where the user - # manually promotes them into real child Metric rows. For - # ``single_choice`` parents the discovered entries are - # supplemental — the chosen child is still picked from the - # predefined children so the exactly-one-true invariant holds. - # The validator rejects this flag on standalone / child metrics. - allow_discovery = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - - # When True, this metric is a "transcript-compare judge": the - # call-import evaluator feeds BOTH the production transcript - # (``call_import_rows.transcript``, CSV-supplied) and the diarised - # transcript (``call_import_rows.diarised_transcript``, worker- - # produced by the STT/diarisation pipeline) to the LLM as a - # labeled pair instead of feeding one transcript. The parent - # evaluation's ``CallImportEvaluation.transcript_source`` is - # ignored for these metrics — they always read both columns. - # Rows where either transcript is missing are skipped per-metric - # with ``skipped="comparison_missing_transcript"`` so the rest of - # the row's metrics still produce scores. The Pydantic validator - # rejects ``compare_transcripts`` combined with ``parent_metric_id`` - # or ``selection_mode`` (i.e. it can't simultaneously be part of - # a parent/child hierarchy). The call-import worker also - # auto-promotes a metric to comparison mode when its description - # references the production / diarised transcripts in well-known - # phrases (see ``_metric_text_references_production`` in - # ``app.workers.tasks.evaluate_call_import_row``). - compare_transcripts = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - - parent = relationship( - "Metric", - remote_side=[id], - backref="children", - ) - - # When true, the LLM-judge is asked to also return a short free-form - # rationale alongside the value (stored under ``metric_scores[id].rationale``). - # Adds a second " - LLM Rationale" column in the call-import CSV export. - capture_rationale = Column(Boolean, nullable=False, default=False) - - enabled = Column(Boolean, nullable=False, default=True) - - # Metadata - is_default = Column(Boolean, nullable=False, default=False) # Pre-defined metrics - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class EvaluatorResult(Base): - """EvaluatorResult - Results from running an evaluator with transcription and metric evaluations.""" - __tablename__ = "evaluator_results" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - result_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every evaluator result belongs to a workspace - # within its org. Stamped from the active workspace at creation time - # (either the X-Workspace-Id header or the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # References - evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True, index=True) # Optional - can be None for test calls without persona/scenario - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Nullable for custom evaluators - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) # Optional - can be None for test calls - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) # Optional - can be None for test calls - - # Result data - name = Column(String, nullable=True) # Scenario name or test call name (optional) - timestamp = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - duration_seconds = Column(Float, nullable=True) # Call duration - status = Column(String(20), nullable=False, default=EvaluatorResultStatus.QUEUED.value) - - # Audio and transcription - audio_s3_key = Column(String, nullable=True) # S3 key for audio file - transcription = Column(String, nullable=True) # Full transcription - speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] - - # Metric scores - JSON object with metric_id as key and score as value - # Format: {"metric_id_1": {"value": 85, "type": "rating"}, "metric_id_2": {"value": true, "type": "boolean"}} - metric_scores = Column(JSON, nullable=True) - - # Celery task tracking - celery_task_id = Column(String, nullable=True, index=True) # Celery task ID for tracking - - # Error information - error_message = Column(String, nullable=True) - - # Call event tracking (similar to CallRecording) - call_event = Column(String, nullable=True, index=True) # Latest call event (e.g., call_started, call_ended) - provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) - provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" - call_data = Column(JSON, nullable=True) # Full call details from provider (like CallRecording) - - # Data-plane shard routing (payload rows on shard DBs when sharding enabled) - shard_id = Column(String(64), nullable=True, index=True) - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -# Enums moved to enums.py - - -class CallRecordingSource(str, enum.Enum): - """Source of the call recording data.""" - - PLAYGROUND = "playground" - WEBHOOK = "webhook" - - -class CallRecording(Base): - """Call Recording model for tracking voice provider calls.""" - __tablename__ = "call_recordings" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every recording belongs to a workspace within - # its org. For playground-origin rows this is stamped from the active - # workspace at creation time; for webhook-origin rows the worker - # looks up the recording's agent and inherits its workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - call_short_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - status = Column(Enum(CallRecordingStatus), nullable=False, default=CallRecordingStatus.PENDING, index=True) - call_event = Column(String, nullable=True, index=True) # Latest webhook event (e.g., call_started, call_ended) - source = Column(Enum(CallRecordingSource), nullable=False, default=CallRecordingSource.PLAYGROUND, index=True) - call_data = Column(JSON, nullable=True) # JSON blob for provider response - provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) - provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Reference to our agent - - # Link to EvaluatorResult for metric evaluations - evaluator_result_id = Column( - UUID(as_uuid=True), - ForeignKey("evaluator_results.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - shard_id = Column(String(64), nullable=True, index=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class EvaluatorResultPayload(Base): - """Heavy evaluator result fields stored on data shards when sharding is enabled.""" - - __tablename__ = "evaluator_result_payloads" - - evaluator_result_id = Column(UUID(as_uuid=True), primary_key=True) - workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) - audio_s3_key = Column(String, nullable=True) - transcription = Column(String, nullable=True) - speaker_segments = Column(JSON, nullable=True) - metric_scores = Column(JSON, nullable=True) - call_data = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class CallRecordingPayload(Base): - """Heavy call recording fields stored on data shards when sharding is enabled.""" - - __tablename__ = "call_recording_payloads" - - call_recording_id = Column(UUID(as_uuid=True), primary_key=True) - workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) - call_data = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class Alert(Base): - """Alert model for configuring monitoring alerts.""" - __tablename__ = "alerts" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - - # Basic information - name = Column(String(255), nullable=False) - description = Column(String, nullable=True) - - # Metric condition configuration - metric_type = Column(String, nullable=False, default=AlertMetricType.NUMBER_OF_CALLS.value) - aggregation = Column(String, nullable=False, default=AlertAggregation.SUM.value) - operator = Column(String, nullable=False, default=AlertOperator.GREATER_THAN.value) - threshold_value = Column(Float, nullable=False) - time_window_minutes = Column(Integer, nullable=False, default=60) # Time window for aggregation - - # Agent selection (JSON array of agent UUIDs, null means all agents) - agent_ids = Column(JSON, nullable=True) - - # Notification configuration - notify_frequency = Column(String, nullable=False, default=AlertNotifyFrequency.IMMEDIATE.value) - notify_emails = Column(JSON, nullable=True) # Array of email addresses - notify_webhooks = Column(JSON, nullable=True) # Array of webhook URLs (Slack, etc.) - - # Status - status = Column(String, nullable=False, default=AlertStatus.ACTIVE.value) - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - # Relationships - alert_history = relationship("AlertHistory", back_populates="alert", cascade="all, delete-orphan") - - -class AlertHistory(Base): - """Alert history model for tracking triggered alerts.""" - __tablename__ = "alert_history" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - alert_id = Column(UUID(as_uuid=True), ForeignKey("alerts.id"), nullable=False, index=True) - - # Trigger information - triggered_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - triggered_value = Column(Float, nullable=False) # The actual value that triggered the alert - threshold_value = Column(Float, nullable=False) # The threshold at time of trigger - - # Status tracking - status = Column(String, nullable=False, default=AlertHistoryStatus.TRIGGERED.value) - - # Notification tracking - notified_at = Column(DateTime(timezone=True), nullable=True) - notification_details = Column(JSON, nullable=True) # Details of sent notifications - - # Resolution - acknowledged_at = Column(DateTime(timezone=True), nullable=True) - acknowledged_by = Column(String, nullable=True) - resolved_at = Column(DateTime(timezone=True), nullable=True) - resolved_by = Column(String, nullable=True) - resolution_notes = Column(String, nullable=True) - - # Additional context - context_data = Column(JSON, nullable=True) # Additional data about the trigger - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - alert = relationship("Alert", back_populates="alert_history") - - -class CronJob(Base): - """Cron job model for scheduling automated evaluator runs.""" - __tablename__ = "cron_jobs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - - # Basic information - name = Column(String(255), nullable=False) - cron_expression = Column(String(100), nullable=False) # e.g., "0 9 * * 1-5" - timezone = Column(String(100), nullable=False, default="UTC") - - # Run configuration - max_runs = Column(Integer, nullable=False, default=10) - current_runs = Column(Integer, nullable=False, default=0) - - # Evaluators to trigger (JSON array of evaluator UUIDs) - evaluator_ids = Column(JSON, nullable=False) - - # Status - status = Column(String, nullable=False, default=CronJobStatus.ACTIVE.value) - - # Run tracking - next_run_at = Column(DateTime(timezone=True), nullable=True) - last_run_at = Column(DateTime(timezone=True), nullable=True) - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class TTSComparisonStatus(str, enum.Enum): - PENDING = "pending" - GENERATING = "generating" - EVALUATING = "evaluating" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSSampleStatus(str, enum.Enum): - PENDING = "pending" - GENERATING = "generating" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSReportJobStatus(str, enum.Enum): - PENDING = "pending" - PROCESSING = "processing" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSComparison(Base): - """TTS Comparison session for A/B testing voice providers.""" - __tablename__ = "tts_comparisons" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every voice playground comparison belongs to - # a workspace within its org. Children (samples, report jobs, blind - # test shares) inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - simulation_id = Column(String(6), unique=True, index=True, nullable=True) - - name = Column(String(255), nullable=True) - status = Column(String(50), nullable=False, default=TTSComparisonStatus.PENDING.value) - - # 'benchmark' = traditional TTS A/B benchmark (provider-generated audio). - # 'blind_test_only' = standalone blind test built from existing recordings - # / uploads / past TTS samples; no TTS generation happens. - mode = Column(String(32), nullable=False, default="benchmark") - - provider_a = Column(String(100), nullable=True) - model_a = Column(String(100), nullable=True) - voices_a = Column(JSON, nullable=True) - - provider_b = Column(String(100), nullable=True) - model_b = Column(String(100), nullable=True) - voices_b = Column(JSON, nullable=True) - - sample_texts = Column(JSON, nullable=False) - num_runs = Column(Integer, nullable=False, default=1) - - blind_test_results = Column(JSON, nullable=True) - evaluation_summary = Column(JSON, nullable=True) - - eval_stt_provider = Column(String(100), nullable=True) - eval_stt_model = Column(String(100), nullable=True) - - celery_task_id = Column(String, nullable=True, index=True) - error_message = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - samples = relationship("TTSSample", back_populates="comparison", cascade="all, delete-orphan") - - -class TTSSample(Base): - """Individual TTS audio sample within a comparison.""" - __tablename__ = "tts_samples" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - provider = Column(String(100), nullable=True) - model = Column(String(100), nullable=True) - voice_id = Column(String(255), nullable=True) - voice_name = Column(String(255), nullable=True) - side = Column(String(1), nullable=True) # "A" or "B" - sample_index = Column(Integer, nullable=False) - run_index = Column(Integer, nullable=False, default=0) - - # 'tts' (default, audio is synthesized by a provider), 'recording' (audio - # is reused from a CallImportRow recording), or 'upload' (audio was - # uploaded by the user). Non-tts samples are marked completed up-front - # by the API and skipped by the generation worker. - source_type = Column(String(32), nullable=False, default="tts") - # When source_type == 'recording', references CallImportRow.id (no FK - # constraint to keep cascading deletes simple if a call import is later - # removed; the audio_s3_key is what's actually used). - source_ref_id = Column(UUID(as_uuid=True), nullable=True) - - text = Column(String, nullable=False) - audio_s3_key = Column(String(512), nullable=True) - duration_seconds = Column(Float, nullable=True) - latency_ms = Column(Float, nullable=True) - ttfb_ms = Column(Float, nullable=True) - - evaluation_metrics = Column(JSON, nullable=True) - status = Column(String(50), nullable=False, default=TTSSampleStatus.PENDING.value) - error_message = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - comparison = relationship("TTSComparison", back_populates="samples") - - -class TTSReportJob(Base): - """Asynchronous PDF report generation jobs for Voice Playground.""" - __tablename__ = "tts_report_jobs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) - - status = Column(String(50), nullable=False, default=TTSReportJobStatus.PENDING.value) - format = Column(String(20), nullable=False, default="pdf") - filename = Column(String(255), nullable=True) - s3_key = Column(String(512), nullable=True) - error_message = Column(String, nullable=True) - celery_task_id = Column(String, nullable=True, index=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - comparison = relationship("TTSComparison") - - -class TTSBlindTestShareStatus(str, enum.Enum): - OPEN = "open" - CLOSED = "closed" - - -class TTSBlindTestShare(Base): - """A publicly sharable blind test for a TTSComparison. - - The share_token is the capability: anyone holding it can open the public - form and submit a response. Each comparison has at most one share row. - """ - __tablename__ = "tts_blind_test_shares" - __table_args__ = ( - UniqueConstraint("comparison_id", name="uq_blind_test_shares_comparison"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - comparison_id = Column( - UUID(as_uuid=True), - ForeignKey("tts_comparisons.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - share_token = Column(String(64), unique=True, nullable=False, index=True) - - title = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - - # Internal notes visible only to the share creator (e.g. which voice - # corresponds to which side, source notes for standalone blind tests). - # Never exposed via the public blind test payload. - creator_notes = Column(Text, nullable=True) - - # JSON list: [{ "key": str, "label": str, "type": "rating"|"comment", "scale": int? }] - custom_metrics = Column(JSON, nullable=False) - - status = Column(String(20), nullable=False, default=TTSBlindTestShareStatus.OPEN.value) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - closed_at = Column(DateTime(timezone=True), nullable=True) - created_by = Column(String, nullable=True) - - comparison = relationship("TTSComparison") - responses = relationship( - "TTSBlindTestResponse", - back_populates="share", - cascade="all, delete-orphan", - ) - - -class TTSBlindTestResponse(Base): - """A single rater's submission against a TTSBlindTestShare.""" - __tablename__ = "tts_blind_test_responses" - __table_args__ = ( - UniqueConstraint("share_id", "rater_email", name="uq_blind_test_response_share_email"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - share_id = Column( - UUID(as_uuid=True), - ForeignKey("tts_blind_test_shares.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent TTSBlindTestShare's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - rater_name = Column(String(255), nullable=False) - rater_email = Column(String(320), nullable=False, index=True) - - # JSON list keyed by sample_index. Server stores in TRUE A/B orientation - # (already de-flipped from whatever the rater's UI showed): - # [{ - # "sample_index": int, - # "preferred": "A" | "B", - # "ratings_a": { metric_key: number }, - # "ratings_b": { metric_key: number }, - # "comment": str? - # }] - responses = Column(JSON, nullable=False) - - ip = Column(String(64), nullable=True) - user_agent = Column(String(512), nullable=True) - - submitted_at = Column(DateTime(timezone=True), server_default=func.now()) - - share = relationship("TTSBlindTestShare", back_populates="responses") - - -class PromptPartial(Base): - """Prompt Partial - Reusable prompt templates with version history.""" - __tablename__ = "prompt_partials" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every prompt partial belongs to a workspace - # within its org. Versions inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(String, nullable=True) - content = Column(Text, nullable=False) - tags = Column(JSON, nullable=True) - current_version = Column(Integer, nullable=False, default=1) - # Cached LLM-generated flowchart for imported production agent prompts. - # Shape: AgentFlowGraph JSON (nodes[], edges[]). - agent_flowchart = Column(JSON, nullable=True) - agent_flowchart_status = Column(String(20), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - versions = relationship("PromptPartialVersion", back_populates="prompt_partial", cascade="all, delete-orphan", order_by="PromptPartialVersion.version.desc()") - - -class PromptPartialVersion(Base): - """Version history for a prompt partial.""" - __tablename__ = "prompt_partial_versions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - prompt_partial_id = Column(UUID(as_uuid=True), ForeignKey("prompt_partials.id", ondelete="CASCADE"), nullable=False, index=True) - # Workspace isolation: mirrors the parent PromptPartial's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - version = Column(Integer, nullable=False) - content = Column(Text, nullable=False) - change_summary = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - created_by = Column(String, nullable=True) - - prompt_partial = relationship("PromptPartial", back_populates="versions") - - __table_args__ = ( - UniqueConstraint('prompt_partial_id', 'version', name='uq_prompt_partial_version'), - ) - - -class CustomTTSVoice(Base): - """Organization-scoped custom TTS voice metadata.""" - __tablename__ = "custom_tts_voices" - __table_args__ = ( - UniqueConstraint("organization_id", "provider", "voice_id", name="uq_custom_tts_voice_org_provider_voice_id"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String(100), nullable=False, index=True) - voice_id = Column(String(255), nullable=False) - name = Column(String(255), nullable=False) - gender = Column(String(50), nullable=True) - accent = Column(String(100), nullable=True) - description = Column(Text, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - -class PromptOptimizationRun(Base): - """A single GEPA prompt optimization run for an agent.""" - __tablename__ = "prompt_optimization_runs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every optimization run belongs to a workspace - # within its org. Candidates inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) - evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) - - seed_prompt = Column(Text, nullable=False) - best_prompt = Column(Text, nullable=True) - best_score = Column(Float, nullable=True) - - status = Column(String(20), nullable=False, default=PromptOptimizationStatus.PENDING.value) - config = Column(JSON, nullable=True) - reflection_trace = Column(JSON, nullable=True) - metric_history = Column(JSON, nullable=True) - - num_iterations = Column(Integer, nullable=True) - num_metric_calls = Column(Integer, nullable=True) - - celery_task_id = Column(String, nullable=True, index=True) - error_message = Column(Text, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - candidates = relationship("PromptOptimizationCandidate", back_populates="optimization_run", cascade="all, delete-orphan") - - -class PromptOptimizationCandidate(Base): - """A candidate prompt generated during an optimization run.""" - __tablename__ = "prompt_optimization_candidates" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - optimization_run_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_runs.id", ondelete="CASCADE"), nullable=False, index=True) - # Workspace isolation: mirrors the parent PromptOptimizationRun's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - prompt_text = Column(Text, nullable=False) - score = Column(Float, nullable=True) - metric_breakdown = Column(JSON, nullable=True) - reflection_summary = Column(Text, nullable=True) - - parent_candidate_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_candidates.id"), nullable=True) - - is_accepted = Column(Boolean, nullable=False, default=False) - pushed_to_provider_at = Column(DateTime(timezone=True), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - optimization_run = relationship("PromptOptimizationRun", back_populates="candidates") - - -class TelephonyIntegration(Base): - """Per-organization telephony provider credentials and configuration. - - Multiple rows per (organization_id, provider) are allowed so that an - organization can keep several Plivo / Exotel accounts side-by-side. - A partial unique index in migration 028 enforces at most one row with - is_default = TRUE per (org, provider); resolution falls back to that - default row when the caller does not pin a specific credential. - """ - - __tablename__ = "telephony_integrations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String(50), nullable=False, default="plivo") - name = Column(String(255), nullable=True) # Optional friendly name to disambiguate multiple credentials - - auth_id = Column(String(255), nullable=False) - auth_token = Column(String(512), nullable=False) - - verify_app_uuid = Column(String(255), nullable=True) - voice_app_id = Column(String(255), nullable=True) - sip_domain = Column(String(255), nullable=True) - masking_config = Column(JSON, nullable=True) - - is_active = Column(Boolean, default=True, nullable=False) - is_default = Column(Boolean, default=False, nullable=False) - last_tested_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyPhoneNumber(Base): - """Inventory of telephony phone numbers owned by an organization.""" - - __tablename__ = "telephony_phone_numbers" - __table_args__ = ( - UniqueConstraint("organization_id", "phone_number", name="uq_telephony_number_org_phone"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - telephony_integration_id = Column( - UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=True, index=True - ) - - phone_number = Column(String(20), nullable=False, index=True) - country_iso2 = Column(String(2), nullable=True) - region = Column(String(100), nullable=True) - number_type = Column(String(20), nullable=True) - capabilities = Column(JSON, nullable=True) - provider_app_id = Column(String(255), nullable=True) - - is_masking_pool = Column(Boolean, default=False, nullable=False) - inbound_enabled = Column(Boolean, default=True, nullable=False) - outbound_enabled = Column(Boolean, default=True, nullable=False) - source = Column(String(20), nullable=False, default="imported") - agent_id = Column( - UUID(as_uuid=True), - ForeignKey( - "agents.id", - ondelete="SET NULL", - use_alter=True, - name="fk_telephony_phone_numbers_agent_id", - ), - nullable=True, - index=True, - ) - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyDialTarget(Base): - """Org-scoped saved destination numbers for outbound test calls.""" - - __tablename__ = "telephony_dial_targets" - __table_args__ = ( - UniqueConstraint("organization_id", "phone_number", name="uq_telephony_dial_target_org_phone"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - phone_number = Column(String(20), nullable=False, index=True) - label = Column(String(255), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyVerifySession(Base): - """Tracks voice OTP verification sessions via telephony provider.""" - - __tablename__ = "telephony_verify_sessions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider_session_uuid = Column(String(255), nullable=False, unique=True, index=True) - recipient_number = Column(String(20), nullable=False) - channel = Column(String(10), nullable=False, default="voice") - status = Column(String(20), nullable=False, default="pending") - initiated_by = Column(String(255), nullable=True) - verify_app_uuid = Column(String(255), nullable=True) - verified_at = Column(DateTime(timezone=True), nullable=True) - expires_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyMaskedSession(Base): - """Number-masking session between two parties through a middle number.""" - - __tablename__ = "telephony_masked_sessions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - telephony_integration_id = Column(UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=False) - masked_number_id = Column( - UUID(as_uuid=True), ForeignKey("telephony_phone_numbers.id"), nullable=False, index=True - ) - masked_number = Column(String(20), nullable=False) - party_a_number = Column(String(20), nullable=False) - party_b_number = Column(String(20), nullable=False) - status = Column(String(20), nullable=False, default="active") - expires_at = Column(DateTime(timezone=True), nullable=True) - ended_at = Column(DateTime(timezone=True), nullable=True) - session_metadata = Column("metadata", JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class CallImportSchema(Base): - """Reusable Input Parameter schema for the call-uploads flow. - - A schema is workspace-scoped: users define a named bundle of typed - Input Parameters once (e.g. "Standard Voice QA" with conversation_id + - recording_url + transcript + agent_name) and then map those parameters - to CSV/Excel headers each time they upload a new batch. - - Every schema MUST contain exactly one parameter with - ``type='conversation_id'`` and ``is_required=True`` - that's the - mandatory identity field every imported row needs. The invariant is - enforced in app code on create/update (no DB-level CHECK because the - parent + children are written across two tables in one transaction). - """ - - __tablename__ = "call_import_schemas" - __table_args__ = ( - # Case-insensitive uniqueness is enforced via the matching partial - # index on ``LOWER(name)`` in the migration; this constraint here - # would be case-sensitive and is intentionally omitted to avoid - # confusing the user. - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - created_by_user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="SET NULL"), - nullable=True, - ) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - parameters = relationship( - "CallImportSchemaParameter", - back_populates="schema", - cascade="all, delete-orphan", - order_by="CallImportSchemaParameter.ordering", - ) - - -class CallImportSchemaParameter(Base): - """A single typed parameter inside a :class:`CallImportSchema`. - - ``type`` is one of the strings tracked by - :data:`app.models.enums.CallImportParameterType`. ``conversation_id`` - is reserved for the mandatory identity parameter every schema must - contain. - """ - - __tablename__ = "call_import_schema_parameters" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - schema_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_schemas.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - type = Column(String(32), nullable=False) - description = Column(Text, nullable=True) - is_required = Column(Boolean, nullable=False, default=False) - # Stable ordering so the UI renders parameters in the order the - # schema author defined them (matters when conversation_id is pinned - # first and the user re-orders the rest). - ordering = Column(Integer, nullable=False, default=0) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - schema = relationship("CallImportSchema", back_populates="parameters") - - -class CallImport(Base): - """Batch record for a CSV-driven call import job.""" - - __tablename__ = "call_imports" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every imported batch belongs to a workspace - # within its org. The /upload endpoint stamps it from the active - # workspace header (or the org's Default if absent). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - created_by_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) - - # Telephony provider key (e.g. ``'exotel'``, ``'plivo'``). In the - # legacy one-shot ``POST /upload`` endpoint this is supplied with the - # file; in the three-stage flow (UPLOAD -> MAP -> IMPORT) the value - # isn't known until the IMPORT stage, so the column is nullable for - # ``uploaded`` / ``mapped`` batches. - provider = Column(String(50), nullable=True, default="exotel") - # Pin a specific telephony credential for this batch so the worker - # downloads recordings using *that* row instead of the org default. - # NULL preserves legacy behavior (resolve by provider + default). - telephony_integration_id = Column( - UUID(as_uuid=True), - ForeignKey("telephony_integrations.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - original_filename = Column(String(512), nullable=True) - # When the source file was a multi-sheet Excel workbook, this records - # the worksheet the rows came from (one batch per sheet). NULL for CSV - # uploads since CSV has no sheet concept. - sheet_name = Column(String(255), nullable=True) - - # --- Source-file staging (UPLOAD stage) --------------------------- - # The raw CSV / Excel file is stored in S3 between stages so the - # user can come back later to MAP and IMPORT without re-uploading. - # ``source_s3_key`` is NULL on legacy batches that were imported via - # the one-shot endpoint (those batches stay read-only post-import). - source_s3_key = Column(Text, nullable=True) - source_format = Column(String(16), nullable=True) - source_size_bytes = Column(BigInteger, nullable=True) - source_content_type = Column(String(255), nullable=True) - - # Snapshot of the file's sheets + headers captured at UPLOAD time - # so the MAP UI doesn't need to re-fetch the source bytes from S3. - # Shape: ``[{"name": str, "headers": [str, ...], "row_count": int}, ...]``. - available_sheets = Column(JSON, nullable=True) - - # User's explicit "drop these columns" decision captured at MAP - # time. Was validation-only and ephemeral in the legacy flow; now - # persisted so the IMPORT stage can re-parse the file with the same - # mapping/skip intent. - skipped_columns = Column(JSON, nullable=False, default=list) - # Rows skipped at parse time (missing/invalid conversation_id or URL). - # Shape: ``[{"source_row": int, "reason": str, "message": str}, ...]``. - source_row_skips = Column(JSON, nullable=False, default=list) - - # Free-text high-level segregation label. Powers the "Dataset" filter - # at the top of the imports page; multiple imports can share a value. - dataset = Column(String(255), nullable=True, index=True) - - # Reusable Input Parameter schema this batch was uploaded against. - # NULL on legacy batches uploaded before the schema-driven flow - # shipped; those still render via ``column_mapping`` + ``extra_columns`` - # + ``custom_column_mapping`` below. - schema_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_schemas.id", ondelete="RESTRICT"), - nullable=True, - index=True, - ) - # New schema-driven mapping: ``{schema_parameter_name: csv_header}``. - # Populated for new uploads; empty dict on legacy batches. - parameter_mapping = Column(JSON, nullable=False, default=dict) - - # Legacy free-form mapping (pre-schema-flow). Kept on the model so - # batches that were uploaded before the schema feature shipped still - # render correctly on the detail page; new uploads stop writing here. - # Keys: external_call_id (required), transcript, recording_url. - # (DB column ``external_call_id`` is now ``conversation_id``; this - # JSON key stays as-is for historical batches.) - # Values: original CSV header strings (preserve user casing for export). - column_mapping = Column(JSON, nullable=False, default=dict) - # Ordered list of additional CSV header strings the uploader wants - # preserved verbatim into the evaluation export CSV. - extra_columns = Column(JSON, nullable=False, default=list) - # User-defined ``{custom_field_name: csv_header}`` mappings on top of - # the three system fields above. Cells from the mapped CSV columns are - # preserved per row (keyed by the CSV header in ``raw_columns``) and - # surface in the evaluation export under the uploader-chosen name. - custom_column_mapping = Column(JSON, nullable=False, default=dict) - - total_rows = Column(Integer, nullable=False, default=0) - completed_rows = Column(Integer, nullable=False, default=0) - failed_rows = Column(Integer, nullable=False, default=0) - - status = Column( - Enum(CallImportStatus, values_callable=get_enum_values), - nullable=False, - default=CallImportStatus.PENDING, - index=True, - ) - error_message = Column(Text, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - rows = relationship( - "CallImportRow", - back_populates="call_import", - cascade="all, delete-orphan", - order_by="CallImportRow.row_index", - ) - tags = relationship( - "CallImportTag", - secondary="call_import_tag_assignments", - backref="call_imports", - lazy="selectin", - ) - evaluations = relationship( - "CallImportEvaluation", - back_populates="call_import", - cascade="all, delete-orphan", - ) - - -class CallImportShardSlice(Base): - """Registry row: which shard stores a slice of rows for an import.""" - - __tablename__ = "call_import_shard_slices" - - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - primary_key=True, - ) - slice_id = Column(Integer, primary_key=True) - shard_id = Column(String(64), nullable=False, index=True) - row_index_min = Column(Integer, nullable=False) - row_index_max = Column(Integer, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - -class CallImportRow(Base): - """A single row within a CallImport batch (one CSV line / one external call).""" - - __tablename__ = "call_import_rows" - __table_args__ = ( - UniqueConstraint("call_import_id", "row_index", name="uq_call_import_row_index"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - row_index = Column(Integer, nullable=False) - # Was historically named ``external_call_id``; renamed to - # ``conversation_id`` so the new schema-driven upload flow can refer - # to it by a single canonical name across the schema definition, - # exports, and downstream evaluation tables. - conversation_id = Column(String(255), nullable=False, index=True) - # Supplied via CSV for Exotel credentialed imports (required per row). - # Nullable in the schema for legacy rows imported before recording_url - # was mandatory on every Exotel upload. - recording_url = Column(Text, nullable=True) - # Date-only call recording date supplied by the import schema. Used - # for historical report comparisons without timezone/time ambiguity. - recording_date = Column(Date, nullable=True, index=True) - # The "production" transcript: the value supplied via the CSV - # upload mapping. Never overwritten by the diarisation worker — - # the worker writes its output into ``diarised_transcript`` so - # the user keeps both versions side by side. - transcript = Column(Text, nullable=True) - # Snapshot of the original CSV row keyed by the user's headers so the - # evaluation export can reproduce every column the uploader supplied - # (mapped + extra). NULL on legacy rows imported before this column. - raw_columns = Column(JSON, nullable=True) - - # Where the value in ``transcript`` came from. ``csv`` = supplied via - # the upload mapping, ``edited`` = manually changed in the UI. NULL - # on rows that have never had a production transcript. - # (Worker-produced transcripts now live in ``diarised_transcript`` - # and are tracked via ``diarised_transcript_*`` metadata below.) - transcript_source = Column(String(20), nullable=True) - # Provider/model recorded by the (legacy) post-hoc transcription - # worker. New worker runs leave these NULL and write into the - # ``diarised_transcript_*`` columns instead; kept on the model for - # backwards compatibility with pre-split rows that still carry the - # original transcription metadata here. - transcript_provider = Column(String(50), nullable=True) - transcript_model = Column(String(100), nullable=True) - # Lifecycle status for the legacy transcription workflow itself, - # independent of the row's recording-fetch ``status``. ``idle`` = - # no transcribe task has touched this column. New diarisation runs - # update ``diarised_transcript_status`` instead. - transcript_status = Column( - String(20), - nullable=False, - default="idle", - ) - transcript_error = Column(Text, nullable=True) - transcribed_at = Column(DateTime(timezone=True), nullable=True) - - # The "diarised" transcript: produced by the post-hoc - # transcription/diarisation worker. Stored separately so a manual - # diarisation run never clobbers the production transcript above. - # Evaluations can be configured to score against either column - # (see ``CallImportEvaluation.transcript_source``). - diarised_transcript = Column(Text, nullable=True) - # Provider/model the diarisation worker used. Surfaced in the UI - # as "Diarised via deepgram/nova-2" next to the diarised - # transcript section. - diarised_transcript_provider = Column(String(50), nullable=True) - diarised_transcript_model = Column(String(100), nullable=True) - # Lifecycle status for the diarisation workflow. - # ``idle`` = no diarisation task has run; ``pending``/``running`` = - # a Celery task is queued or in flight; ``completed``/``failed`` = - # terminal. Independent of ``transcript_status`` so the two - # transcripts can be in different lifecycle states. - diarised_transcript_status = Column( - String(20), - nullable=False, - default="idle", - server_default="idle", - ) - diarised_transcript_error = Column(Text, nullable=True) - diarised_at = Column(DateTime(timezone=True), nullable=True) - - # Structured speaker turns produced by the diarisation worker — - # ``[{ "speaker": "agent"|"user"|"speaker_3", "text": "...", - # "start": float, "end": float, "raw_speaker": "Speaker 1" }, ...]`` - # The plain-text ``diarised_transcript`` above is a rendered view - # of this list (``: `` per line). When the worker - # cannot recover structured turns (no pyannote token / single- - # speaker recording / provider that doesn't surface segments) this - # column stays NULL and the plain-text path is still populated. - diarised_segments = Column(JSON, nullable=True) - # When True the ``agent`` <-> ``user`` mapping inside - # ``diarised_segments`` is inverted at render / export time. The - # worker writes the canonical mapping using the "first speaker is - # the agent" heuristic; reviewers can flip the toggle from the row - # detail panel without re-running diarisation. - diarised_speaker_swap = Column( - Boolean, - nullable=False, - default=False, - server_default="false", - ) - # LLM that turned the STT plain-text output into structured - # ``diarised_segments``. The legacy diarisation worker used - # pyannote and left these NULL; the current path always runs an - # LLM with the operator-supplied (or default) ``diarised_prompt`` - # below, and records exactly which model + prompt produced each - # row so reviewers can reproduce a specific run. - diarised_llm_provider = Column(String(50), nullable=True) - diarised_llm_model = Column(String(100), nullable=True) - diarised_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - diarised_prompt = Column(Text, nullable=True) - # Which diarisation pipeline produced this row's turns. - # * ``"stt_llm"`` (default) — two-stage: STT then LLM diariser. - # ``diarised_transcript_provider``/``_model`` describe the STT - # side; ``diarised_llm_provider``/``_model`` the LLM side. - # * ``"llm_only"`` — single-stage: audio fed straight to a - # multimodal LLM. ``diarised_transcript_provider`` is stamped - # with the sentinel ``"llm_only"``; the real model is on - # ``diarised_llm_*``. - # Persisting it on the row (not just the run) lets the row detail - # panel render the right "Diarised via …" label even for ad-hoc - # standalone transcribes (no parent evaluation). - transcribe_mode = Column( - String(20), - nullable=False, - default="stt_llm", - server_default="stt_llm", - ) - - status = Column( - Enum(CallImportRowStatus, values_callable=get_enum_values), - nullable=False, - default=CallImportRowStatus.PENDING, - index=True, - ) - - recording_s3_key = Column(String(1024), nullable=True) - recording_content_type = Column(String(128), nullable=True) - recording_size_bytes = Column(Integer, nullable=True) - - error_message = Column(Text, nullable=True) - attempts = Column(Integer, nullable=False, default=0) - celery_task_id = Column(String(255), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - call_import = relationship("CallImport", back_populates="rows") - - -@event.listens_for(CallImportRow, "before_insert") -def _call_import_row_fill_workspace_id(_mapper, connection, target): - """Denormalize workspace_id from the parent import when omitted.""" - if target.workspace_id is not None or target.call_import_id is None: - return - workspace_id = connection.execute( - select(CallImport.workspace_id).where( - CallImport.id == target.call_import_id - ) - ).scalar_one_or_none() - if workspace_id is not None: - target.workspace_id = workspace_id - - -class CallImportTag(Base): - """User-defined tag that can be attached to one or more call imports. - - Tags coexist with the free-text ``CallImport.dataset`` column: dataset - is the primary high-level segregation, tags are an optional secondary - classification (an import can have many tags). - """ - - __tablename__ = "call_import_tags" - __table_args__ = ( - UniqueConstraint("organization_id", "name", name="uq_call_import_tag_org_name"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - name = Column(String(255), nullable=False) - color = Column(String(32), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - -class CallImportTagAssignment(Base): - """Many-to-many join table between CallImport and CallImportTag.""" - - __tablename__ = "call_import_tag_assignments" - - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - primary_key=True, - ) - tag_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_tags.id", ondelete="CASCADE"), - primary_key=True, - index=True, - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - -class CallImportEvaluation(Base): - """Parent record for an evaluation run over a CallImport batch. - - A user picks a subset of org ``Metric`` rows and triggers an evaluation; - we fan out one ``CallImportEvaluationRow`` per source row and roll up - counters as workers finish. Status mirrors ``CallImportStatus`` plus a - ``RUNNING`` value so the UI can distinguish "queued" from "in flight". - """ - - __tablename__ = "call_import_evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent CallImport's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - created_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id"), nullable=True - ) - - # Optional user-supplied label for this run. Lets the UI surface - # something more meaningful than the UUID prefix (e.g. "March QA pass"). - name = Column(String(255), nullable=True) - - # JSON list of Metric UUID strings selected for this run. Stored as text - # in JSON so we don't have to deal with PG arrays of UUIDs / cascade - # delete policies when metrics are removed; the loader filters for - # still-existing org metrics at run time. - selected_metric_ids = Column(JSON, nullable=False, default=list) - # Hierarchy grouping snapshot: ``{parent_id_str: [child_id_str, ...]}``. - # Captures which children belong to which parent for THIS run so the UI - # / aggregator can reconstruct the tree even when the user selected - # only a subset of children, or after metrics are deleted / renamed. - # NULL on legacy rows means "no hierarchy" → fall back to flat - # ``selected_metric_ids`` semantics. - selected_metric_groups = Column(JSON, nullable=True) - # User-driven merges of LLM-discovered candidate sub-labels for - # ``allow_discovery`` parents. Shape: - # ``{"": {"": "", ...}}``. - # Populated via ``POST .../discovered-labels/merge``; consulted by - # the discovered-labels aggregator, the flow graph builder, and the - # worker so that rows finishing AFTER a merge cannot reintroduce - # the merged-away slug. Empty dict on fresh rows. - discovered_label_aliases = Column( - JSON, nullable=False, default=dict, server_default="{}" - ) - - # Per-run opt-in for top-level metric discovery. When True, the LLM - # is asked to propose brand-new top-level metrics (boolean / rating / - # category) observed in the transcripts in addition to scoring the - # ``selected_metric_ids`` for the row. Candidates surface in a - # "Discovered metrics" panel on the evaluation's Flow tab and can - # be promoted into real standalone ``Metric`` rows via - # ``POST /metrics/from-discovered``. Defaults to False so existing - # evaluation creation payloads keep their previous behaviour. - discover_new_metrics = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - # Flat slug-to-slug redirect map for user merges + tombstones of - # discovered top-level metric candidates. Mirrors - # ``discovered_label_aliases`` but is NOT nested per parent — - # top-level metric discovery is not scoped to any parent. Shape:: - # - # {"": "", ...} - # - # An empty-string value tombstones the slug so workers finishing - # later can't re-introduce it. - discovered_metric_aliases = Column( - JSON, nullable=False, default=dict, server_default="{}" - ) - - # Run-level LLM config picked from the Run Evaluation modal. NULL on - # legacy rows means "use the historical OpenAI/gpt-4o default" — the - # worker checks for this and falls back accordingly. ``llm_credential_id`` - # pins a specific AIProvider row when the org has multiple credentials - # for the same provider. - llm_provider = Column(String(50), nullable=True) - llm_model = Column(String(100), nullable=True) - llm_credential_id = Column( - UUID(as_uuid=True), - ForeignKey("aiproviders.id", ondelete="SET NULL"), - nullable=True, - ) - llm_config = Column(JSON, nullable=True) - # Optional per-metric LLM override: - # ``{"": {"provider": "...", "model": "...", "credential_id": "..."}}``. - # Each entry overrides the run-level default for that metric only; - # missing keys = use run-level default. Stored as JSON so the UI can - # round-trip arbitrary {provider, model} pairs without migrations. - metric_llm_overrides = Column(JSON, nullable=True) - - # When ``auto_transcribe`` was set on the create payload, record the - # STT provider/model used so the UI can show "Auto-transcribed via - # deepgram/nova-2" on the evaluation header. ``stt_credential_id`` is - # untyped (no FK) because STT keys may live in either ``aiproviders`` - # (OpenAI) or ``integrations`` (Deepgram, ElevenLabs) — the - # transcription service handles the lookup. - stt_provider = Column(String(50), nullable=True) - stt_model = Column(String(100), nullable=True) - stt_credential_id = Column(UUID(as_uuid=True), nullable=True) - - # Run-level LLM diariser config. Used when the create-run / - # retry-run paths chain a ``transcribe_call_import_row_task`` - # because the row is missing a diarised transcript. Persisted on - # the run so a retry uses the same diariser the original create - # call picked (unless the retry payload explicitly overrides). - diarisation_llm_provider = Column(String(50), nullable=True) - diarisation_llm_model = Column(String(100), nullable=True) - diarisation_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - diarisation_prompt = Column(Text, nullable=True) - # Mode the run was *created* with for its auto-transcribe step. - # Retry chains read this to decide whether to enqueue an STT+LLM - # transcribe or a single-stage multimodal LLM transcribe — without - # it we'd have to infer the mode from "stt_provider is NULL", which - # would silently break legacy rows that simply never configured - # auto-transcribe. See migration 041 for the column DDL. - transcribe_mode = Column( - String(20), - nullable=False, - default="stt_llm", - server_default="stt_llm", - ) - - # Which of the two transcripts on each ``CallImportRow`` this run - # scored against. ``'production'`` reads ``CallImportRow.transcript`` - # (the CSV-supplied value); ``'diarised'`` reads - # ``CallImportRow.diarised_transcript`` (the worker output). When - # the user ticks both checkboxes in the Run Evaluation modal we - # create two ``CallImportEvaluation`` rows — one per source — so - # the two scorings can be compared side-by-side. Defaults to - # ``'production'`` so legacy runs (which always read the single - # historical ``transcript`` column) keep their semantics. - transcript_source = Column( - String(20), - nullable=False, - default="production", - server_default="production", - ) - - # Cached LLM-generated TLDR rendered above the Visualizations charts. - # Populated lazily by ``POST /evaluations/{eval_id}/insights`` so we - # never auto-burn LLM tokens on page load. Shape:: - # {"narrative": str, "patterns": [str, ...], - # "generated_at": iso8601, "generated_at_completed_rows": int, - # "provider": str, "model": str} - # NULL on rows that have never been summarised. - tldr_summary = Column(JSON, nullable=True) - - # Cached LLM-generated user insights for External Audit PDF section 03. - # Populated by a background Celery job triggered alongside TLDR generation. - # Shape: EvaluationUserInsightsState JSON (status, insights[], progress, …). - user_insights = Column(JSON, nullable=True) - - # Cached per-metric failure clustering for internal diagnostics PDF/UI. - # Shape: EvaluationMetricClustersState JSON (status, groups[], …). - metric_clusters = Column(JSON, nullable=True) - - # Cached LLM-generated prompt improvement suggestions keyed to an - # imported agent (PromptPartial tagged __imported_agent__). - # Shape: EvaluationPromptImprovementsState JSON. - prompt_improvements = Column(JSON, nullable=True) - - # Cached LLM explanations for week-over-week metric deltas keyed by - # baseline evaluation id + completed row counts. - period_delta_explanations = Column(JSON, nullable=True) - - status = Column(String(20), nullable=False, default="pending", index=True) - - total_rows = Column(Integer, nullable=False, default=0) - completed_rows = Column(Integer, nullable=False, default=0) - failed_rows = Column(Integer, nullable=False, default=0) - # Flexprice pass-level delta billing watermark: rows already emitted - # on ``call_import.evaluation_completed`` for this evaluation run. - billed_completed_rows = Column( - Integer, nullable=False, default=0, server_default="0" - ) - error_message = Column(Text, nullable=True) - celery_group_id = Column(String(255), nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - call_import = relationship("CallImport", back_populates="evaluations") - row_results = relationship( - "CallImportEvaluationRow", - back_populates="evaluation", - cascade="all, delete-orphan", - ) - - -class CallImportEvaluationRow(Base): - """Per-source-row scoring output for a CallImportEvaluation parent.""" - - __tablename__ = "call_import_evaluation_rows" - __table_args__ = ( - UniqueConstraint( - "evaluation_id", "call_import_row_id", name="uq_call_import_evaluation_row" - ), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - call_import_row_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_rows.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - status = Column(String(20), nullable=False, default="pending", index=True) - # Same shape as EvaluatorResult.metric_scores: {metric_id_str: {value, type, metric_name, ...}} - metric_scores = Column(JSON, nullable=False, default=dict) - error_message = Column(Text, nullable=True) - celery_task_id = Column(String(255), nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - evaluation = relationship("CallImportEvaluation", back_populates="row_results") - source_row = relationship("CallImportRow") - - -@event.listens_for(CallImportEvaluationRow, "before_insert") -def _call_import_evaluation_row_fill_workspace_id(_mapper, connection, target): - """Denormalize workspace_id from the parent evaluation when omitted.""" - if target.workspace_id is not None or target.evaluation_id is None: - return - workspace_id = connection.execute( - select(CallImportEvaluation.workspace_id).where( - CallImportEvaluation.id == target.evaluation_id - ) - ).scalar_one_or_none() - if workspace_id is not None: - target.workspace_id = workspace_id - - -class CallImportEvaluationReportSnapshot(Base): - """Persisted PDF-report aggregate used for period-over-period deltas.""" - - __tablename__ = "call_import_evaluation_report_snapshots" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - period_label = Column(String(64), nullable=True, index=True) - period_start = Column(Date, nullable=True, index=True) - period_end = Column(Date, nullable=True, index=True) - report_config = Column(JSON, nullable=False, default=dict, server_default="{}") - selected_metric_ids = Column(JSON, nullable=False, default=list, server_default="[]") - metric_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") - insight_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") - narrative = Column(JSON, nullable=True) - total_calls = Column(Integer, nullable=False, default=0) - selected_metric_count = Column(Integer, nullable=False, default=0) - total_metric_count = Column(Integer, nullable=False, default=0) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - -# --------------------------------------------------------------------------- -# Judge Alignment (AlignEval-style hybrid integration) -# -# Three tables back the "Judge Alignment" surface: -# - JudgeDataset: a labeled dataset materialised from one of three sources -# (voice transcripts, existing Metric/Evaluator outputs, -# or a generic CSV upload). Holds the dataset's source -# config + which fields play the role of input/output. -# - JudgeSample: one row in a dataset (input/output pair plus an -# optional binary pass/fail human label). -# - JudgeRun: a single run of an LLM-judge (existing Evaluator) over -# a subset of samples, with computed alignment metrics -# (precision/recall/F1/Cohen's kappa) and per-sample -# predictions. Optionally links to a GEPA optimization -# run when the user kicks off prompt tuning from a -# dataset. -# --------------------------------------------------------------------------- - - -class JudgeDataset(Base): - """Container for binary-labeled samples used to calibrate an LLM-judge.""" - - __tablename__ = "judge_datasets" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - # Workspace isolation: every judge dataset belongs to a workspace - # within its org. Samples and runs inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - - # One of: "transcript", "metric_output", "csv" - source_type = Column(String(32), nullable=False, index=True) - # Source-specific config. Examples: - # transcript: {"transcription_ids": [...]} or {"agent_id": "..."} - # metric_output: {"metric_id": "...", "evaluator_id": "..."} - # csv: {"s3_key": "...", "filename": "..."} - source_config = Column(JSON, nullable=False, default=dict) - - # Field roles - which textual content is "input" vs "output" for the judge. - # For voice transcripts both default to the transcript text but can be - # tightened (e.g. agent-only turns vs full conversation). - input_field = Column(String(64), nullable=False, default="input") - output_field = Column(String(64), nullable=False, default="output") - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - samples = relationship( - "JudgeSample", - back_populates="dataset", - cascade="all, delete-orphan", - order_by="JudgeSample.created_at", - ) - runs = relationship( - "JudgeRun", - back_populates="dataset", - cascade="all, delete-orphan", - order_by="JudgeRun.created_at.desc()", - ) - - -class JudgeSample(Base): - """One labelable input/output pair within a JudgeDataset.""" - - __tablename__ = "judge_samples" - __table_args__ = ( - UniqueConstraint("dataset_id", "external_id", name="uq_judge_samples_dataset_external"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - dataset_id = Column( - UUID(as_uuid=True), - ForeignKey("judge_datasets.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent JudgeDataset's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Stable identifier within the source (e.g. transcription UUID, CSV row id). - # Used to dedupe re-imports and link back to the originating record. - external_id = Column(String(128), nullable=True, index=True) - - input_text = Column(Text, nullable=False) - output_text = Column(Text, nullable=False) - - # Binary human label: "pass" | "fail" | null (unlabeled). - # Stored as string (rather than enum) so it stays trivially extendable. - label = Column(String(16), nullable=True, index=True) - labeled_by = Column(String(255), nullable=True) - labeled_at = Column(DateTime(timezone=True), nullable=True) - - # Source-specific context (e.g. agent_id, original metric value, csv row). - extra = Column(JSON, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - dataset = relationship("JudgeDataset", back_populates="samples") - - -class JudgeRun(Base): - """One execution of an LLM-judge against a JudgeDataset, with alignment metrics.""" - - __tablename__ = "judge_runs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - dataset_id = Column( - UUID(as_uuid=True), - ForeignKey("judge_datasets.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - # Workspace isolation: mirrors the parent JudgeDataset's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Reuses the existing Evaluator row (its custom_prompt + llm_provider + llm_model - # define the judge under test). Nullable so a run may target an inline prompt - # in the future without inflating the Evaluator table. - evaluator_id = Column( - UUID(as_uuid=True), ForeignKey("evaluators.id", ondelete="SET NULL"), nullable=True, index=True - ) - - # Which subset was scored: "all" | "dev" | "test" - split = Column(String(16), nullable=False, default="all") - - # Snapshot of the model used (so a later Evaluator edit doesn't rewrite history). - llm_provider = Column(String(64), nullable=True) - llm_model = Column(String(128), nullable=True) - - # Computed alignment metrics: - # {"precision": float, "recall": float, "f1": float, "kappa": float, - # "tp": int, "fp": int, "tn": int, "fn": int, "n": int} - metrics = Column(JSON, nullable=True) - - # Per-sample predictions, keyed by sample_id (UUID string): - # {sample_id: {"prediction": "pass"|"fail", "explanation": str, "raw": str}} - predictions = Column(JSON, nullable=True) - - # Run lifecycle. - status = Column(String(20), nullable=False, default="pending", index=True) - error_message = Column(Text, nullable=True) - celery_task_id = Column(String, nullable=True, index=True) - - # Optional link to a GEPA optimization run kicked off from this dataset. - gepa_optimization_id = Column( - UUID(as_uuid=True), - ForeignKey("prompt_optimization_runs.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - dataset = relationship("JudgeDataset", back_populates="runs") +"""SQLAlchemy database models.""" + +from sqlalchemy import ( + BigInteger, + Boolean, + Column, + Date, + DateTime, + DDL, + Enum, + event, + Float, + ForeignKey, + Integer, + JSON, + String, + Text, + UniqueConstraint, + select, + text, +) +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import relationship +from sqlalchemy.sql import func +import uuid +import enum +from app.models.enums import ( + EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, + LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, + IntegrationPlatform, ModelProvider, VoiceBundleType, TestAgentConversationStatus, + MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, + AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, + PromptOptimizationStatus, CallImportStatus, CallImportRowStatus, +) + +def get_enum_values(enum_class): + """Helper to get values from enum class for SQLAlchemy.""" + return [e.value for e in enum_class] + +from app.database import Base + + +# Enums moved to enums.py + + +class Organization(Base): + """Organization model for multi-tenancy.""" + + __tablename__ = "organizations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + name = Column(String(255), nullable=False) + voice_playground_threshold_overrides = Column(JSON, nullable=True) + # AlignEval-style judge alignment thresholds. + # Shape: {"min_labels_to_evaluate": int, "min_labels_to_optimize": int} + # Falls back to system defaults (20 / 50) when null. + judge_alignment_settings = Column(JSON, nullable=True) + # Per-org LLM gateway overrides (enabled, gateway_type, base_url, keys). + llm_gateway_settings = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + api_keys = relationship("APIKey", back_populates="organization") + members = relationship("OrganizationMember", back_populates="organization", cascade="all, delete-orphan") + invitations = relationship("Invitation", back_populates="organization", cascade="all, delete-orphan") + workspaces = relationship( + "Workspace", + back_populates="organization", + cascade="all, delete-orphan", + ) + workspace_roles = relationship( + "WorkspaceRole", + back_populates="organization", + cascade="all, delete-orphan", + ) + + +class Workspace(Base): + """Workspace - in-org isolation boundary for call imports and metrics. + + Every organization has at least one workspace (``is_default = True``, + seeded by migration 033). Users pick an "active workspace" in the UI; + list endpoints filter by it so users only see calls/metrics from the + project they're currently working in. Access is governed by + ``workspace_members`` and org-scoped ``workspace_roles`` (capability + bundles); org admins implicitly access all workspaces. + """ + + __tablename__ = "workspaces" + __table_args__ = ( + UniqueConstraint("organization_id", "slug", name="uq_workspaces_org_slug"), + ) + + # ``server_default`` is required so that raw-SQL INSERTs (e.g. the + # per-org Default seed in migration 033) can omit ``id`` and let the + # database fill it in. Without it, ``create_all`` produces a column + # with NOT NULL but no DEFAULT, and the migration crashes with + # ``null value in column "id"``. + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + slug = Column(String(255), nullable=False) + # At most one default per org. Enforced on Postgres by the partial + # unique index attached via the after_create event below; on + # SQLite (test runs) we rely on the route-level _check_slug_unique + # check + the Default-workspace conftest fixture instead, because + # SQLite doesn't support partial indexes the same way. + is_default = Column(Boolean, nullable=False, default=False, server_default="false") + # Reusable PDF/report branding metadata scoped to this workspace. Images + # live in S3. Shape: {"heading": str|null, "images": [{id, s3_key, + # content_type, filename, size_bytes, updated_at}, ...]}. + report_branding = Column(JSON, nullable=True) + created_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + organization = relationship("Organization", back_populates="workspaces") + members = relationship( + "WorkspaceMember", + back_populates="workspace", + cascade="all, delete-orphan", + ) + + +class WorkspaceRole(Base): + """Org-scoped workspace role (system or custom) as a capability bundle.""" + + __tablename__ = "workspace_roles" + __table_args__ = ( + UniqueConstraint("organization_id", "name", name="uq_workspace_roles_org_name"), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + capabilities = Column(JSON, nullable=False, default=list) + is_system = Column(Boolean, nullable=False, default=False, server_default="false") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + organization = relationship("Organization", back_populates="workspace_roles") + members = relationship("WorkspaceMember", back_populates="role") + + +class WorkspaceMember(Base): + """User membership in a workspace with an assigned workspace role.""" + + __tablename__ = "workspace_members" + __table_args__ = ( + UniqueConstraint("workspace_id", "user_id", name="uq_workspace_members_ws_user"), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + role_id = Column( + UUID(as_uuid=True), + ForeignKey("workspace_roles.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + added_by_user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + workspace = relationship("Workspace", back_populates="members") + user = relationship("User", foreign_keys=[user_id]) + role = relationship("WorkspaceRole", back_populates="members") + added_by = relationship("User", foreign_keys=[added_by_user_id]) + + +# Partial unique index: "at most one default workspace per org". This +# is attached as an after_create event (rather than declared in +# ``__table_args__``) because SQLAlchemy's ``Index(..., +# postgresql_where=...)`` silently degrades to a *full* unique index on +# SQLite - which then forbids any second workspace per org and breaks +# the test suite. ``execute_if(dialect="postgresql")`` makes this DDL +# a no-op on SQLite while still emitting it on Postgres (prod, CI). +event.listen( + Workspace.__table__, + "after_create", + DDL( + "CREATE UNIQUE INDEX IF NOT EXISTS uq_workspaces_org_default " + "ON workspaces (organization_id) WHERE is_default" + ).execute_if(dialect="postgresql"), +) + + +class User(Base): + """User model for authentication and profile management.""" + + __tablename__ = "users" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + email = Column(String(255), unique=True, nullable=False, index=True) + name = Column(String(255), nullable=True) + first_name = Column(String(255), nullable=True) + last_name = Column(String(255), nullable=True) + password_hash = Column(String(255), nullable=True) # Nullable for users created via invitation + external_id = Column(String(255), unique=True, nullable=True, index=True) + auth_provider = Column(String(50), nullable=True) + mfa_enabled = Column(Boolean, default=False, nullable=False) + last_login_at = Column(DateTime(timezone=True), nullable=True) + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + organization_memberships = relationship("OrganizationMember", back_populates="user", cascade="all, delete-orphan") + api_keys = relationship("APIKey", back_populates="user") + invitations = relationship("Invitation", back_populates="invited_user", foreign_keys="Invitation.invited_user_id") + refresh_tokens = relationship("RefreshToken", back_populates="user", cascade="all, delete-orphan") + + +class RefreshToken(Base): + """Opaque refresh token for extending local-password sessions.""" + + __tablename__ = "refresh_tokens" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True) + token_hash = Column(String(64), unique=True, nullable=False, index=True) + expires_at = Column(DateTime(timezone=True), nullable=False) + revoked_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + user = relationship("User", back_populates="refresh_tokens") + + +class OrganizationMember(Base): + """Organization membership with role.""" + + __tablename__ = "organization_members" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True) + role = Column(String, nullable=False, default=RoleEnum.READER.value) + + # User preferences for this organization + default_agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) + + joined_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Unique constraint: one membership per user per organization + __table_args__ = ( + UniqueConstraint('organization_id', 'user_id', name='uq_org_user'), + ) + + # Relationships + organization = relationship("Organization", back_populates="members") + user = relationship("User", back_populates="organization_memberships") + default_agent = relationship("Agent", foreign_keys=[default_agent_id]) + + +class Invitation(Base): + """Invitation model for inviting users to organizations.""" + + __tablename__ = "invitations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + invited_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Null if user doesn't exist yet + invited_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) + email = Column(String(255), nullable=False) # Email of invited user + role = Column(String, nullable=False, default=RoleEnum.READER.value) + status = Column(String, nullable=False, default=InvitationStatus.PENDING.value) + + + + token = Column(String(255), unique=True, nullable=False, index=True) # Invitation token + expires_at = Column(DateTime(timezone=True), nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + accepted_at = Column(DateTime(timezone=True), nullable=True) + + # Relationships + organization = relationship("Organization", back_populates="invitations") + invited_user = relationship("User", foreign_keys=[invited_user_id], back_populates="invitations") + invited_by = relationship("User", foreign_keys=[invited_by_id]) + + +class APIKey(Base): + """API Key model for authentication.""" + + __tablename__ = "api_keys" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + key = Column(String(255), unique=True, nullable=False, index=True) + name = Column(String(255), nullable=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Optional: link to user + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + last_used = Column(DateTime(timezone=True), nullable=True) + + # Relationships + organization = relationship("Organization", back_populates="api_keys") + user = relationship("User", back_populates="api_keys") + + +class AudioFile(Base): + """Audio file model.""" + + __tablename__ = "audio_files" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + filename = Column(String(255), nullable=False) + file_path = Column(String(512), nullable=False) + file_size = Column(Integer, nullable=False) # Size in bytes + duration = Column(Float, nullable=True) # Duration in seconds + sample_rate = Column(Integer, nullable=True) + channels = Column(Integer, nullable=True) + format = Column(String(10), nullable=False) # wav, mp3, flac, etc. + uploaded_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Relationships + evaluations = relationship("Evaluation", back_populates="audio_file") + + +class Evaluation(Base): + """Evaluation job model.""" + + __tablename__ = "evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every legacy audio evaluation belongs to a + # workspace within its org. Stamped from the X-Workspace-Id header + # (falling back to the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + audio_id = Column(UUID(as_uuid=True), ForeignKey("audio_files.id"), nullable=False) + reference_text = Column(String, nullable=True) # For WER calculation + evaluation_type = Column(String, nullable=False) + model_name = Column(String(100), nullable=True) + status = Column(String, default=EvaluationStatus.PENDING.value, nullable=False) + + + + metrics_requested = Column(JSON, nullable=True) # List of requested metrics + created_at = Column(DateTime(timezone=True), server_default=func.now()) + started_at = Column(DateTime(timezone=True), nullable=True) + completed_at = Column(DateTime(timezone=True), nullable=True) + error_message = Column(String, nullable=True) + + # Relationships + audio_file = relationship("AudioFile", back_populates="evaluations") + result = relationship("EvaluationResult", back_populates="evaluation", uselist=False) + + +class EvaluationResult(Base): + """Evaluation result model.""" + + __tablename__ = "evaluation_results" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column(UUID(as_uuid=True), ForeignKey("evaluations.id"), nullable=False, unique=True) + # Workspace isolation: mirrors the parent Evaluation's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + transcript = Column(String, nullable=True) + metrics = Column(JSON, nullable=False) # {"wer": 0.05, "latency_ms": 1250, ...} + raw_output = Column(JSON, nullable=True) # Full model output + processing_time = Column(Float, nullable=True) # Processing time in seconds + model_used = Column(String(100), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Relationships + evaluation = relationship("Evaluation", back_populates="result") + + +# ============================================ +# VAIOPS MODELS - Voice AI Ops +# ============================================ + +# Enums moved to enums.py + + +class Agent(Base): + """Test Agent - The voice AI agent being evaluated""" + __tablename__ = "agents" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + agent_id = Column(String(6), unique=True, nullable=True, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every agent belongs to a workspace within its + # org. Stamped from the X-Workspace-Id header (falling back to the + # org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String, nullable=False) + phone_number = Column(String, nullable=True) # Optional, required only for phone_call + language = Column(String, nullable=False, default=LanguageEnum.ENGLISH.value) + description = Column(String) + provider_prompt = Column(Text, nullable=True) + provider_prompt_synced_at = Column(DateTime(timezone=True), nullable=True) + call_type = Column(String, nullable=False, default=CallTypeEnum.OUTBOUND.value) + call_medium = Column(String, nullable=False, default=CallMediumEnum.PHONE_CALL.value) + telephony_phone_number_id = Column( + UUID(as_uuid=True), + ForeignKey("telephony_phone_numbers.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + + + + # Voice configuration - either voice_bundle_id OR ai_provider_id OR voice_ai_integration_id (mutually exclusive) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True, index=True) + ai_provider_id = Column(UUID(as_uuid=True), ForeignKey("aiproviders.id"), nullable=True, index=True) + + # Voice AI agent integration (Retell, Vapi, etc.) + voice_ai_integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id"), nullable=True, index=True) + voice_ai_agent_id = Column(String, nullable=True) # Agent ID from the external provider (Retell/Vapi) + prompt_variables = Column(JSON, nullable=True) + silence_hangup_secs = Column(Integer, nullable=False, server_default="15") + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +class Persona(Base): + """Persona - TTS provider-tied voice identity for testing""" + __tablename__ = "personas" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every persona belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String, nullable=False) + gender = Column(String, nullable=False, default=GenderEnum.NEUTRAL.value) + tts_provider = Column(String(100), nullable=True) + tts_voice_id = Column(String(255), nullable=True) + tts_voice_name = Column(String(255), nullable=True) + is_custom = Column(Boolean, default=False) + description = Column(Text, nullable=True) + tts_config = Column(JSON, nullable=True) + llm_temperature = Column(Float, nullable=True) + llm_max_tokens = Column(Integer, nullable=True) + response_delay_ms = Column(Integer, nullable=True) + max_turns = Column(Integer, nullable=True) + allow_interruptions = Column(Boolean, nullable=True) + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +class Scenario(Base): + """Scenario - The conversation scenario/test case""" + __tablename__ = "scenarios" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every scenario belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) + name = Column(String, nullable=False) + description = Column(String) + required_info = Column(JSON) + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +# Enums moved to enums.py + + +class Integration(Base): + """Integration model for connecting with external voice AI platforms.""" + __tablename__ = "integrations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + platform = Column(String, nullable=False) + + + + name = Column(String, nullable=True) # Optional friendly name + api_key = Column(String, nullable=False) # Encrypted Private API key for the platform + public_key = Column(String, nullable=True) # Optional Public API key (e.g. for Vapi) + is_active = Column(Boolean, default=True, nullable=False) + # Multiple credentials per (org, platform) are allowed. is_default marks + # the row used when a caller does not explicitly select a credential. + # A partial unique index in migration 028 enforces at most one default + # per (org, platform) at the DB level. + is_default = Column(Boolean, default=False, nullable=False) + # inherit | gateway | direct — per-credential LLM routing override + routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated + + +class ManualTranscription(Base): + """Manual transcription model for storing transcriptions from S3 audio files.""" + + __tablename__ = "manual_transcriptions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + name = Column(String(255), nullable=True) # User-friendly name for the transcription + audio_file_key = Column(String(512), nullable=False) # S3 key or file path + transcript = Column(String, nullable=False) # Full transcript text + speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] + stt_model = Column(String(100), nullable=True) # STT model used (e.g., "whisper-1", "google-speech-v2") + stt_provider = Column(String, nullable=True) # Provider used + + + + language = Column(String(10), nullable=True) # Detected or specified language + processing_time = Column(Float, nullable=True) # Processing time in seconds + raw_output = Column(JSON, nullable=True) # Full model output for reference + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class ConversationEvaluation(Base): + """Conversation evaluation model for evaluating manual transcriptions against agent objectives.""" + + __tablename__ = "conversation_evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + transcription_id = Column(UUID(as_uuid=True), ForeignKey("manual_transcriptions.id"), nullable=False, index=True) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) + + # Evaluation results + objective_achieved = Column(Boolean, nullable=False) # Binary: was the conversation objective achieved? + objective_achieved_reason = Column(String, nullable=True) # Explanation for the binary result + additional_metrics = Column(JSON, nullable=True) # Additional evaluation metrics (e.g., professionalism, clarity, etc.) + overall_score = Column(Float, nullable=True) # Overall score (0.0 to 1.0) + + # LLM metadata + llm_provider = Column(Enum(ModelProvider, native_enum=False), nullable=True) + + llm_model = Column(String(100), nullable=True) + llm_response = Column(JSON, nullable=True) # Full LLM response for reference + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class AIProvider(Base): + """AI Provider - Stores API keys for different AI platforms.""" + __tablename__ = "aiproviders" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String, nullable=False) + + + + api_key = Column(String, nullable=False) # Encrypted API key + name = Column(String, nullable=True) # Optional friendly name + # Azure OpenAI resource endpoint (e.g. https://my-resource.openai.azure.com). + # Only used when provider is azure; other providers ignore this column. + endpoint_url = Column(String, nullable=True) + is_active = Column(Boolean, default=True, nullable=False) + # Multiple AIProvider rows per (org, provider) are allowed. is_default + # marks the row resolved when no explicit credential id is selected. + # A partial unique index in migration 028 enforces at most one default. + is_default = Column(Boolean, default=False, nullable=False) + # inherit | gateway | direct — per-credential LLM routing override + routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") + # Bifrost custom model ID used when routing via gateway + gateway_model = Column(String(255), nullable=True) + # inherit | litellm_shim | native_openai — Bifrost API surface override + gateway_interface = Column(String(20), nullable=False, default="inherit", server_default="inherit") + # Optional per-credential Bifrost/gateway base URL override + gateway_base_url = Column(String(512), nullable=True) + # Optional auth header for Bifrost (e.g. x-bf-vk, Authorization, x-api-key) + gateway_auth_header = Column(String(64), nullable=True) + # Env var name whose value is sent as the gateway auth secret + gateway_auth_secret_env = Column(String(128), nullable=True) + # Encrypted inline gateway auth secret (alternative to env var) + gateway_auth_secret = Column(String, nullable=True) + # Arbitrary HTTP headers sent with gateway-routed LiteLLM calls + gateway_extra_headers = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated + + +# Enums moved to enums.py + + +class VoiceBundle(Base): + """VoiceBundle - Composable unit combining STT, LLM, and TTS for voice AI testing, or S2S models.""" + __tablename__ = "voicebundles" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + name = Column(String, nullable=False) + description = Column(String, nullable=True) + + # Bundle type: either STT+LLM+TTS or S2S + # Using String instead of Enum to avoid SQLAlchemy enum conversion issues + # The enum conversion is handled in the Pydantic schemas + bundle_type = Column(String(50), nullable=False, default=VoiceBundleType.STT_LLM_TTS.value) + + # STT Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + stt_provider = Column(String, nullable=True) + # Optional explicit credential row (aiproviders.id or integrations.id). + # When NULL the credential resolver picks the default row for the + # provider. No FK is set because the target table varies by provider. + stt_credential_id = Column(UUID(as_uuid=True), nullable=True) + + stt_model = Column(String, nullable=True) # e.g., "whisper-1", "google-speech-v2" + + # LLM Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + llm_provider = Column(String, nullable=True) + llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + + llm_model = Column(String, nullable=True) # e.g., "gpt-4", "claude-3-opus" + llm_temperature = Column(Float, nullable=True, default=0.7) + llm_max_tokens = Column(Integer, nullable=True) + llm_config = Column(JSON, nullable=True) # Additional LLM configuration (extensible) + + # TTS Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + tts_provider = Column(String, nullable=True) + tts_credential_id = Column(UUID(as_uuid=True), nullable=True) + + tts_model = Column(String, nullable=True) # e.g., "tts-1", "neural-voice" + tts_voice = Column(String, nullable=True) # Voice selection if applicable + tts_config = Column(JSON, nullable=True) # Additional TTS configuration (extensible) + + # S2S Configuration - required for S2S type, optional for STT_LLM_TTS + s2s_provider = Column(String, nullable=True) + s2s_credential_id = Column(UUID(as_uuid=True), nullable=True) + + + + s2s_model = Column(String, nullable=True) # e.g., "gpt-4o-transcribe", speech-to-speech model + s2s_config = Column(JSON, nullable=True) # Additional S2S configuration (extensible) + + # Additional configuration for extensibility + extra_metadata = Column(JSON, nullable=True) # For future extensions (renamed from 'metadata' to avoid SQLAlchemy conflict) + + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +# Enums moved to enums.py + + +class TestAgentConversation(Base): + """Test Agent Conversation - Records conversations between test AI agent and voice AI agent.""" + __tablename__ = "test_agent_conversations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every playground conversation belongs to a + # workspace within its org. Stamped from the X-Workspace-Id header + # (falling back to the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Configuration + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=False) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) + + # Conversation data + status = Column(String, nullable=False, default=TestAgentConversationStatus.INITIALIZING.value) + + + + live_transcription = Column(JSON, nullable=True) # Array of conversation turns with timestamps + conversation_audio_key = Column(String, nullable=True) # S3 key for recorded conversation audio + full_transcript = Column(String, nullable=True) # Full conversation transcript + + # Metadata + started_at = Column(DateTime(timezone=True), server_default=func.now()) + ended_at = Column(DateTime(timezone=True), nullable=True) + duration_seconds = Column(Float, nullable=True) + + # Additional metadata + conversation_metadata = Column(JSON, nullable=True) # Additional conversation metadata + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class EvaluatorSuite(Base): + """Evaluator suite — one agent + one persona + N scenario combinations.""" + + __tablename__ = "evaluator_suites" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + name = Column(String, nullable=True) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) + metric_ids = Column(JSON, nullable=True) + llm_provider = Column(String, nullable=True) + llm_model = Column(String, nullable=True) + llm_config = Column(JSON, nullable=True) + tags = Column(JSON, nullable=True) + default_runs_per_combination = Column(Integer, nullable=False, default=1) + round_robin_index = Column(Integer, nullable=False, default=0) + is_active = Column(Boolean, nullable=False, default=False) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class Evaluator(Base): + """Evaluator - Configuration for testing agents with specific persona and scenario combinations, or custom prompt evaluators.""" + __tablename__ = "evaluators" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluator_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every evaluator belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Display name (required for custom evaluators, optional for standard) + name = Column(String, nullable=True) + + # Parent suite (nullable for legacy/custom evaluators) + suite_id = Column( + UUID(as_uuid=True), + ForeignKey("evaluator_suites.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + + # Standard evaluator configuration (nullable for custom evaluators) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) + + # Custom evaluator prompt (used instead of agent/persona/scenario) + custom_prompt = Column(Text, nullable=True) + + # Custom evaluator metric selection. When set, the worker filters the + # enabled-org metrics down to only these IDs (list of metric UUID strings). + # Standard evaluators leave this NULL and use all enabled agent metrics. + metric_ids = Column(JSON, nullable=True) + + # LLM configuration for evaluation (overrides hardcoded defaults) + llm_provider = Column(String, nullable=True) # e.g. "openai", "anthropic", "google" + llm_model = Column(String, nullable=True) # e.g. "gpt-4.1", "claude-sonnet-4-20250514" + llm_config = Column(JSON, nullable=True) + + # Tags for categorization + tags = Column(JSON, nullable=True) # Array of tag strings + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +# Enums moved to enums.py + + +class Metric(Base): + """Metric - Configuration for evaluation metrics. + + Supports a 2-level hierarchy via ``parent_metric_id``: a "category" + parent metric (e.g. "Call Outcome") owns N child sub-metric labels + (e.g. "happy_completion", "angry_hangup"). ``selection_mode`` is set + only on parents and controls how the LLM scores children together + (``single_choice`` = pick exactly one; ``multi_label`` = independent + yes/no with logical consistency). Children are always boolean. + """ + __tablename__ = "metrics" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: two-shape column. + # + # * ``workspace_id = `` — workspace-scoped metric. Only + # visible inside that workspace (the default behavior; existing + # rows all look like this). + # * ``workspace_id IS NULL`` — org-shared metric. Surfaces in + # every workspace's listing under this org so users don't have + # to recreate the same metric per workspace. + # + # Children always inherit their parent's ``workspace_id`` (including + # NULL) so a category metric's whole subtree shares one scope; the + # add-child / promote-discovered endpoints enforce this. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=True, + index=True, + ) + + # Basic information + name = Column(String, nullable=False) + description = Column(String, nullable=True) + # Free-form illustrative example used to sharpen the LLM judge's + # rubric. Today this is consumed by child sub-labels of a + # categorization parent metric so each label can carry "what does + # this look like in a transcript?" text alongside the rubric in + # ``description``. The column lives on every Metric row for + # forward-compat: a standalone metric could later surface its own + # example without another migration. + example = Column(Text, nullable=True) + + # Configuration + metric_type = Column(String, nullable=False, default=MetricType.RATING.value) + metric_category = Column( + String(30), + nullable=False, + default=MetricCategory.QUALITY.value, + server_default=MetricCategory.QUALITY.value, + ) + trigger = Column(String, nullable=False, default=MetricTrigger.ALWAYS.value) + metric_origin = Column(String(30), nullable=False, default="default") + supported_surfaces = Column(JSON, nullable=False, default=list) # ["agent", "voice_playground", "blind_test"] + enabled_surfaces = Column(JSON, nullable=False, default=list) # subset of supported_surfaces + custom_data_type = Column(String(30), nullable=True) # "boolean" | "enum" | "number_range" + custom_config = Column(JSON, nullable=True) # enum options / number range config + tags = Column(JSON, nullable=True) # ["tone", "latency", ...] + + # Hierarchy: NULL = standalone or parent. When set, this row is a + # child sub-metric of the referenced parent. ON DELETE CASCADE so + # deleting a category removes its children atomically. + parent_metric_id = Column( + UUID(as_uuid=True), + ForeignKey("metrics.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + # Set only on parent rows (``parent_metric_id IS NULL``). Either + # ``single_choice`` or ``multi_label``. NULL = legacy / non-hierarchical + # metric (no children). + selection_mode = Column(String(20), nullable=True) + + # When true on a parent metric (any selection_mode), the LLM is + # invited during call-import evaluation to emit additional + # candidate sub-labels beyond the user-defined children. The + # candidates surface in a "Discovered labels" panel where the user + # manually promotes them into real child Metric rows. For + # ``single_choice`` parents the discovered entries are + # supplemental — the chosen child is still picked from the + # predefined children so the exactly-one-true invariant holds. + # The validator rejects this flag on standalone / child metrics. + allow_discovery = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + + # When True, this metric is a "transcript-compare judge": the + # call-import evaluator feeds BOTH the production transcript + # (``call_import_rows.transcript``, CSV-supplied) and the diarised + # transcript (``call_import_rows.diarised_transcript``, worker- + # produced by the STT/diarisation pipeline) to the LLM as a + # labeled pair instead of feeding one transcript. The parent + # evaluation's ``CallImportEvaluation.transcript_source`` is + # ignored for these metrics — they always read both columns. + # Rows where either transcript is missing are skipped per-metric + # with ``skipped="comparison_missing_transcript"`` so the rest of + # the row's metrics still produce scores. The Pydantic validator + # rejects ``compare_transcripts`` combined with ``parent_metric_id`` + # or ``selection_mode`` (i.e. it can't simultaneously be part of + # a parent/child hierarchy). The call-import worker also + # auto-promotes a metric to comparison mode when its description + # references the production / diarised transcripts in well-known + # phrases (see ``_metric_text_references_production`` in + # ``app.workers.tasks.evaluate_call_import_row``). + compare_transcripts = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + + parent = relationship( + "Metric", + remote_side=[id], + backref="children", + ) + + # When true, the LLM-judge is asked to also return a short free-form + # rationale alongside the value (stored under ``metric_scores[id].rationale``). + # Adds a second " - LLM Rationale" column in the call-import CSV export. + capture_rationale = Column(Boolean, nullable=False, default=False) + + enabled = Column(Boolean, nullable=False, default=True) + + # Metadata + is_default = Column(Boolean, nullable=False, default=False) # Pre-defined metrics + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class EvaluatorResult(Base): + """EvaluatorResult - Results from running an evaluator with transcription and metric evaluations.""" + __tablename__ = "evaluator_results" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + result_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every evaluator result belongs to a workspace + # within its org. Stamped from the active workspace at creation time + # (either the X-Workspace-Id header or the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # References + evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True, index=True) # Optional - can be None for test calls without persona/scenario + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Nullable for custom evaluators + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) # Optional - can be None for test calls + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) # Optional - can be None for test calls + + # Result data + name = Column(String, nullable=True) # Scenario name or test call name (optional) + timestamp = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + duration_seconds = Column(Float, nullable=True) # Call duration + status = Column(String(20), nullable=False, default=EvaluatorResultStatus.QUEUED.value) + + # Audio and transcription + audio_s3_key = Column(String, nullable=True) # S3 key for audio file + transcription = Column(String, nullable=True) # Full transcription + speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] + + # Metric scores - JSON object with metric_id as key and score as value + # Format: {"metric_id_1": {"value": 85, "type": "rating"}, "metric_id_2": {"value": true, "type": "boolean"}} + metric_scores = Column(JSON, nullable=True) + + # Celery task tracking + celery_task_id = Column(String, nullable=True, index=True) # Celery task ID for tracking + + # Error information + error_message = Column(String, nullable=True) + + # Call event tracking (similar to CallRecording) + call_event = Column(String, nullable=True, index=True) # Latest call event (e.g., call_started, call_ended) + provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) + provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" + call_data = Column(JSON, nullable=True) # Full call details from provider (like CallRecording) + + # Data-plane shard routing (payload rows on shard DBs when sharding enabled) + shard_id = Column(String(64), nullable=True, index=True) + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +# Enums moved to enums.py + + +class CallRecordingSource(str, enum.Enum): + """Source of the call recording data.""" + + PLAYGROUND = "playground" + WEBHOOK = "webhook" + + +class CallRecording(Base): + """Call Recording model for tracking voice provider calls.""" + __tablename__ = "call_recordings" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every recording belongs to a workspace within + # its org. For playground-origin rows this is stamped from the active + # workspace at creation time; for webhook-origin rows the worker + # looks up the recording's agent and inherits its workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + call_short_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + status = Column(Enum(CallRecordingStatus), nullable=False, default=CallRecordingStatus.PENDING, index=True) + call_event = Column(String, nullable=True, index=True) # Latest webhook event (e.g., call_started, call_ended) + source = Column(Enum(CallRecordingSource), nullable=False, default=CallRecordingSource.PLAYGROUND, index=True) + call_data = Column(JSON, nullable=True) # JSON blob for provider response + provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) + provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Reference to our agent + + # Link to EvaluatorResult for metric evaluations + evaluator_result_id = Column( + UUID(as_uuid=True), + ForeignKey("evaluator_results.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + shard_id = Column(String(64), nullable=True, index=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class EvaluatorResultPayload(Base): + """Heavy evaluator result fields stored on data shards when sharding is enabled.""" + + __tablename__ = "evaluator_result_payloads" + + evaluator_result_id = Column(UUID(as_uuid=True), primary_key=True) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + audio_s3_key = Column(String, nullable=True) + transcription = Column(String, nullable=True) + speaker_segments = Column(JSON, nullable=True) + metric_scores = Column(JSON, nullable=True) + call_data = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class CallRecordingPayload(Base): + """Heavy call recording fields stored on data shards when sharding is enabled.""" + + __tablename__ = "call_recording_payloads" + + call_recording_id = Column(UUID(as_uuid=True), primary_key=True) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + call_data = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class Alert(Base): + """Alert model for configuring monitoring alerts.""" + __tablename__ = "alerts" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + + # Basic information + name = Column(String(255), nullable=False) + description = Column(String, nullable=True) + + # Metric condition configuration + metric_type = Column(String, nullable=False, default=AlertMetricType.NUMBER_OF_CALLS.value) + aggregation = Column(String, nullable=False, default=AlertAggregation.SUM.value) + operator = Column(String, nullable=False, default=AlertOperator.GREATER_THAN.value) + threshold_value = Column(Float, nullable=False) + time_window_minutes = Column(Integer, nullable=False, default=60) # Time window for aggregation + + # Agent selection (JSON array of agent UUIDs, null means all agents) + agent_ids = Column(JSON, nullable=True) + + # Notification configuration + notify_frequency = Column(String, nullable=False, default=AlertNotifyFrequency.IMMEDIATE.value) + notify_emails = Column(JSON, nullable=True) # Array of email addresses + notify_webhooks = Column(JSON, nullable=True) # Array of webhook URLs (Slack, etc.) + + # Status + status = Column(String, nullable=False, default=AlertStatus.ACTIVE.value) + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + # Relationships + alert_history = relationship("AlertHistory", back_populates="alert", cascade="all, delete-orphan") + + +class AlertHistory(Base): + """Alert history model for tracking triggered alerts.""" + __tablename__ = "alert_history" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + alert_id = Column(UUID(as_uuid=True), ForeignKey("alerts.id"), nullable=False, index=True) + + # Trigger information + triggered_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + triggered_value = Column(Float, nullable=False) # The actual value that triggered the alert + threshold_value = Column(Float, nullable=False) # The threshold at time of trigger + + # Status tracking + status = Column(String, nullable=False, default=AlertHistoryStatus.TRIGGERED.value) + + # Notification tracking + notified_at = Column(DateTime(timezone=True), nullable=True) + notification_details = Column(JSON, nullable=True) # Details of sent notifications + + # Resolution + acknowledged_at = Column(DateTime(timezone=True), nullable=True) + acknowledged_by = Column(String, nullable=True) + resolved_at = Column(DateTime(timezone=True), nullable=True) + resolved_by = Column(String, nullable=True) + resolution_notes = Column(String, nullable=True) + + # Additional context + context_data = Column(JSON, nullable=True) # Additional data about the trigger + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + alert = relationship("Alert", back_populates="alert_history") + + +class CronJob(Base): + """Cron job model for scheduling automated evaluator runs.""" + __tablename__ = "cron_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + + # Basic information + name = Column(String(255), nullable=False) + cron_expression = Column(String(100), nullable=False) # e.g., "0 9 * * 1-5" + timezone = Column(String(100), nullable=False, default="UTC") + + # Run configuration + max_runs = Column(Integer, nullable=False, default=10) + current_runs = Column(Integer, nullable=False, default=0) + + # Evaluators to trigger (JSON array of evaluator UUIDs) + evaluator_ids = Column(JSON, nullable=False) + + # Status + status = Column(String, nullable=False, default=CronJobStatus.ACTIVE.value) + + # Run tracking + next_run_at = Column(DateTime(timezone=True), nullable=True) + last_run_at = Column(DateTime(timezone=True), nullable=True) + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class TTSComparisonStatus(str, enum.Enum): + PENDING = "pending" + GENERATING = "generating" + EVALUATING = "evaluating" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSSampleStatus(str, enum.Enum): + PENDING = "pending" + GENERATING = "generating" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSReportJobStatus(str, enum.Enum): + PENDING = "pending" + PROCESSING = "processing" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSComparison(Base): + """TTS Comparison session for A/B testing voice providers.""" + __tablename__ = "tts_comparisons" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every voice playground comparison belongs to + # a workspace within its org. Children (samples, report jobs, blind + # test shares) inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + simulation_id = Column(String(6), unique=True, index=True, nullable=True) + + name = Column(String(255), nullable=True) + status = Column(String(50), nullable=False, default=TTSComparisonStatus.PENDING.value) + + # 'benchmark' = traditional TTS A/B benchmark (provider-generated audio). + # 'blind_test_only' = standalone blind test built from existing recordings + # / uploads / past TTS samples; no TTS generation happens. + mode = Column(String(32), nullable=False, default="benchmark") + + provider_a = Column(String(100), nullable=True) + model_a = Column(String(100), nullable=True) + voices_a = Column(JSON, nullable=True) + + provider_b = Column(String(100), nullable=True) + model_b = Column(String(100), nullable=True) + voices_b = Column(JSON, nullable=True) + + sample_texts = Column(JSON, nullable=False) + num_runs = Column(Integer, nullable=False, default=1) + + blind_test_results = Column(JSON, nullable=True) + evaluation_summary = Column(JSON, nullable=True) + + eval_stt_provider = Column(String(100), nullable=True) + eval_stt_model = Column(String(100), nullable=True) + + celery_task_id = Column(String, nullable=True, index=True) + error_message = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + samples = relationship("TTSSample", back_populates="comparison", cascade="all, delete-orphan") + + +class TTSSample(Base): + """Individual TTS audio sample within a comparison.""" + __tablename__ = "tts_samples" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + provider = Column(String(100), nullable=True) + model = Column(String(100), nullable=True) + voice_id = Column(String(255), nullable=True) + voice_name = Column(String(255), nullable=True) + side = Column(String(1), nullable=True) # "A" or "B" + sample_index = Column(Integer, nullable=False) + run_index = Column(Integer, nullable=False, default=0) + + # 'tts' (default, audio is synthesized by a provider), 'recording' (audio + # is reused from a CallImportRow recording), or 'upload' (audio was + # uploaded by the user). Non-tts samples are marked completed up-front + # by the API and skipped by the generation worker. + source_type = Column(String(32), nullable=False, default="tts") + # When source_type == 'recording', references CallImportRow.id (no FK + # constraint to keep cascading deletes simple if a call import is later + # removed; the audio_s3_key is what's actually used). + source_ref_id = Column(UUID(as_uuid=True), nullable=True) + + text = Column(String, nullable=False) + audio_s3_key = Column(String(512), nullable=True) + duration_seconds = Column(Float, nullable=True) + latency_ms = Column(Float, nullable=True) + ttfb_ms = Column(Float, nullable=True) + + evaluation_metrics = Column(JSON, nullable=True) + status = Column(String(50), nullable=False, default=TTSSampleStatus.PENDING.value) + error_message = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + comparison = relationship("TTSComparison", back_populates="samples") + + +class TTSReportJob(Base): + """Asynchronous PDF report generation jobs for Voice Playground.""" + __tablename__ = "tts_report_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) + + status = Column(String(50), nullable=False, default=TTSReportJobStatus.PENDING.value) + format = Column(String(20), nullable=False, default="pdf") + filename = Column(String(255), nullable=True) + s3_key = Column(String(512), nullable=True) + error_message = Column(String, nullable=True) + celery_task_id = Column(String, nullable=True, index=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + comparison = relationship("TTSComparison") + + +class TTSBlindTestShareStatus(str, enum.Enum): + OPEN = "open" + CLOSED = "closed" + + +class TTSBlindTestShare(Base): + """A publicly sharable blind test for a TTSComparison. + + The share_token is the capability: anyone holding it can open the public + form and submit a response. Each comparison has at most one share row. + """ + __tablename__ = "tts_blind_test_shares" + __table_args__ = ( + UniqueConstraint("comparison_id", name="uq_blind_test_shares_comparison"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + comparison_id = Column( + UUID(as_uuid=True), + ForeignKey("tts_comparisons.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + share_token = Column(String(64), unique=True, nullable=False, index=True) + + title = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + + # Internal notes visible only to the share creator (e.g. which voice + # corresponds to which side, source notes for standalone blind tests). + # Never exposed via the public blind test payload. + creator_notes = Column(Text, nullable=True) + + # JSON list: [{ "key": str, "label": str, "type": "rating"|"comment", "scale": int? }] + custom_metrics = Column(JSON, nullable=False) + + status = Column(String(20), nullable=False, default=TTSBlindTestShareStatus.OPEN.value) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + closed_at = Column(DateTime(timezone=True), nullable=True) + created_by = Column(String, nullable=True) + + comparison = relationship("TTSComparison") + responses = relationship( + "TTSBlindTestResponse", + back_populates="share", + cascade="all, delete-orphan", + ) + + +class TTSBlindTestResponse(Base): + """A single rater's submission against a TTSBlindTestShare.""" + __tablename__ = "tts_blind_test_responses" + __table_args__ = ( + UniqueConstraint("share_id", "rater_email", name="uq_blind_test_response_share_email"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + share_id = Column( + UUID(as_uuid=True), + ForeignKey("tts_blind_test_shares.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent TTSBlindTestShare's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + rater_name = Column(String(255), nullable=False) + rater_email = Column(String(320), nullable=False, index=True) + + # JSON list keyed by sample_index. Server stores in TRUE A/B orientation + # (already de-flipped from whatever the rater's UI showed): + # [{ + # "sample_index": int, + # "preferred": "A" | "B", + # "ratings_a": { metric_key: number }, + # "ratings_b": { metric_key: number }, + # "comment": str? + # }] + responses = Column(JSON, nullable=False) + + ip = Column(String(64), nullable=True) + user_agent = Column(String(512), nullable=True) + + submitted_at = Column(DateTime(timezone=True), server_default=func.now()) + + share = relationship("TTSBlindTestShare", back_populates="responses") + + +class PromptPartial(Base): + """Prompt Partial - Reusable prompt templates with version history.""" + __tablename__ = "prompt_partials" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every prompt partial belongs to a workspace + # within its org. Versions inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(String, nullable=True) + content = Column(Text, nullable=False) + tags = Column(JSON, nullable=True) + current_version = Column(Integer, nullable=False, default=1) + # Cached LLM-generated flowchart for imported production agent prompts. + # Shape: AgentFlowGraph JSON (nodes[], edges[]). + agent_flowchart = Column(JSON, nullable=True) + agent_flowchart_status = Column(String(20), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + versions = relationship("PromptPartialVersion", back_populates="prompt_partial", cascade="all, delete-orphan", order_by="PromptPartialVersion.version.desc()") + + +class PromptPartialVersion(Base): + """Version history for a prompt partial.""" + __tablename__ = "prompt_partial_versions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + prompt_partial_id = Column(UUID(as_uuid=True), ForeignKey("prompt_partials.id", ondelete="CASCADE"), nullable=False, index=True) + # Workspace isolation: mirrors the parent PromptPartial's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + version = Column(Integer, nullable=False) + content = Column(Text, nullable=False) + change_summary = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + created_by = Column(String, nullable=True) + + prompt_partial = relationship("PromptPartial", back_populates="versions") + + __table_args__ = ( + UniqueConstraint('prompt_partial_id', 'version', name='uq_prompt_partial_version'), + ) + + +class CustomTTSVoice(Base): + """Organization-scoped custom TTS voice metadata.""" + __tablename__ = "custom_tts_voices" + __table_args__ = ( + UniqueConstraint("organization_id", "provider", "voice_id", name="uq_custom_tts_voice_org_provider_voice_id"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String(100), nullable=False, index=True) + voice_id = Column(String(255), nullable=False) + name = Column(String(255), nullable=False) + gender = Column(String(50), nullable=True) + accent = Column(String(100), nullable=True) + description = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + +class PromptOptimizationRun(Base): + """A single GEPA prompt optimization run for an agent.""" + __tablename__ = "prompt_optimization_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every optimization run belongs to a workspace + # within its org. Candidates inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) + evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) + + seed_prompt = Column(Text, nullable=False) + best_prompt = Column(Text, nullable=True) + best_score = Column(Float, nullable=True) + + status = Column(String(20), nullable=False, default=PromptOptimizationStatus.PENDING.value) + config = Column(JSON, nullable=True) + reflection_trace = Column(JSON, nullable=True) + metric_history = Column(JSON, nullable=True) + + num_iterations = Column(Integer, nullable=True) + num_metric_calls = Column(Integer, nullable=True) + + celery_task_id = Column(String, nullable=True, index=True) + error_message = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + candidates = relationship("PromptOptimizationCandidate", back_populates="optimization_run", cascade="all, delete-orphan") + + +class PromptOptimizationCandidate(Base): + """A candidate prompt generated during an optimization run.""" + __tablename__ = "prompt_optimization_candidates" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + optimization_run_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_runs.id", ondelete="CASCADE"), nullable=False, index=True) + # Workspace isolation: mirrors the parent PromptOptimizationRun's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + prompt_text = Column(Text, nullable=False) + score = Column(Float, nullable=True) + metric_breakdown = Column(JSON, nullable=True) + reflection_summary = Column(Text, nullable=True) + + parent_candidate_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_candidates.id"), nullable=True) + + is_accepted = Column(Boolean, nullable=False, default=False) + pushed_to_provider_at = Column(DateTime(timezone=True), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + optimization_run = relationship("PromptOptimizationRun", back_populates="candidates") + + +class TelephonyIntegration(Base): + """Per-organization telephony provider credentials and configuration. + + Multiple rows per (organization_id, provider) are allowed so that an + organization can keep several Plivo / Exotel accounts side-by-side. + A partial unique index in migration 028 enforces at most one row with + is_default = TRUE per (org, provider); resolution falls back to that + default row when the caller does not pin a specific credential. + """ + + __tablename__ = "telephony_integrations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String(50), nullable=False, default="plivo") + name = Column(String(255), nullable=True) # Optional friendly name to disambiguate multiple credentials + + auth_id = Column(String(255), nullable=False) + auth_token = Column(String(512), nullable=False) + + verify_app_uuid = Column(String(255), nullable=True) + voice_app_id = Column(String(255), nullable=True) + sip_domain = Column(String(255), nullable=True) + masking_config = Column(JSON, nullable=True) + + is_active = Column(Boolean, default=True, nullable=False) + is_default = Column(Boolean, default=False, nullable=False) + last_tested_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyPhoneNumber(Base): + """Inventory of telephony phone numbers owned by an organization.""" + + __tablename__ = "telephony_phone_numbers" + __table_args__ = ( + UniqueConstraint("organization_id", "phone_number", name="uq_telephony_number_org_phone"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + telephony_integration_id = Column( + UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=True, index=True + ) + + phone_number = Column(String(20), nullable=False, index=True) + country_iso2 = Column(String(2), nullable=True) + region = Column(String(100), nullable=True) + number_type = Column(String(20), nullable=True) + capabilities = Column(JSON, nullable=True) + provider_app_id = Column(String(255), nullable=True) + + is_masking_pool = Column(Boolean, default=False, nullable=False) + inbound_enabled = Column(Boolean, default=True, nullable=False) + outbound_enabled = Column(Boolean, default=True, nullable=False) + source = Column(String(20), nullable=False, default="imported") + agent_id = Column( + UUID(as_uuid=True), + ForeignKey( + "agents.id", + ondelete="SET NULL", + use_alter=True, + name="fk_telephony_phone_numbers_agent_id", + ), + nullable=True, + index=True, + ) + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyDialTarget(Base): + """Org-scoped saved destination numbers for outbound test calls.""" + + __tablename__ = "telephony_dial_targets" + __table_args__ = ( + UniqueConstraint("organization_id", "phone_number", name="uq_telephony_dial_target_org_phone"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + phone_number = Column(String(20), nullable=False, index=True) + label = Column(String(255), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyVerifySession(Base): + """Tracks voice OTP verification sessions via telephony provider.""" + + __tablename__ = "telephony_verify_sessions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider_session_uuid = Column(String(255), nullable=False, unique=True, index=True) + recipient_number = Column(String(20), nullable=False) + channel = Column(String(10), nullable=False, default="voice") + status = Column(String(20), nullable=False, default="pending") + initiated_by = Column(String(255), nullable=True) + verify_app_uuid = Column(String(255), nullable=True) + verified_at = Column(DateTime(timezone=True), nullable=True) + expires_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyMaskedSession(Base): + """Number-masking session between two parties through a middle number.""" + + __tablename__ = "telephony_masked_sessions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + telephony_integration_id = Column(UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=False) + masked_number_id = Column( + UUID(as_uuid=True), ForeignKey("telephony_phone_numbers.id"), nullable=False, index=True + ) + masked_number = Column(String(20), nullable=False) + party_a_number = Column(String(20), nullable=False) + party_b_number = Column(String(20), nullable=False) + status = Column(String(20), nullable=False, default="active") + expires_at = Column(DateTime(timezone=True), nullable=True) + ended_at = Column(DateTime(timezone=True), nullable=True) + session_metadata = Column("metadata", JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class CallImportSchema(Base): + """Reusable Input Parameter schema for the call-uploads flow. + + A schema is workspace-scoped: users define a named bundle of typed + Input Parameters once (e.g. "Standard Voice QA" with conversation_id + + recording_url + transcript + agent_name) and then map those parameters + to CSV/Excel headers each time they upload a new batch. + + Every schema MUST contain exactly one parameter with + ``type='conversation_id'`` and ``is_required=True`` - that's the + mandatory identity field every imported row needs. The invariant is + enforced in app code on create/update (no DB-level CHECK because the + parent + children are written across two tables in one transaction). + """ + + __tablename__ = "call_import_schemas" + __table_args__ = ( + # Case-insensitive uniqueness is enforced via the matching partial + # index on ``LOWER(name)`` in the migration; this constraint here + # would be case-sensitive and is intentionally omitted to avoid + # confusing the user. + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + created_by_user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + parameters = relationship( + "CallImportSchemaParameter", + back_populates="schema", + cascade="all, delete-orphan", + order_by="CallImportSchemaParameter.ordering", + ) + + +class CallImportSchemaParameter(Base): + """A single typed parameter inside a :class:`CallImportSchema`. + + ``type`` is one of the strings tracked by + :data:`app.models.enums.CallImportParameterType`. ``conversation_id`` + is reserved for the mandatory identity parameter every schema must + contain. + """ + + __tablename__ = "call_import_schema_parameters" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + schema_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_schemas.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + type = Column(String(32), nullable=False) + description = Column(Text, nullable=True) + is_required = Column(Boolean, nullable=False, default=False) + # Stable ordering so the UI renders parameters in the order the + # schema author defined them (matters when conversation_id is pinned + # first and the user re-orders the rest). + ordering = Column(Integer, nullable=False, default=0) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + schema = relationship("CallImportSchema", back_populates="parameters") + + +class CallImport(Base): + """Batch record for a CSV-driven call import job.""" + + __tablename__ = "call_imports" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every imported batch belongs to a workspace + # within its org. The /upload endpoint stamps it from the active + # workspace header (or the org's Default if absent). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + created_by_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + last_updated_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + + # Telephony provider key (e.g. ``'exotel'``, ``'plivo'``). In the + # legacy one-shot ``POST /upload`` endpoint this is supplied with the + # file; in the three-stage flow (UPLOAD -> MAP -> IMPORT) the value + # isn't known until the IMPORT stage, so the column is nullable for + # ``uploaded`` / ``mapped`` batches. + provider = Column(String(50), nullable=True, default="exotel") + # Pin a specific telephony credential for this batch so the worker + # downloads recordings using *that* row instead of the org default. + # NULL preserves legacy behavior (resolve by provider + default). + telephony_integration_id = Column( + UUID(as_uuid=True), + ForeignKey("telephony_integrations.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + original_filename = Column(String(512), nullable=True) + # When the source file was a multi-sheet Excel workbook, this records + # the worksheet the rows came from (one batch per sheet). NULL for CSV + # uploads since CSV has no sheet concept. + sheet_name = Column(String(255), nullable=True) + + # --- Source-file staging (UPLOAD stage) --------------------------- + # The raw CSV / Excel file is stored in S3 between stages so the + # user can come back later to MAP and IMPORT without re-uploading. + # ``source_s3_key`` is NULL on legacy batches that were imported via + # the one-shot endpoint (those batches stay read-only post-import). + source_s3_key = Column(Text, nullable=True) + source_format = Column(String(16), nullable=True) + source_size_bytes = Column(BigInteger, nullable=True) + source_content_type = Column(String(255), nullable=True) + + # Snapshot of the file's sheets + headers captured at UPLOAD time + # so the MAP UI doesn't need to re-fetch the source bytes from S3. + # Shape: ``[{"name": str, "headers": [str, ...], "row_count": int}, ...]``. + available_sheets = Column(JSON, nullable=True) + + # User's explicit "drop these columns" decision captured at MAP + # time. Was validation-only and ephemeral in the legacy flow; now + # persisted so the IMPORT stage can re-parse the file with the same + # mapping/skip intent. + skipped_columns = Column(JSON, nullable=False, default=list) + # Rows skipped at parse time (missing/invalid conversation_id or URL). + # Shape: ``[{"source_row": int, "reason": str, "message": str}, ...]``. + source_row_skips = Column(JSON, nullable=False, default=list) + + # Free-text high-level segregation label. Powers the "Dataset" filter + # at the top of the imports page; multiple imports can share a value. + dataset = Column(String(255), nullable=True, index=True) + + # Reusable Input Parameter schema this batch was uploaded against. + # NULL on legacy batches uploaded before the schema-driven flow + # shipped; those still render via ``column_mapping`` + ``extra_columns`` + # + ``custom_column_mapping`` below. + schema_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_schemas.id", ondelete="RESTRICT"), + nullable=True, + index=True, + ) + # New schema-driven mapping: ``{schema_parameter_name: csv_header}``. + # Populated for new uploads; empty dict on legacy batches. + parameter_mapping = Column(JSON, nullable=False, default=dict) + + # Legacy free-form mapping (pre-schema-flow). Kept on the model so + # batches that were uploaded before the schema feature shipped still + # render correctly on the detail page; new uploads stop writing here. + # Keys: external_call_id (required), transcript, recording_url. + # (DB column ``external_call_id`` is now ``conversation_id``; this + # JSON key stays as-is for historical batches.) + # Values: original CSV header strings (preserve user casing for export). + column_mapping = Column(JSON, nullable=False, default=dict) + # Ordered list of additional CSV header strings the uploader wants + # preserved verbatim into the evaluation export CSV. + extra_columns = Column(JSON, nullable=False, default=list) + # User-defined ``{custom_field_name: csv_header}`` mappings on top of + # the three system fields above. Cells from the mapped CSV columns are + # preserved per row (keyed by the CSV header in ``raw_columns``) and + # surface in the evaluation export under the uploader-chosen name. + custom_column_mapping = Column(JSON, nullable=False, default=dict) + + total_rows = Column(Integer, nullable=False, default=0) + completed_rows = Column(Integer, nullable=False, default=0) + failed_rows = Column(Integer, nullable=False, default=0) + + status = Column( + Enum(CallImportStatus, values_callable=get_enum_values), + nullable=False, + default=CallImportStatus.PENDING, + index=True, + ) + error_message = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + rows = relationship( + "CallImportRow", + back_populates="call_import", + cascade="all, delete-orphan", + order_by="CallImportRow.row_index", + ) + tags = relationship( + "CallImportTag", + secondary="call_import_tag_assignments", + backref="call_imports", + lazy="selectin", + ) + evaluations = relationship( + "CallImportEvaluation", + back_populates="call_import", + cascade="all, delete-orphan", + ) + + +class CallImportShardSlice(Base): + """Registry row: which shard stores a slice of rows for an import.""" + + __tablename__ = "call_import_shard_slices" + + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + primary_key=True, + ) + slice_id = Column(Integer, primary_key=True) + shard_id = Column(String(64), nullable=False, index=True) + row_index_min = Column(Integer, nullable=False) + row_index_max = Column(Integer, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class CallImportRow(Base): + """A single row within a CallImport batch (one CSV line / one external call).""" + + __tablename__ = "call_import_rows" + __table_args__ = ( + UniqueConstraint("call_import_id", "row_index", name="uq_call_import_row_index"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + row_index = Column(Integer, nullable=False) + # Was historically named ``external_call_id``; renamed to + # ``conversation_id`` so the new schema-driven upload flow can refer + # to it by a single canonical name across the schema definition, + # exports, and downstream evaluation tables. + conversation_id = Column(String(255), nullable=False, index=True) + # Supplied via CSV for Exotel credentialed imports (required per row). + # Nullable in the schema for legacy rows imported before recording_url + # was mandatory on every Exotel upload. + recording_url = Column(Text, nullable=True) + # Date-only call recording date supplied by the import schema. Used + # for historical report comparisons without timezone/time ambiguity. + recording_date = Column(Date, nullable=True, index=True) + # The "production" transcript: the value supplied via the CSV + # upload mapping. Never overwritten by the diarisation worker — + # the worker writes its output into ``diarised_transcript`` so + # the user keeps both versions side by side. + transcript = Column(Text, nullable=True) + # Snapshot of the original CSV row keyed by the user's headers so the + # evaluation export can reproduce every column the uploader supplied + # (mapped + extra). NULL on legacy rows imported before this column. + raw_columns = Column(JSON, nullable=True) + + # Where the value in ``transcript`` came from. ``csv`` = supplied via + # the upload mapping, ``edited`` = manually changed in the UI. NULL + # on rows that have never had a production transcript. + # (Worker-produced transcripts now live in ``diarised_transcript`` + # and are tracked via ``diarised_transcript_*`` metadata below.) + transcript_source = Column(String(20), nullable=True) + # Provider/model recorded by the (legacy) post-hoc transcription + # worker. New worker runs leave these NULL and write into the + # ``diarised_transcript_*`` columns instead; kept on the model for + # backwards compatibility with pre-split rows that still carry the + # original transcription metadata here. + transcript_provider = Column(String(50), nullable=True) + transcript_model = Column(String(100), nullable=True) + # Lifecycle status for the legacy transcription workflow itself, + # independent of the row's recording-fetch ``status``. ``idle`` = + # no transcribe task has touched this column. New diarisation runs + # update ``diarised_transcript_status`` instead. + transcript_status = Column( + String(20), + nullable=False, + default="idle", + ) + transcript_error = Column(Text, nullable=True) + transcribed_at = Column(DateTime(timezone=True), nullable=True) + + # The "diarised" transcript: produced by the post-hoc + # transcription/diarisation worker. Stored separately so a manual + # diarisation run never clobbers the production transcript above. + # Evaluations can be configured to score against either column + # (see ``CallImportEvaluation.transcript_source``). + diarised_transcript = Column(Text, nullable=True) + # Provider/model the diarisation worker used. Surfaced in the UI + # as "Diarised via deepgram/nova-2" next to the diarised + # transcript section. + diarised_transcript_provider = Column(String(50), nullable=True) + diarised_transcript_model = Column(String(100), nullable=True) + # Lifecycle status for the diarisation workflow. + # ``idle`` = no diarisation task has run; ``pending``/``running`` = + # a Celery task is queued or in flight; ``completed``/``failed`` = + # terminal. Independent of ``transcript_status`` so the two + # transcripts can be in different lifecycle states. + diarised_transcript_status = Column( + String(20), + nullable=False, + default="idle", + server_default="idle", + ) + diarised_transcript_error = Column(Text, nullable=True) + diarised_at = Column(DateTime(timezone=True), nullable=True) + + # Structured speaker turns produced by the diarisation worker — + # ``[{ "speaker": "agent"|"user"|"speaker_3", "text": "...", + # "start": float, "end": float, "raw_speaker": "Speaker 1" }, ...]`` + # The plain-text ``diarised_transcript`` above is a rendered view + # of this list (``: `` per line). When the worker + # cannot recover structured turns (no pyannote token / single- + # speaker recording / provider that doesn't surface segments) this + # column stays NULL and the plain-text path is still populated. + diarised_segments = Column(JSON, nullable=True) + # When True the ``agent`` <-> ``user`` mapping inside + # ``diarised_segments`` is inverted at render / export time. The + # worker writes the canonical mapping using the "first speaker is + # the agent" heuristic; reviewers can flip the toggle from the row + # detail panel without re-running diarisation. + diarised_speaker_swap = Column( + Boolean, + nullable=False, + default=False, + server_default="false", + ) + # LLM that turned the STT plain-text output into structured + # ``diarised_segments``. The legacy diarisation worker used + # pyannote and left these NULL; the current path always runs an + # LLM with the operator-supplied (or default) ``diarised_prompt`` + # below, and records exactly which model + prompt produced each + # row so reviewers can reproduce a specific run. + diarised_llm_provider = Column(String(50), nullable=True) + diarised_llm_model = Column(String(100), nullable=True) + diarised_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + diarised_prompt = Column(Text, nullable=True) + # Which diarisation pipeline produced this row's turns. + # * ``"stt_llm"`` (default) — two-stage: STT then LLM diariser. + # ``diarised_transcript_provider``/``_model`` describe the STT + # side; ``diarised_llm_provider``/``_model`` the LLM side. + # * ``"llm_only"`` — single-stage: audio fed straight to a + # multimodal LLM. ``diarised_transcript_provider`` is stamped + # with the sentinel ``"llm_only"``; the real model is on + # ``diarised_llm_*``. + # Persisting it on the row (not just the run) lets the row detail + # panel render the right "Diarised via …" label even for ad-hoc + # standalone transcribes (no parent evaluation). + transcribe_mode = Column( + String(20), + nullable=False, + default="stt_llm", + server_default="stt_llm", + ) + + status = Column( + Enum(CallImportRowStatus, values_callable=get_enum_values), + nullable=False, + default=CallImportRowStatus.PENDING, + index=True, + ) + + recording_s3_key = Column(String(1024), nullable=True) + recording_content_type = Column(String(128), nullable=True) + recording_size_bytes = Column(Integer, nullable=True) + + error_message = Column(Text, nullable=True) + attempts = Column(Integer, nullable=False, default=0) + celery_task_id = Column(String(255), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + call_import = relationship("CallImport", back_populates="rows") + + +@event.listens_for(CallImportRow, "before_insert") +def _call_import_row_fill_workspace_id(_mapper, connection, target): + """Denormalize workspace_id from the parent import when omitted.""" + if target.workspace_id is not None or target.call_import_id is None: + return + workspace_id = connection.execute( + select(CallImport.workspace_id).where( + CallImport.id == target.call_import_id + ) + ).scalar_one_or_none() + if workspace_id is not None: + target.workspace_id = workspace_id + + +class CallImportTag(Base): + """User-defined tag that can be attached to one or more call imports. + + Tags coexist with the free-text ``CallImport.dataset`` column: dataset + is the primary high-level segregation, tags are an optional secondary + classification (an import can have many tags). + """ + + __tablename__ = "call_import_tags" + __table_args__ = ( + UniqueConstraint("organization_id", "name", name="uq_call_import_tag_org_name"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + name = Column(String(255), nullable=False) + color = Column(String(32), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class CallImportTagAssignment(Base): + """Many-to-many join table between CallImport and CallImportTag.""" + + __tablename__ = "call_import_tag_assignments" + + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + primary_key=True, + ) + tag_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_tags.id", ondelete="CASCADE"), + primary_key=True, + index=True, + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class CallImportEvaluation(Base): + """Parent record for an evaluation run over a CallImport batch. + + A user picks a subset of org ``Metric`` rows and triggers an evaluation; + we fan out one ``CallImportEvaluationRow`` per source row and roll up + counters as workers finish. Status mirrors ``CallImportStatus`` plus a + ``RUNNING`` value so the UI can distinguish "queued" from "in flight". + """ + + __tablename__ = "call_import_evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent CallImport's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + created_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id"), nullable=True + ) + last_updated_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + + # Optional user-supplied label for this run. Lets the UI surface + # something more meaningful than the UUID prefix (e.g. "March QA pass"). + name = Column(String(255), nullable=True) + + # JSON list of Metric UUID strings selected for this run. Stored as text + # in JSON so we don't have to deal with PG arrays of UUIDs / cascade + # delete policies when metrics are removed; the loader filters for + # still-existing org metrics at run time. + selected_metric_ids = Column(JSON, nullable=False, default=list) + # Hierarchy grouping snapshot: ``{parent_id_str: [child_id_str, ...]}``. + # Captures which children belong to which parent for THIS run so the UI + # / aggregator can reconstruct the tree even when the user selected + # only a subset of children, or after metrics are deleted / renamed. + # NULL on legacy rows means "no hierarchy" → fall back to flat + # ``selected_metric_ids`` semantics. + selected_metric_groups = Column(JSON, nullable=True) + # User-driven merges of LLM-discovered candidate sub-labels for + # ``allow_discovery`` parents. Shape: + # ``{"": {"": "", ...}}``. + # Populated via ``POST .../discovered-labels/merge``; consulted by + # the discovered-labels aggregator, the flow graph builder, and the + # worker so that rows finishing AFTER a merge cannot reintroduce + # the merged-away slug. Empty dict on fresh rows. + discovered_label_aliases = Column( + JSON, nullable=False, default=dict, server_default="{}" + ) + + # Per-run opt-in for top-level metric discovery. When True, the LLM + # is asked to propose brand-new top-level metrics (boolean / rating / + # category) observed in the transcripts in addition to scoring the + # ``selected_metric_ids`` for the row. Candidates surface in a + # "Discovered metrics" panel on the evaluation's Flow tab and can + # be promoted into real standalone ``Metric`` rows via + # ``POST /metrics/from-discovered``. Defaults to False so existing + # evaluation creation payloads keep their previous behaviour. + discover_new_metrics = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + # Flat slug-to-slug redirect map for user merges + tombstones of + # discovered top-level metric candidates. Mirrors + # ``discovered_label_aliases`` but is NOT nested per parent — + # top-level metric discovery is not scoped to any parent. Shape:: + # + # {"": "", ...} + # + # An empty-string value tombstones the slug so workers finishing + # later can't re-introduce it. + discovered_metric_aliases = Column( + JSON, nullable=False, default=dict, server_default="{}" + ) + + # Run-level LLM config picked from the Run Evaluation modal. NULL on + # legacy rows means "use the historical OpenAI/gpt-4o default" — the + # worker checks for this and falls back accordingly. ``llm_credential_id`` + # pins a specific AIProvider row when the org has multiple credentials + # for the same provider. + llm_provider = Column(String(50), nullable=True) + llm_model = Column(String(100), nullable=True) + llm_credential_id = Column( + UUID(as_uuid=True), + ForeignKey("aiproviders.id", ondelete="SET NULL"), + nullable=True, + ) + llm_config = Column(JSON, nullable=True) + # Optional per-metric LLM override: + # ``{"": {"provider": "...", "model": "...", "credential_id": "..."}}``. + # Each entry overrides the run-level default for that metric only; + # missing keys = use run-level default. Stored as JSON so the UI can + # round-trip arbitrary {provider, model} pairs without migrations. + metric_llm_overrides = Column(JSON, nullable=True) + + # When ``auto_transcribe`` was set on the create payload, record the + # STT provider/model used so the UI can show "Auto-transcribed via + # deepgram/nova-2" on the evaluation header. ``stt_credential_id`` is + # untyped (no FK) because STT keys may live in either ``aiproviders`` + # (OpenAI) or ``integrations`` (Deepgram, ElevenLabs) — the + # transcription service handles the lookup. + stt_provider = Column(String(50), nullable=True) + stt_model = Column(String(100), nullable=True) + stt_credential_id = Column(UUID(as_uuid=True), nullable=True) + + # Run-level LLM diariser config. Used when the create-run / + # retry-run paths chain a ``transcribe_call_import_row_task`` + # because the row is missing a diarised transcript. Persisted on + # the run so a retry uses the same diariser the original create + # call picked (unless the retry payload explicitly overrides). + diarisation_llm_provider = Column(String(50), nullable=True) + diarisation_llm_model = Column(String(100), nullable=True) + diarisation_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + diarisation_prompt = Column(Text, nullable=True) + # Mode the run was *created* with for its auto-transcribe step. + # Retry chains read this to decide whether to enqueue an STT+LLM + # transcribe or a single-stage multimodal LLM transcribe — without + # it we'd have to infer the mode from "stt_provider is NULL", which + # would silently break legacy rows that simply never configured + # auto-transcribe. See migration 041 for the column DDL. + transcribe_mode = Column( + String(20), + nullable=False, + default="stt_llm", + server_default="stt_llm", + ) + + # Which of the two transcripts on each ``CallImportRow`` this run + # scored against. ``'production'`` reads ``CallImportRow.transcript`` + # (the CSV-supplied value); ``'diarised'`` reads + # ``CallImportRow.diarised_transcript`` (the worker output). When + # the user ticks both checkboxes in the Run Evaluation modal we + # create two ``CallImportEvaluation`` rows — one per source — so + # the two scorings can be compared side-by-side. Defaults to + # ``'production'`` so legacy runs (which always read the single + # historical ``transcript`` column) keep their semantics. + transcript_source = Column( + String(20), + nullable=False, + default="production", + server_default="production", + ) + + # Cached LLM-generated TLDR rendered above the Visualizations charts. + # Populated lazily by ``POST /evaluations/{eval_id}/insights`` so we + # never auto-burn LLM tokens on page load. Shape:: + # {"narrative": str, "patterns": [str, ...], + # "generated_at": iso8601, "generated_at_completed_rows": int, + # "provider": str, "model": str} + # NULL on rows that have never been summarised. + tldr_summary = Column(JSON, nullable=True) + + # Cached LLM-generated user insights for External Audit PDF section 03. + # Populated by a background Celery job triggered alongside TLDR generation. + # Shape: EvaluationUserInsightsState JSON (status, insights[], progress, …). + user_insights = Column(JSON, nullable=True) + + # Cached per-metric failure clustering for internal diagnostics PDF/UI. + # Shape: EvaluationMetricClustersState JSON (status, groups[], …). + metric_clusters = Column(JSON, nullable=True) + + # Cached LLM-generated prompt improvement suggestions keyed to an + # imported agent (PromptPartial tagged __imported_agent__). + # Shape: EvaluationPromptImprovementsState JSON. + prompt_improvements = Column(JSON, nullable=True) + + # Cached LLM explanations for week-over-week metric deltas keyed by + # baseline evaluation id + completed row counts. + period_delta_explanations = Column(JSON, nullable=True) + + status = Column(String(20), nullable=False, default="pending", index=True) + + total_rows = Column(Integer, nullable=False, default=0) + completed_rows = Column(Integer, nullable=False, default=0) + failed_rows = Column(Integer, nullable=False, default=0) + # Flexprice pass-level delta billing watermark: rows already emitted + # on ``call_import.evaluation_completed`` for this evaluation run. + billed_completed_rows = Column( + Integer, nullable=False, default=0, server_default="0" + ) + error_message = Column(Text, nullable=True) + celery_group_id = Column(String(255), nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + call_import = relationship("CallImport", back_populates="evaluations") + row_results = relationship( + "CallImportEvaluationRow", + back_populates="evaluation", + cascade="all, delete-orphan", + ) + + +class CallImportEvaluationRow(Base): + """Per-source-row scoring output for a CallImportEvaluation parent.""" + + __tablename__ = "call_import_evaluation_rows" + __table_args__ = ( + UniqueConstraint( + "evaluation_id", "call_import_row_id", name="uq_call_import_evaluation_row" + ), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_row_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_rows.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + status = Column(String(20), nullable=False, default="pending", index=True) + # Same shape as EvaluatorResult.metric_scores: {metric_id_str: {value, type, metric_name, ...}} + metric_scores = Column(JSON, nullable=False, default=dict) + error_message = Column(Text, nullable=True) + celery_task_id = Column(String(255), nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + evaluation = relationship("CallImportEvaluation", back_populates="row_results") + source_row = relationship("CallImportRow") + + +@event.listens_for(CallImportEvaluationRow, "before_insert") +def _call_import_evaluation_row_fill_workspace_id(_mapper, connection, target): + """Denormalize workspace_id from the parent evaluation when omitted.""" + if target.workspace_id is not None or target.evaluation_id is None: + return + workspace_id = connection.execute( + select(CallImportEvaluation.workspace_id).where( + CallImportEvaluation.id == target.evaluation_id + ) + ).scalar_one_or_none() + if workspace_id is not None: + target.workspace_id = workspace_id + + +class CallImportEvaluationReportSnapshot(Base): + """Persisted PDF-report aggregate used for period-over-period deltas.""" + + __tablename__ = "call_import_evaluation_report_snapshots" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + period_label = Column(String(64), nullable=True, index=True) + period_start = Column(Date, nullable=True, index=True) + period_end = Column(Date, nullable=True, index=True) + report_config = Column(JSON, nullable=False, default=dict, server_default="{}") + selected_metric_ids = Column(JSON, nullable=False, default=list, server_default="[]") + metric_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") + insight_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") + narrative = Column(JSON, nullable=True) + total_calls = Column(Integer, nullable=False, default=0) + selected_metric_count = Column(Integer, nullable=False, default=0) + total_metric_count = Column(Integer, nullable=False, default=0) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class CallImportEvaluationPdfReport(Base): + """Stored PDF artifact for a call import evaluation report generation.""" + + __tablename__ = "call_import_evaluation_pdf_reports" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + snapshot_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluation_report_snapshots.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + vendor_name = Column(String(120), nullable=False) + report_type = Column(String(20), nullable=False, default="external") + filename = Column(String(255), nullable=True) + s3_key = Column(String(512), nullable=True) + report_config = Column(JSON, nullable=False, default=dict, server_default="{}") + cache_fingerprint = Column(String(64), nullable=True) + created_by = Column(String, nullable=True) + created_by_user_id = Column(UUID(as_uuid=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +# --------------------------------------------------------------------------- +# Judge Alignment (AlignEval-style hybrid integration) +# +# Three tables back the "Judge Alignment" surface: +# - JudgeDataset: a labeled dataset materialised from one of three sources +# (voice transcripts, existing Metric/Evaluator outputs, +# or a generic CSV upload). Holds the dataset's source +# config + which fields play the role of input/output. +# - JudgeSample: one row in a dataset (input/output pair plus an +# optional binary pass/fail human label). +# - JudgeRun: a single run of an LLM-judge (existing Evaluator) over +# a subset of samples, with computed alignment metrics +# (precision/recall/F1/Cohen's kappa) and per-sample +# predictions. Optionally links to a GEPA optimization +# run when the user kicks off prompt tuning from a +# dataset. +# --------------------------------------------------------------------------- + + +class JudgeDataset(Base): + """Container for binary-labeled samples used to calibrate an LLM-judge.""" + + __tablename__ = "judge_datasets" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + # Workspace isolation: every judge dataset belongs to a workspace + # within its org. Samples and runs inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + + # One of: "transcript", "metric_output", "csv" + source_type = Column(String(32), nullable=False, index=True) + # Source-specific config. Examples: + # transcript: {"transcription_ids": [...]} or {"agent_id": "..."} + # metric_output: {"metric_id": "...", "evaluator_id": "..."} + # csv: {"s3_key": "...", "filename": "..."} + source_config = Column(JSON, nullable=False, default=dict) + + # Field roles - which textual content is "input" vs "output" for the judge. + # For voice transcripts both default to the transcript text but can be + # tightened (e.g. agent-only turns vs full conversation). + input_field = Column(String(64), nullable=False, default="input") + output_field = Column(String(64), nullable=False, default="output") + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + samples = relationship( + "JudgeSample", + back_populates="dataset", + cascade="all, delete-orphan", + order_by="JudgeSample.created_at", + ) + runs = relationship( + "JudgeRun", + back_populates="dataset", + cascade="all, delete-orphan", + order_by="JudgeRun.created_at.desc()", + ) + + +class JudgeSample(Base): + """One labelable input/output pair within a JudgeDataset.""" + + __tablename__ = "judge_samples" + __table_args__ = ( + UniqueConstraint("dataset_id", "external_id", name="uq_judge_samples_dataset_external"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + dataset_id = Column( + UUID(as_uuid=True), + ForeignKey("judge_datasets.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent JudgeDataset's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Stable identifier within the source (e.g. transcription UUID, CSV row id). + # Used to dedupe re-imports and link back to the originating record. + external_id = Column(String(128), nullable=True, index=True) + + input_text = Column(Text, nullable=False) + output_text = Column(Text, nullable=False) + + # Binary human label: "pass" | "fail" | null (unlabeled). + # Stored as string (rather than enum) so it stays trivially extendable. + label = Column(String(16), nullable=True, index=True) + labeled_by = Column(String(255), nullable=True) + labeled_at = Column(DateTime(timezone=True), nullable=True) + + # Source-specific context (e.g. agent_id, original metric value, csv row). + extra = Column(JSON, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + dataset = relationship("JudgeDataset", back_populates="samples") + + +class JudgeRun(Base): + """One execution of an LLM-judge against a JudgeDataset, with alignment metrics.""" + + __tablename__ = "judge_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + dataset_id = Column( + UUID(as_uuid=True), + ForeignKey("judge_datasets.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + # Workspace isolation: mirrors the parent JudgeDataset's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Reuses the existing Evaluator row (its custom_prompt + llm_provider + llm_model + # define the judge under test). Nullable so a run may target an inline prompt + # in the future without inflating the Evaluator table. + evaluator_id = Column( + UUID(as_uuid=True), ForeignKey("evaluators.id", ondelete="SET NULL"), nullable=True, index=True + ) + + # Which subset was scored: "all" | "dev" | "test" + split = Column(String(16), nullable=False, default="all") + + # Snapshot of the model used (so a later Evaluator edit doesn't rewrite history). + llm_provider = Column(String(64), nullable=True) + llm_model = Column(String(128), nullable=True) + + # Computed alignment metrics: + # {"precision": float, "recall": float, "f1": float, "kappa": float, + # "tp": int, "fp": int, "tn": int, "fn": int, "n": int} + metrics = Column(JSON, nullable=True) + + # Per-sample predictions, keyed by sample_id (UUID string): + # {sample_id: {"prediction": "pass"|"fail", "explanation": str, "raw": str}} + predictions = Column(JSON, nullable=True) + + # Run lifecycle. + status = Column(String(20), nullable=False, default="pending", index=True) + error_message = Column(Text, nullable=True) + celery_task_id = Column(String, nullable=True, index=True) + + # Optional link to a GEPA optimization run kicked off from this dataset. + gepa_optimization_id = Column( + UUID(as_uuid=True), + ForeignKey("prompt_optimization_runs.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + dataset = relationship("JudgeDataset", back_populates="runs") diff --git a/app/models/schemas.py b/app/models/schemas.py index 78dd14f9..de5c203e 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -3168,6 +3168,8 @@ class CallImportResponse(BaseModel): error_message: Optional[str] = None created_at: datetime updated_at: datetime + created_by_email: Optional[str] = None + last_updated_by_email: Optional[str] = None model_config = ConfigDict(from_attributes=True) @@ -3981,6 +3983,8 @@ class CallImportEvaluationResponse(BaseModel): finished_at: Optional[datetime] = None created_at: datetime updated_at: datetime + created_by_email: Optional[str] = None + last_updated_by_email: Optional[str] = None # Cached LLM-generated TLDR for the Visualizations tab. Lazily # populated by ``POST /evaluations/{eval_id}/insights``; ``None`` # for runs the user has not summarised yet. ``is_stale`` on the diff --git a/app/services/call_imports/audit.py b/app/services/call_imports/audit.py new file mode 100644 index 00000000..cc940562 --- /dev/null +++ b/app/services/call_imports/audit.py @@ -0,0 +1,103 @@ +"""Actor stamping and email resolution for call import audit fields.""" + +from __future__ import annotations + +from typing import Dict, Iterable, Optional, Set, Tuple +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.core.auth.principal import Principal +from app.models.database import CallImport, CallImportEvaluation, User + + +def stamp_call_import_actor( + call_import: CallImport, + principal: Principal, + *, + creating: bool = False, +) -> None: + if creating and principal.user_id is not None: + call_import.created_by_user_id = principal.user_id + if principal.user_id is not None: + call_import.last_updated_by_user_id = principal.user_id + + +def stamp_evaluation_actor( + evaluation: CallImportEvaluation, + principal: Principal, + *, + creating: bool = False, +) -> None: + if creating and principal.user_id is not None: + evaluation.created_by_user_id = principal.user_id + if principal.user_id is not None: + evaluation.last_updated_by_user_id = principal.user_id + + +def user_ids_from_call_imports(imports: Iterable[CallImport]) -> Set[UUID]: + ids: Set[UUID] = set() + for row in imports: + created_by = getattr(row, "created_by_user_id", None) + updated_by = getattr(row, "last_updated_by_user_id", None) + if created_by is not None: + ids.add(created_by) + if updated_by is not None: + ids.add(updated_by) + return ids + + +def user_ids_from_evaluations( + evaluations: Iterable[CallImportEvaluation], +) -> Set[UUID]: + ids: Set[UUID] = set() + for row in evaluations: + created_by = getattr(row, "created_by_user_id", None) + updated_by = getattr(row, "last_updated_by_user_id", None) + if created_by is not None: + ids.add(created_by) + if updated_by is not None: + ids.add(updated_by) + return ids + + +def emails_for_user_ids(db: Session, user_ids: Iterable[UUID]) -> Dict[UUID, str]: + unique = {uid for uid in user_ids if uid is not None} + if not unique: + return {} + rows = db.query(User.id, User.email).filter(User.id.in_(unique)).all() + return {row.id: row.email for row in rows if row.email} + + +def actor_emails_for_call_import( + call_import: CallImport, + email_by_id: Dict[UUID, str], +) -> Tuple[Optional[str], Optional[str]]: + created = ( + email_by_id.get(call_import.created_by_user_id) + if call_import.created_by_user_id + else None + ) + updated = ( + email_by_id.get(call_import.last_updated_by_user_id) + if call_import.last_updated_by_user_id + else None + ) + return created, updated + + +def actor_emails_for_evaluation( + evaluation: CallImportEvaluation, + email_by_id: Dict[UUID, str], +) -> Tuple[Optional[str], Optional[str]]: + created = ( + email_by_id.get(evaluation.created_by_user_id) + if evaluation.created_by_user_id + else None + ) + updated = ( + email_by_id.get(evaluation.last_updated_by_user_id) + if evaluation.last_updated_by_user_id + else None + ) + return created, updated diff --git a/app/services/reporting/call_import_pdf_report_storage.py b/app/services/reporting/call_import_pdf_report_storage.py new file mode 100644 index 00000000..17992d05 --- /dev/null +++ b/app/services/reporting/call_import_pdf_report_storage.py @@ -0,0 +1,207 @@ +"""Helpers for call import evaluation PDF report storage and config fingerprinting.""" + +from __future__ import annotations + +import hashlib +import json +from typing import TYPE_CHECKING, Any +from uuid import UUID + +from app.services.storage.s3_service import s3_service + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + from app.models.database import CallImportEvaluationPdfReport + + +def build_pdf_report_s3_key( + *, + organization_id: UUID, + call_import_id: UUID, + evaluation_id: UUID, + report_id: UUID, +) -> str: + prefix = s3_service.prefix or "" + return ( + f"{prefix}organizations/{organization_id}/call_imports/{call_import_id}/" + f"evaluations/{evaluation_id}/reports/{report_id}.pdf" + ) + + +def _canonicalize_report_config(value: Any) -> Any: + if isinstance(value, dict): + return {str(k): _canonicalize_report_config(v) for k, v in sorted(value.items())} + if isinstance(value, list): + normalized = [_canonicalize_report_config(item) for item in value] + try: + return sorted( + normalized, + key=lambda item: json.dumps(item, sort_keys=True, default=str), + ) + except TypeError: + return normalized + return value + + +def _fingerprint_digest(payload: dict[str, Any]) -> str: + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _json_fingerprint_value(value: Any) -> Any: + if value is None: + return None + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + if isinstance(value, dict): + return {str(k): _json_fingerprint_value(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_json_fingerprint_value(item) for item in value] + return value + + +def compute_pdf_report_config_fingerprint( + *, + report_type: str, + include_period_delta: bool, + include_weekly_delta: bool, + baseline_evaluation_id: str | None, + internal_brand_image_id: str | None, + external_brand_image_id: str | None, + use_case: str | None, + report_config: dict[str, Any], + report_heading: str | None, + vendor_name: str | None = None, + platform_base_url: str | None = None, + period_label: str | None = None, +) -> str: + payload = { + "report_type": report_type, + "include_period_delta": include_period_delta, + "include_weekly_delta": include_weekly_delta, + "baseline_evaluation_id": baseline_evaluation_id, + "internal_brand_image_id": internal_brand_image_id, + "external_brand_image_id": external_brand_image_id, + "use_case": use_case, + "report_config": _canonicalize_report_config(report_config or {}), + "report_heading": (report_heading or "").strip(), + "vendor_name": (vendor_name or "").strip(), + "platform_base_url": (platform_base_url or "").strip(), + "period_label": (period_label or "").strip(), + } + return _fingerprint_digest(payload) + + +def compute_pdf_report_content_fingerprint( + *, + evaluation_status: str, + completed_rows: int, + total_rows: int, + failed_rows: int, + metric_aggregates: list[dict[str, Any]], + insight_aggregates: list[dict[str, Any]], + period_delta_by_metric: dict[str, Any], + benchmark_context: Any, + metric_metadata: list[dict[str, Any]], + failure_policies: dict[str, Any], + tldr_summary: Any = None, + user_insights_for_pdf: Any = None, + metric_clusters_for_pdf: Any = None, + prompt_improvements_for_pdf: Any = None, +) -> str: + payload = { + "evaluation_status": evaluation_status, + "completed_rows": completed_rows, + "total_rows": total_rows, + "failed_rows": failed_rows, + "metric_aggregates": _canonicalize_report_config(metric_aggregates or []), + "insight_aggregates": _canonicalize_report_config(insight_aggregates or []), + "period_delta_by_metric": _canonicalize_report_config(period_delta_by_metric or {}), + "benchmark_context": _json_fingerprint_value(benchmark_context), + "metric_metadata": _canonicalize_report_config(metric_metadata or []), + "failure_policies": _json_fingerprint_value(failure_policies or {}), + "tldr_summary": _json_fingerprint_value(tldr_summary), + "user_insights_for_pdf": _json_fingerprint_value(user_insights_for_pdf), + "metric_clusters_for_pdf": _json_fingerprint_value(metric_clusters_for_pdf), + "prompt_improvements_for_pdf": _json_fingerprint_value(prompt_improvements_for_pdf), + } + return _fingerprint_digest(payload) + + +def compute_pdf_report_cache_fingerprint( + *, + config_fingerprint: str, + content_fingerprint: str, +) -> str: + combined = f"{config_fingerprint}:{content_fingerprint}" + return hashlib.sha256(combined.encode("utf-8")).hexdigest() + + +def find_cached_pdf_report( + db: "Session", + *, + evaluation_id: UUID, + organization_id: UUID, + cache_fingerprint: str, +) -> "CallImportEvaluationPdfReport | None": + from sqlalchemy import desc + + from app.models.database import CallImportEvaluationPdfReport + + if not cache_fingerprint: + return None + return ( + db.query(CallImportEvaluationPdfReport) + .filter( + CallImportEvaluationPdfReport.evaluation_id == evaluation_id, + CallImportEvaluationPdfReport.organization_id == organization_id, + CallImportEvaluationPdfReport.cache_fingerprint == cache_fingerprint, + CallImportEvaluationPdfReport.s3_key.isnot(None), + ) + .order_by(desc(CallImportEvaluationPdfReport.created_at)) + .first() + ) + + +def config_summary_from_report_config(report_config: dict[str, Any] | None) -> str: + cfg = report_config if isinstance(report_config, dict) else {} + quality_ids = cfg.get("quality_metric_ids") or [] + insight_ids = cfg.get("insights") or [] + user_insight_ids = cfg.get("user_insight_ids") or [] + metric_count = len(quality_ids) if isinstance(quality_ids, list) else 0 + insight_count = len(insight_ids) if isinstance(insight_ids, list) else 0 + user_count = len(user_insight_ids) if isinstance(user_insight_ids, list) else 0 + parts: list[str] = [] + if metric_count: + parts.append(f"{metric_count} quality metric{'s' if metric_count != 1 else ''}") + if insight_count: + parts.append(f"{insight_count} insight{'s' if insight_count != 1 else ''}") + if user_count: + parts.append(f"{user_count} user insight{'s' if user_count != 1 else ''}") + return ", ".join(parts) if parts else "default sections" + + +def presigned_urls_for_pdf_report( + s3_key: str, + filename: str, + *, + expiration: int = 3600, +) -> tuple[str | None, str | None]: + if not s3_key or not s3_service.is_enabled(): + return None, None + safe_name = filename.replace('"', "'") + try: + preview_url = s3_service.generate_presigned_url_by_key( + s3_key, + expiration=expiration, + response_content_disposition="inline", + ) + download_url = s3_service.generate_presigned_url_by_key( + s3_key, + expiration=expiration, + response_content_disposition=f'attachment; filename="{safe_name}"', + ) + return preview_url, download_url + except Exception: + return None, None diff --git a/app/services/storage/azure_blob_service.py b/app/services/storage/azure_blob_service.py index 3b1cb19e..9e6e8425 100644 --- a/app/services/storage/azure_blob_service.py +++ b/app/services/storage/azure_blob_service.py @@ -553,7 +553,13 @@ def generate_presigned_url( key = self._get_key(file_id, file_format) return self.generate_presigned_url_by_key(key, expiration=expiration) - def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str: + def generate_presigned_url_by_key( + self, + key: str, + expiration: int = 3600, + *, + response_content_disposition: str | None = None, + ) -> str: """Generate a SAS URL for temporary file access by key.""" self._ensure_initialized() if not self.is_enabled(): @@ -574,14 +580,17 @@ def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str ) try: - sas_token = generate_blob_sas( - account_name=account_name, - container_name=self.bucket_name, - blob_name=key, - account_key=account_key, - permission=BlobSasPermissions(read=True), - expiry=datetime.now(UTC) + timedelta(seconds=expiration), - ) + sas_kwargs: dict = { + "account_name": account_name, + "container_name": self.bucket_name, + "blob_name": key, + "account_key": account_key, + "permission": BlobSasPermissions(read=True), + "expiry": datetime.now(UTC) + timedelta(seconds=expiration), + } + if response_content_disposition: + sas_kwargs["content_disposition"] = response_content_disposition + sas_token = generate_blob_sas(**sas_kwargs) blob_client = self.container_client.get_blob_client(key) return f"{blob_client.url}?{sas_token}" except Exception as e: diff --git a/app/services/storage/blob_storage_service.py b/app/services/storage/blob_storage_service.py index 5f4b3bc4..03374464 100644 --- a/app/services/storage/blob_storage_service.py +++ b/app/services/storage/blob_storage_service.py @@ -128,8 +128,18 @@ def generate_presigned_url( ) -> str: return self._backend().generate_presigned_url(file_id, file_format, expiration) - def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str: - return self._backend().generate_presigned_url_by_key(key, expiration) + def generate_presigned_url_by_key( + self, + key: str, + expiration: int = 3600, + *, + response_content_disposition: str | None = None, + ) -> str: + return self._backend().generate_presigned_url_by_key( + key, + expiration, + response_content_disposition=response_content_disposition, + ) blob_storage_service = BlobStorageService() diff --git a/app/services/storage/gcs_service.py b/app/services/storage/gcs_service.py index 06a53550..07213e62 100644 --- a/app/services/storage/gcs_service.py +++ b/app/services/storage/gcs_service.py @@ -553,7 +553,13 @@ def generate_presigned_url( key = self._get_key(file_id, file_format) return self.generate_presigned_url_by_key(key, expiration=expiration) - def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str: + def generate_presigned_url_by_key( + self, + key: str, + expiration: int = 3600, + *, + response_content_disposition: str | None = None, + ) -> str: """Generate a signed URL for temporary file access by key.""" self._ensure_initialized() if not self.is_enabled(): @@ -567,23 +573,27 @@ def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str if credentials is None and iam_params is None: raise StorageError(_GCS_SIGNING_UNAVAILABLE_MSG) + signed_url_kwargs: dict = { + "version": "v4", + "expiration": timedelta(seconds=expiration), + "method": "GET", + } + if response_content_disposition: + signed_url_kwargs["response_disposition"] = response_content_disposition + try: blob = self.bucket.blob(key) if credentials is not None: url = blob.generate_signed_url( - version="v4", - expiration=timedelta(seconds=expiration), - method="GET", credentials=credentials, + **signed_url_kwargs, ) else: sa_email, access_token = iam_params url = blob.generate_signed_url( - version="v4", - expiration=timedelta(seconds=expiration), - method="GET", service_account_email=sa_email, access_token=access_token, + **signed_url_kwargs, ) return url except GoogleCloudError as e: diff --git a/app/services/storage/s3_service.py b/app/services/storage/s3_service.py index 1c22d168..b8376a85 100644 --- a/app/services/storage/s3_service.py +++ b/app/services/storage/s3_service.py @@ -478,7 +478,13 @@ def generate_presigned_url(self, file_id: uuid.UUID, file_format: str, expiratio except Exception as e: raise StorageError(f"Unexpected error generating presigned URL: {str(e)}") - def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str: + def generate_presigned_url_by_key( + self, + key: str, + expiration: int = 3600, + *, + response_content_disposition: str | None = None, + ) -> str: """Generate a presigned URL for temporary file access by key.""" self._ensure_initialized() if not self.is_enabled(): @@ -486,9 +492,12 @@ def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str raise StorageError(error_msg) try: + params: dict[str, str] = {"Bucket": self.bucket_name, "Key": key} + if response_content_disposition: + params["ResponseContentDisposition"] = response_content_disposition url = self.s3_client.generate_presigned_url( "get_object", - Params={"Bucket": self.bucket_name, "Key": key}, + Params=params, ExpiresIn=expiration, ) return url diff --git a/app/workers/concurrency/eval_dispatch.py b/app/workers/concurrency/eval_dispatch.py index 89debff8..f2dd59d7 100644 --- a/app/workers/concurrency/eval_dispatch.py +++ b/app/workers/concurrency/eval_dispatch.py @@ -163,6 +163,28 @@ def recover_eval_row_for_eval_chain(eval_row: CallImportEvaluationRow) -> None: eval_row.finished_at = None +def build_eval_chain_import_apply_async( + *, + source_row: CallImportRow, + eval_row: CallImportEvaluationRow, + reserved_task_id: str, +): + """Build a Celery ``apply_async`` for eval-chain recording import.""" + from app.workers.tasks.process_call_import_row import ( + process_call_import_row_task, + ) + + return process_call_import_row_task.apply_async( + args=(str(source_row.id),), + kwargs={ + "_eval_slot_task_id": reserved_task_id, + "run_eval_row_id": str(eval_row.id), + }, + queue=IMPORTS_QUEUE, + task_id=reserved_task_id, + ) + + def build_eval_chain_transcribe_apply_async( *, evaluation: CallImportEvaluation, @@ -376,10 +398,6 @@ def _try_dispatch_single_row( from app.workers.tasks.evaluate_call_import_row_core import ( row_needs_audio_phase, ) - from app.workers.tasks.process_call_import_row import ( - process_call_import_row_task, - ) - attached = _attach_sharded_eval_dispatch_rows( db, evaluation, @@ -442,14 +460,10 @@ def _try_dispatch_single_row( def _enqueue_import(reserved_task_id: str): source_row.celery_task_id = reserved_task_id mutate_db.flush() - return process_call_import_row_task.apply_async( - args=(str(source_row.id),), - kwargs={ - "_eval_slot_task_id": reserved_task_id, - "run_eval_row_id": str(eval_row.id), - }, - queue=IMPORTS_QUEUE, - task_id=reserved_task_id, + return build_eval_chain_import_apply_async( + source_row=source_row, + eval_row=eval_row, + reserved_task_id=reserved_task_id, ) if _reserve_slot_and_enqueue( diff --git a/config.docker.yml b/config.docker.yml index 6b84d505..490f43d3 100644 --- a/config.docker.yml +++ b/config.docker.yml @@ -134,4 +134,4 @@ judge_alignment: # Enterprise License (JWT signed with RS256). Unlocks gated features like # oidc_sso, mfa_enforce, audit_export, voice_playground, gepa_optimization. # license: -# key: "eyJhbGciOi..." +# key: "eyJhbGciOi..." \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 066b7832..552884f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,6 +47,8 @@ services: context: . dockerfile: docker/Dockerfile.api container_name: efficientai_api + env_file: + - .env environment: DATABASE_URL: postgresql://${POSTGRES_USER:-efficientai}:${POSTGRES_PASSWORD:-password}@db:5432/${POSTGRES_DB:-efficientai} REDIS_URL: redis://redis:6379/0 @@ -95,6 +97,8 @@ services: args: INSTALL_EXTRAS: "qualitative-voice" container_name: efficientai_media + env_file: + - .env environment: DATABASE_URL: postgresql://${POSTGRES_USER:-efficientai}:${POSTGRES_PASSWORD:-password}@db:5432/${POSTGRES_DB:-efficientai} REDIS_URL: redis://redis:6379/0 @@ -126,6 +130,8 @@ services: args: INSTALL_EXTRAS: "qualitative-voice" container_name: efficientai_worker + env_file: + - .env environment: # Worker uses Docker network, so it reaches DB/Redis via service names DATABASE_URL: postgresql://${POSTGRES_USER:-efficientai}:${POSTGRES_PASSWORD:-password}@db:5432/${POSTGRES_DB:-efficientai} @@ -162,6 +168,8 @@ services: args: INSTALL_EXTRAS: "" container_name: efficientai_worker_imports + env_file: + - .env environment: DATABASE_URL: postgresql://${POSTGRES_USER:-efficientai}:${POSTGRES_PASSWORD:-password}@db:5432/${POSTGRES_DB:-efficientai} REDIS_URL: redis://redis:6379/0 diff --git a/env.example b/env.example index ff2f226a..cd34c25a 100644 --- a/env.example +++ b/env.example @@ -24,6 +24,12 @@ UPLOAD_DIR=/app/uploads MAX_FILE_SIZE_MB=500 ALLOWED_AUDIO_FORMATS=wav,mp3,flac,m4a +# S3 (optional — used when config.docker.yml references ${S3_ACCESS_KEY_ID}) +# Required for call-import PDF report history (stored artifacts); without blob storage, +# PDF generation still streams inline but is not versioned. +# S3_ACCESS_KEY_ID= +# S3_SECRET_ACCESS_KEY= + # Celery Configuration CELERY_BROKER_URL=redis://redis:6379/0 CELERY_RESULT_BACKEND=redis://redis:6379/0 @@ -41,7 +47,7 @@ RATE_LIMIT_PER_MINUTE=60 # Comma-separated or JSON list. See config.yml.example for full docs. # OSS default: api_key,local_password # Enterprise SSO: api_key,external_oidc (Okta / Azure AD / Google / Cognito / Auth0 / ...) -AUTH_PROVIDERS=api_key,local_password +AUTH_PROVIDERS=["api_key","local_password"] # Local password (HS256 Bearer tokens signed with SECRET_KEY) AUTH_LOCAL_TOKEN_TTL_MINUTES=15 diff --git a/frontend/src/components/callImports/AuditMetaChips.tsx b/frontend/src/components/callImports/AuditMetaChips.tsx new file mode 100644 index 00000000..fbe0d8c6 --- /dev/null +++ b/frontend/src/components/callImports/AuditMetaChips.tsx @@ -0,0 +1,142 @@ +import type { ReactNode } from 'react' + +/** Inline audit metadata for call imports and evaluations. */ + +export function formatMetaDateTime(iso: string | null | undefined): string { + if (!iso) return '—' + const parsed = new Date(iso) + if (Number.isNaN(parsed.getTime())) return '—' + return parsed.toLocaleString() +} + +type AuditMetaInlineItemProps = { + label: string + value: string | null | undefined + /** `text-xs` for dense rows (evaluation list); default `text-sm` */ + dense?: boolean + className?: string +} + +export function AuditMetaChip({ + label, + value, + dense, + className = '', +}: AuditMetaInlineItemProps) { + const display = value?.trim() || '—' + const sizeClass = dense ? 'text-xs' : 'text-sm' + return ( + + {label}: + {display} + + ) +} + +type AuditMetaRowProps = { + className?: string + dense?: boolean + /** Join parent flex row (chips become siblings of status/provider). */ + inline?: boolean + children: ReactNode +} + +function AuditMetaRow({ className = '', dense, inline, children }: AuditMetaRowProps) { + const textClass = dense ? 'text-xs' : 'text-sm' + if (inline) { + return
{children}
+ } + return ( +
+ {children} +
+ ) +} + +type CallImportAuditMetaProps = { + createdAt: string | null | undefined + updatedAt: string | null | undefined + createdByEmail?: string | null + lastUpdatedByEmail?: string | null + className?: string + dense?: boolean + inline?: boolean +} + +/** Created / updated timestamps + actor emails for a call-import batch. */ +export function CallImportAuditMeta({ + createdAt, + updatedAt, + createdByEmail, + lastUpdatedByEmail, + className, + dense, + inline, +}: CallImportAuditMetaProps) { + return ( + + + + + + + ) +} + +type EvaluationAuditMetaProps = { + createdAt?: string | null | undefined + updatedAt?: string | null | undefined + startedAt?: string | null + finishedAt?: string | null + runByEmail?: string | null + createdByEmail?: string | null + lastUpdatedByEmail?: string | null + formatDate?: (iso: string | null | undefined) => string + className?: string + dense?: boolean + inline?: boolean + showRunTimes?: boolean + showTimestamps?: boolean +} + +/** Evaluation run metadata (detail header or list card). */ +export function EvaluationAuditMeta({ + createdAt, + updatedAt, + startedAt, + finishedAt, + runByEmail, + createdByEmail, + lastUpdatedByEmail, + formatDate = formatMetaDateTime, + className, + dense, + inline, + showRunTimes = true, + showTimestamps = false, +}: EvaluationAuditMetaProps) { + const runner = runByEmail ?? createdByEmail + return ( + + + + {showTimestamps && createdAt ? ( + + ) : null} + {showTimestamps && updatedAt ? ( + + ) : null} + {showRunTimes && startedAt ? ( + + ) : null} + {showRunTimes && finishedAt ? ( + + ) : null} + + ) +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 04e1b025..15724db3 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -43,6 +43,8 @@ import type { CallImportPreviewResponse, CallImportEvaluation, CallImportEvaluationBaselineCandidatesResponse, + CallImportEvaluationPdfReport, + CallImportEvaluationPdfReportListResponse, CallImportEvaluationLLMOverride, CallImportEvaluationListResponse, CallImportEvaluationRow, @@ -2882,7 +2884,7 @@ class ApiClient { reportConfig?: Record platformBaseUrl?: string | null }, - ): Promise { + ): Promise { const response = await this.client.post( `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/pdf-report`, { @@ -2897,7 +2899,33 @@ class ApiClient { report_config: options?.reportConfig || {}, platform_base_url: options?.platformBaseUrl || null, }, - { responseType: 'blob' }, + { responseType: 'arraybuffer' }, + ) + const contentType = String(response.headers['content-type'] || '') + if (contentType.includes('application/pdf')) { + return new Blob([response.data], { type: 'application/pdf' }) + } + const text = new TextDecoder().decode(response.data) + return JSON.parse(text) as CallImportEvaluationPdfReport + } + + async listCallImportEvaluationPdfReports( + callImportId: string, + evaluationId: string, + ): Promise { + const response = await this.client.get( + `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/pdf-reports`, + ) + return response.data + } + + async getCallImportEvaluationPdfReport( + callImportId: string, + evaluationId: string, + reportId: string, + ): Promise { + const response = await this.client.get( + `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/pdf-reports/${reportId}`, ) return response.data } diff --git a/frontend/src/pages/callImports/CallImportDetail.tsx b/frontend/src/pages/callImports/CallImportDetail.tsx index bc8687b8..98a41f3a 100644 --- a/frontend/src/pages/callImports/CallImportDetail.tsx +++ b/frontend/src/pages/callImports/CallImportDetail.tsx @@ -57,6 +57,9 @@ import Button from '../../components/Button' import ConfirmModal from '../../components/ConfirmModal' import Pagination from '../../components/Pagination' import StatusBadge from '../../components/shared/StatusBadge' +import { + CallImportAuditMeta, +} from '../../components/callImports/AuditMetaChips' import DiariseStatusPill from '../../components/callImports/DiariseStatusPill' import ProviderModelPicker, { type ProviderModelValue, @@ -1234,9 +1237,6 @@ export default function CallImportDetail() { }, }) - // Tracks which evaluation run is currently being aborted so we can show - // a tiny spinner inline on its Abort button without blocking the rest - // of the evaluations list. Cleared on success or error. const [cancellingEvalId, setCancellingEvalId] = useState(null) const cancelEvaluationMutation = useMutation({ @@ -1251,12 +1251,7 @@ export default function CallImportDetail() { } queryClient.invalidateQueries({ queryKey: evaluationsQueryKey }) }, - onError: (err: any) => { - // Cancel is idempotent on the server — the most likely failure - // is a 404 because the run was just deleted, in which case the - // refetch above will reconcile state. Surface a console.error - // so dev-tools shows it without breaking the layout. - // eslint-disable-next-line no-console + onError: (err: unknown) => { console.error('cancelEvaluationMutation failed:', err) }, onSettled: () => { @@ -1264,12 +1259,7 @@ export default function CallImportDetail() { }, }) - // Bulk-abort companion to ``cancelEvaluationMutation`` — fires the - // single-run cancel endpoint per selected run in parallel. We - // deliberately keep this client-side fan-out (rather than a new - // bulk endpoint) so the UI surface stays small; cancel is cheap - // server-side because each call only revokes the rows that are - // still in-flight. + // Bulk-abort — fires the single-run cancel endpoint per selected run in parallel. const bulkCancelEvalsMutation = useMutation({ mutationFn: async (ids: string[]) => { const results = await Promise.allSettled( @@ -1668,24 +1658,25 @@ export default function CallImportDetail() { {data.original_filename || '(unnamed import)'}
{data.id}
-
+
- + Provider:{' '} - + {data.provider || ( - + not selected yet )} - - Created: {new Date(data.created_at).toLocaleString()} - - - Updated: {new Date(data.updated_at).toLocaleString()} - +
@@ -3123,32 +3114,51 @@ export default function CallImportDetail() { No evaluations have been run for this dataset yet.

) : ( -
- {(() => { - const items = evaluationsData?.items ?? [] - const allSelected = - items.length > 0 && - items.every((row) => selectedEvalIds.has(row.id)) - return ( -
- { - if (e.target.checked) { - setSelectedEvalIds(new Set(items.map((row) => row.id))) - } else { - setSelectedEvalIds(new Set()) - } - }} - /> - - Select all ({items.length}) - -
- ) - })()} +
+ + + + + + + + + + + + {evaluationsData?.items.map((evaluation: CallImportEvaluation) => { const isSelected = selectedEvalIds.has(evaluation.id) const bulkOperationActive = evaluationHasActiveBulkOperation( @@ -3157,91 +3167,118 @@ export default function CallImportDetail() { const headerLabel = evaluation.name?.trim() ? evaluation.name : `Evaluation ${evaluation.id.slice(0, 8)}` + const runBy = evaluation.created_by_email?.trim() || '—' + const lastUpdatedBy = evaluation.last_updated_by_email?.trim() || '—' return ( -
+ navigate( + `/call-imports/${id}/evaluations/${evaluation.id}`, + ) + } + onKeyDown={(e) => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault() + navigate( + `/call-imports/${id}/evaluations/${evaluation.id}`, + ) + } + }} + className={`cursor-pointer group hover:bg-gray-50 ${ + isSelected ? 'bg-primary-50/40' : '' }`} > - { - setSelectedEvalIds((prev) => { - const next = new Set(prev) - if (e.target.checked) next.add(evaluation.id) - else next.delete(evaluation.id) - return next - }) - }} - /> - e.stopPropagation()} > + { + setSelectedEvalIds((prev) => { + const next = new Set(prev) + if (e.target.checked) next.add(evaluation.id) + else next.delete(evaluation.id) + return next + }) + }} + onClick={(e) => e.stopPropagation()} + /> + +
+ + + + + + ) })} + +
+ {(() => { + const items = evaluationsData?.items ?? [] + const allSelected = + items.length > 0 && + items.every((row) => selectedEvalIds.has(row.id)) + return ( + { + if (e.target.checked) { + setSelectedEvalIds(new Set(items.map((row) => row.id))) + } else { + setSelectedEvalIds(new Set()) + } + }} + /> + ) + })()} + + Evaluation + + Run by + + Last updated by + + Progress + + Status + +
-

+

{headerLabel} -

-

- Created {new Date(evaluation.created_at).toLocaleString()} -

+
+
+ {evaluation.id.slice(0, 8)} +
-
- - {evaluation.bulk_operation && ( -

- - {evaluationBulkOperationLabel(evaluation.bulk_operation)} -

- )} -

- {evaluation.completed_rows}/{evaluation.total_rows} rows +

+ {runBy} + + {lastUpdatedBy} + + {evaluation.completed_rows}/{evaluation.total_rows} rows + + + {evaluation.bulk_operation && ( +

+ + {evaluationBulkOperationLabel(evaluation.bulk_operation)}

- - - {(evaluation.status === 'pending' || - evaluation.status === 'running') && ( - - )} - + )} + 		e.stopPropagation()} + > + {(evaluation.status === 'pending' || + evaluation.status === 'running') && ( + + )} +
)}
diff --git a/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx b/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx index 542fdd6c..9cfbaa22 100644 --- a/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx +++ b/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx @@ -92,6 +92,7 @@ import ProviderModelPicker, { } from '../../components/providers/ProviderModelPicker' import { getActiveWorkspaceId, useWorkspaceStore } from '../../store/workspaceStore' import StatusBadge from '../../components/shared/StatusBadge' +import { EvaluationAuditMeta } from '../../components/callImports/AuditMetaChips' import DiariseStatusPill from '../../components/callImports/DiariseStatusPill' import CallImportProgressBar from './components/CallImportProgressBar' import MetricPromptImprovementsPanel from './components/MetricPromptImprovementsPanel' @@ -434,6 +435,8 @@ export default function CallImportEvaluationDetail() { const [forceFailPendingOpen, setForceFailPendingOpen] = useState(false) const [downloadMenuOpen, setDownloadMenuOpen] = useState(false) const downloadMenuRef = useRef(null) + const [pdfReportMenuOpen, setPdfReportMenuOpen] = useState(false) + const pdfReportMenuRef = useRef(null) const [pdfReportOpen, setPdfReportOpen] = useState(false) const [pdfWizardStep, setPdfWizardStep] = useState(1) const [pdfUserInsightsTriggering, setPdfUserInsightsTriggering] = @@ -889,6 +892,12 @@ export default function CallImportEvaluationDetail() { }, }) + const pdfReportsQuery = useQuery({ + queryKey: ['call-import-evaluation-pdf-reports', activeWorkspaceId, id, evalId], + queryFn: () => apiClient.listCallImportEvaluationPdfReports(id!, evalId!), + enabled: pdfReportMenuOpen && !!id && !!evalId, + }) + useEffect(() => { if (!deepLinkConversationId && !deepLinkRowId) return if (deepLinkConversationId && searchQuery !== deepLinkConversationId) { @@ -1816,6 +1825,33 @@ export default function CallImportEvaluationDetail() { ) } + const openStoredPdfPreview = async (reportId: string) => { + if (!id || !evalId) return + try { + const report = await apiClient.getCallImportEvaluationPdfReport( + id, + evalId, + reportId, + ) + if (!report.preview_url) { + throw new Error('Preview URL is not available.') + } + if (pdfPreviewUrl?.startsWith('blob:')) { + window.URL.revokeObjectURL(pdfPreviewUrl) + } + setPdfPreviewUrl(report.preview_url) + setPdfPreviewFilename(report.filename || 'report.pdf') + setPdfPreviewOpen(true) + setPdfReportMenuOpen(false) + } catch (e: unknown) { + console.error('Failed to open stored PDF report', e) + showToast( + getApiErrorMessage(e, 'Failed to open PDF preview.'), + 'error', + ) + } + } + const handlePdfReportSubmit = async () => { if (!id || !evalId || pdfReportLoading) return const vendorName = pdfVendorName.trim() @@ -1826,20 +1862,45 @@ export default function CallImportEvaluationDetail() { setPdfReportLoadingAction('download') setPdfReportError(null) try { - const blob = await generatePdfReportBlob(vendorName) + const result = await generatePdfReportBlob(vendorName) const vendorSlug = vendorName .toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-+|-+$/g, '') || 'client' - const url = window.URL.createObjectURL(blob) - const link = document.createElement('a') - link.href = url - link.download = `${vendorSlug}-${pdfReportType}-quality-metric-audit-${evalId}.pdf` - document.body.appendChild(link) - link.click() - link.remove() - window.URL.revokeObjectURL(url) + const defaultFilename = `${vendorSlug}-${pdfReportType}-quality-metric-audit-${evalId}.pdf` + + if (result instanceof Blob) { + const url = window.URL.createObjectURL(result) + const link = document.createElement('a') + link.href = url + link.download = defaultFilename + document.body.appendChild(link) + link.click() + link.remove() + window.URL.revokeObjectURL(url) + } else { + const downloadUrl = result.download_url + if (!downloadUrl) { + throw new Error('Download URL is not available.') + } + const link = document.createElement('a') + link.href = downloadUrl + link.download = result.filename || defaultFilename + link.target = '_blank' + link.rel = 'noopener noreferrer' + document.body.appendChild(link) + link.click() + link.remove() + await queryClient.invalidateQueries({ + queryKey: [ + 'call-import-evaluation-pdf-reports', + activeWorkspaceId, + id, + evalId, + ], + }) + } setPdfReportOpen(false) setPdfWizardStep(1) setPdfVendorName('') @@ -1865,18 +1926,43 @@ export default function CallImportEvaluationDetail() { setPdfReportLoadingAction('preview') setPdfReportError(null) try { - const blob = await generatePdfReportBlob(vendorName) + const result = await generatePdfReportBlob(vendorName) const vendorSlug = vendorName .toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-+|-+$/g, '') || 'client' - if (pdfPreviewUrl) window.URL.revokeObjectURL(pdfPreviewUrl) - const url = window.URL.createObjectURL(blob) - setPdfPreviewUrl(url) - setPdfPreviewFilename( - `${vendorSlug}-${pdfReportType}-quality-metric-audit-${evalId}.pdf`, - ) + + if (result instanceof Blob) { + if (pdfPreviewUrl?.startsWith('blob:')) { + window.URL.revokeObjectURL(pdfPreviewUrl) + } + const url = window.URL.createObjectURL(result) + setPdfPreviewUrl(url) + setPdfPreviewFilename( + `${vendorSlug}-${pdfReportType}-quality-metric-audit-${evalId}.pdf`, + ) + } else { + if (!result.preview_url) { + throw new Error('Preview URL is not available.') + } + if (pdfPreviewUrl?.startsWith('blob:')) { + window.URL.revokeObjectURL(pdfPreviewUrl) + } + setPdfPreviewUrl(result.preview_url) + setPdfPreviewFilename( + result.filename || + `${vendorSlug}-${pdfReportType}-quality-metric-audit-${evalId}.pdf`, + ) + await queryClient.invalidateQueries({ + queryKey: [ + 'call-import-evaluation-pdf-reports', + activeWorkspaceId, + id, + evalId, + ], + }) + } setPdfPreviewOpen(true) } catch (e: unknown) { console.error('Failed to preview PDF report', e) @@ -2164,6 +2250,20 @@ export default function CallImportEvaluationDetail() { return () => document.removeEventListener('mousedown', handleClickOutside) }, [downloadMenuOpen]) + useEffect(() => { + if (!pdfReportMenuOpen) return + const handleClickOutside = (event: MouseEvent) => { + if ( + pdfReportMenuRef.current && + !pdfReportMenuRef.current.contains(event.target as Node) + ) { + setPdfReportMenuOpen(false) + } + } + document.addEventListener('mousedown', handleClickOutside) + return () => document.removeEventListener('mousedown', handleClickOutside) + }, [pdfReportMenuOpen]) + if (!id || !evalId) { return
Missing identifiers.
} @@ -2346,20 +2446,90 @@ export default function CallImportEvaluationDetail() { Re-run metrics )} - +
+ + {pdfReportMenuOpen && ( +
+
+ +
+
+ History +
+
+ {pdfReportsQuery.isLoading ? ( +

Loading…

+ ) : (pdfReportsQuery.data?.items?.length ?? 0) === 0 ? ( +

+ No stored reports yet. +

+ ) : ( +
    + {pdfReportsQuery.data!.items.map((item) => ( +
  • + +
    • + ))} +
+ )} +
+
+ )} +
-
+
- - Created: {formatDateTime(evaluation.created_at)} - - {evaluation.started_at && ( - - Started: {formatDateTime(evaluation.started_at)} - - )} - {evaluation.finished_at && ( - - Finished: {formatDateTime(evaluation.finished_at)} - - )} +
diff --git a/frontend/src/pages/callImports/CallImports.tsx b/frontend/src/pages/callImports/CallImports.tsx index c9ccaf26..9dbc680b 100644 --- a/frontend/src/pages/callImports/CallImports.tsx +++ b/frontend/src/pages/callImports/CallImports.tsx @@ -1,560 +1,554 @@ -import { useMemo, useState } from 'react' -import { Link, useNavigate } from 'react-router-dom' -import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { - ChevronLeft, - ChevronRight, - FileAudio, - FileSpreadsheet, - Layers, - Phone, - RefreshCw, - Trash2, - Upload, -} from 'lucide-react' -import { Tag as TagIcon } from 'lucide-react' -import { apiClient } from '../../lib/api' -import { getApiErrorMessage } from '../../lib/apiErrors' -import { useToast } from '../../hooks/useToast' -import { useWorkspaceStore } from '../../store/workspaceStore' -import type { CallImport, CallImportStatus, CallImportTag } from '../../types/api' -import Button from '../../components/Button' -import ConfirmModal from '../../components/ConfirmModal' -import StatusBadge from '../../components/shared/StatusBadge' -import CallImportProgressBar from './components/CallImportProgressBar' -import UploadAudioModal from './components/UploadAudioModal' -import UploadCsvModal from './components/UploadCsvModal' - -const PAGE_SIZE = 20 - -const STATUS_OPTIONS: Array<{ label: string; value: '' | CallImportStatus }> = [ - { label: 'All statuses', value: '' }, - { label: 'Uploaded', value: 'uploaded' }, - { label: 'Mapped', value: 'mapped' }, - { label: 'Pending', value: 'pending' }, - { label: 'Processing', value: 'processing' }, - { label: 'Completed', value: 'completed' }, - { label: 'Partial', value: 'partial' }, - { label: 'Failed', value: 'failed' }, - { label: 'Deleting', value: 'deleting' }, -] - -type UploadTab = 'datasets' | 'audio' - -export default function CallImports() { - const navigate = useNavigate() - const queryClient = useQueryClient() - const { showToast, ToastContainer } = useToast() - // Active workspace is part of every workspace-scoped queryKey so a - // workspace switch produces a clean cache miss instead of leaking - // rows from the previously-active workspace. - const activeWorkspaceId = useWorkspaceStore((s) => s.activeWorkspaceId) - const [page, setPage] = useState(1) - const [statusFilter, setStatusFilter] = useState<'' | CallImportStatus>('') - const [datasetFilter, setDatasetFilter] = useState('') - const [tagFilter, setTagFilter] = useState([]) - const [activeTab, setActiveTab] = useState('datasets') - const [showUpload, setShowUpload] = useState(false) - const [showAudioUpload, setShowAudioUpload] = useState(false) - const [pendingDelete, setPendingDelete] = useState(null) - const [deleteError, setDeleteError] = useState(null) - - const { data: datasets = [] } = useQuery({ - queryKey: ['call-import-datasets', activeWorkspaceId], - queryFn: () => apiClient.listCallImportDatasets(), - }) - - const { data: allTags = [] } = useQuery({ - queryKey: ['call-import-tags', activeWorkspaceId], - queryFn: () => apiClient.listCallImportTags(), - }) - - const deleteMutation = useMutation({ - mutationFn: (id: string) => apiClient.deleteCallImport(id), - onSuccess: (result) => { - queryClient.invalidateQueries({ queryKey: ['call-imports'] }) - setPendingDelete(null) - setDeleteError(null) - if (result.status === 'accepted') { - showToast( - 'Deletion started — large imports may take a minute.', - 'success', - ) - } - }, - onError: (err: unknown) => { - const message = getApiErrorMessage(err, 'Failed to delete import.') - setDeleteError(message) - showToast(message, 'error') - }, - }) - - const queryParams = useMemo( - () => ({ - page, - page_size: PAGE_SIZE, - ...(statusFilter ? { status: statusFilter } : {}), - ...(datasetFilter ? { dataset: datasetFilter } : {}), - ...(tagFilter.length > 0 ? { tag_id: tagFilter } : {}), - source_format: activeTab === 'audio' ? 'audio' : '__non_audio__', - }), - [page, statusFilter, datasetFilter, tagFilter, activeTab], - ) - - const { data, isLoading, isFetching, refetch } = useQuery({ - queryKey: ['call-imports', activeWorkspaceId, queryParams], - queryFn: () => apiClient.listCallImports(queryParams), - refetchInterval: (query) => { - const items = query.state.data?.items ?? [] - const hasActive = items.some( - (i: CallImport) => i.status === 'pending' || i.status === 'processing', - ) - const hasDeleting = items.some( - (i: CallImport) => i.status === 'deleting', - ) - if (hasDeleting) return 3000 - return hasActive ? 5000 : false - }, - }) - - const items = data?.items ?? [] - const total = data?.total ?? 0 - const totalPages = Math.max(1, Math.ceil(total / PAGE_SIZE)) - - return ( -
- -
-
-

Call Imports

-

- Upload datasets from CSV / Excel, or add manual call recordings - directly and use the same diarisation and evaluation tools. -

-
-
- - - - - - - - -
-
- -
- - -
- - {/* - High-level dataset segregation lives at the top of the page so users can - scope all filtering/searching that follows to a specific dataset. We - intentionally render this above the main card to make it visually - distinct from the in-card status/tag filters. - */} -
- - - {datasetFilter && ( - - Showing imports tagged with dataset “{datasetFilter}”. - - )} -
- -
-
-
-
- - -
- {allTags.length > 0 && ( -
- Tags: - {allTags.map((tag: CallImportTag) => { - const active = tagFilter.includes(tag.id) - return ( - - ) - })} - {tagFilter.length > 0 && ( - - )} -
- )} -
-

- {total} {activeTab === 'audio' ? 'manual upload' : 'dataset import'} - {total === 1 ? '' : 's'} -

-
- - {isLoading ? ( -
- -

Loading imports...

-
- ) : items.length === 0 ? ( -
- -

- {statusFilter - ? 'No imports match this filter.' - : activeTab === 'audio' - ? 'No manual audio uploads yet.' - : 'No dataset uploads yet.'} -

- {!statusFilter && ( - - )} -
- ) : ( -
- - - - - - - - - - - - - - {items.map((item: CallImport) => { - const isDeleting = item.status === 'deleting' - return ( - { - if (isDeleting) return - navigate(`/call-imports/${item.id}`) - }} - > - - - - - - - - - ) - })} - -
- Filename - - Provider - - Dataset / Tags - - Progress - - Status - - Created - - Actions -
-
- {item.original_filename || '(unnamed)'} -
-
- {item.id.slice(0, 8)} -
- 		- {item.source_format === 'audio' ? ( - - - Manual upload - - ) : item.provider || ( - - — - - )} - -
- {item.dataset ? ( - - {item.dataset} - - ) : ( - - no dataset - - )} - {item.tags.length > 0 && ( -
- {item.tags.map((tag) => ( - - {tag.name} - - ))} -
- )} -
- 		- - - - - {new Date(item.created_at).toLocaleString()} - e.stopPropagation()} - > -
- - View - - -
-
- - {totalPages > 1 && ( -
-

- Page {page} of {totalPages} -

-
- - -
-
- )} -
- )} -
- - setShowUpload(false)} /> - setShowAudioUpload(false)} - /> - - { - if (!pendingDelete) return '' - const name = pendingDelete.original_filename || '(unnamed)' - const total = pendingDelete.total_rows - const completed = pendingDelete.completed_rows - const inFlight = - pendingDelete.status === 'pending' || - pendingDelete.status === 'processing' || - pendingDelete.status === 'deleting' - const lines = [ - `“${name}” will be permanently deleted, along with all ${total} row record${total === 1 ? '' : 's'} and ${completed} stored recording${completed === 1 ? '' : 's'} in S3.`, - inFlight - ? 'This batch is still processing — pending tasks will be revoked before deletion.' - : '', - 'This cannot be undone.', - deleteError ? `Error: ${deleteError}` : '', - ] - return lines.filter(Boolean).join('\n\n') - })()} - confirmLabel="Delete" - cancelLabel="Cancel" - variant="danger" - isLoading={deleteMutation.isPending} - onConfirm={() => { - if (pendingDelete) deleteMutation.mutate(pendingDelete.id) - }} - onCancel={() => { - if (deleteMutation.isPending) return - setPendingDelete(null) - setDeleteError(null) - }} - /> -
- ) -} +import { useMemo, useState } from 'react' +import { Link, useNavigate } from 'react-router-dom' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { + ChevronLeft, + ChevronRight, + FileAudio, + FileSpreadsheet, + Layers, + Phone, + RefreshCw, + Trash2, + Upload, +} from 'lucide-react' +import { Tag as TagIcon } from 'lucide-react' +import { apiClient } from '../../lib/api' +import { getApiErrorMessage } from '../../lib/apiErrors' +import { useToast } from '../../hooks/useToast' +import { useWorkspaceStore } from '../../store/workspaceStore' +import type { CallImport, CallImportStatus, CallImportTag } from '../../types/api' +import Button from '../../components/Button' +import ConfirmModal from '../../components/ConfirmModal' +import StatusBadge from '../../components/shared/StatusBadge' +import CallImportProgressBar from './components/CallImportProgressBar' +import UploadAudioModal from './components/UploadAudioModal' +import UploadCsvModal from './components/UploadCsvModal' + +const PAGE_SIZE = 20 + +const STATUS_OPTIONS: Array<{ label: string; value: '' | CallImportStatus }> = [ + { label: 'All statuses', value: '' }, + { label: 'Uploaded', value: 'uploaded' }, + { label: 'Mapped', value: 'mapped' }, + { label: 'Pending', value: 'pending' }, + { label: 'Processing', value: 'processing' }, + { label: 'Completed', value: 'completed' }, + { label: 'Partial', value: 'partial' }, + { label: 'Failed', value: 'failed' }, + { label: 'Deleting', value: 'deleting' }, +] + +type UploadTab = 'datasets' | 'audio' + +export default function CallImports() { + const navigate = useNavigate() + const queryClient = useQueryClient() + const { showToast, ToastContainer } = useToast() + // Active workspace is part of every workspace-scoped queryKey so a + // workspace switch produces a clean cache miss instead of leaking + // rows from the previously-active workspace. + const activeWorkspaceId = useWorkspaceStore((s) => s.activeWorkspaceId) + const [page, setPage] = useState(1) + const [statusFilter, setStatusFilter] = useState<'' | CallImportStatus>('') + const [datasetFilter, setDatasetFilter] = useState('') + const [tagFilter, setTagFilter] = useState([]) + const [activeTab, setActiveTab] = useState('datasets') + const [showUpload, setShowUpload] = useState(false) + const [showAudioUpload, setShowAudioUpload] = useState(false) + const [pendingDelete, setPendingDelete] = useState(null) + const [deleteError, setDeleteError] = useState(null) + + const { data: datasets = [] } = useQuery({ + queryKey: ['call-import-datasets', activeWorkspaceId], + queryFn: () => apiClient.listCallImportDatasets(), + }) + + const { data: allTags = [] } = useQuery({ + queryKey: ['call-import-tags', activeWorkspaceId], + queryFn: () => apiClient.listCallImportTags(), + }) + + const deleteMutation = useMutation({ + mutationFn: (id: string) => apiClient.deleteCallImport(id), + onSuccess: (result) => { + queryClient.invalidateQueries({ queryKey: ['call-imports'] }) + setPendingDelete(null) + setDeleteError(null) + if (result.status === 'accepted') { + showToast( + 'Deletion started — large imports may take a minute.', + 'success', + ) + } + }, + onError: (err: unknown) => { + const message = getApiErrorMessage(err, 'Failed to delete import.') + setDeleteError(message) + showToast(message, 'error') + }, + }) + + const queryParams = useMemo( + () => ({ + page, + page_size: PAGE_SIZE, + ...(statusFilter ? { status: statusFilter } : {}), + ...(datasetFilter ? { dataset: datasetFilter } : {}), + ...(tagFilter.length > 0 ? { tag_id: tagFilter } : {}), + source_format: activeTab === 'audio' ? 'audio' : '__non_audio__', + }), + [page, statusFilter, datasetFilter, tagFilter, activeTab], + ) + + const { data, isLoading, isFetching, refetch } = useQuery({ + queryKey: ['call-imports', activeWorkspaceId, queryParams], + queryFn: () => apiClient.listCallImports(queryParams), + refetchInterval: (query) => { + const items = query.state.data?.items ?? [] + const hasActive = items.some( + (i: CallImport) => i.status === 'pending' || i.status === 'processing', + ) + const hasDeleting = items.some( + (i: CallImport) => i.status === 'deleting', + ) + if (hasDeleting) return 3000 + return hasActive ? 5000 : false + }, + }) + + const items = data?.items ?? [] + const total = data?.total ?? 0 + const totalPages = Math.max(1, Math.ceil(total / PAGE_SIZE)) + + return ( +
+ +
+
+

Call Imports

+

+ Upload datasets from CSV / Excel, or add manual call recordings + directly and use the same diarisation and evaluation tools. +

+
+
+ + + + + + + + +
+
+ +
+ + +
+ + {/* + High-level dataset segregation lives at the top of the page so users can + scope all filtering/searching that follows to a specific dataset. We + intentionally render this above the main card to make it visually + distinct from the in-card status/tag filters. + */} +
+ + + {datasetFilter && ( + + Showing imports tagged with dataset “{datasetFilter}”. + + )} +
+ +
+
+
+
+ + +
+ {allTags.length > 0 && ( +
+ Tags: + {allTags.map((tag: CallImportTag) => { + const active = tagFilter.includes(tag.id) + return ( + + ) + })} + {tagFilter.length > 0 && ( + + )} +
+ )} +
+

+ {total} {activeTab === 'audio' ? 'manual upload' : 'dataset import'} + {total === 1 ? '' : 's'} +

+
+ + {isLoading ? ( +
+ +

Loading imports...

+
+ ) : items.length === 0 ? ( +
+ +

+ {statusFilter + ? 'No imports match this filter.' + : activeTab === 'audio' + ? 'No manual audio uploads yet.' + : 'No dataset uploads yet.'} +

+ {!statusFilter && ( + + )} +
+ ) : ( +
+ + + + + + + + + + + + + {items.map((item: CallImport) => { + const isDeleting = item.status === 'deleting' + return ( + { + if (isDeleting) return + navigate(`/call-imports/${item.id}`) + }} + > + + + + + + + + ) + })} + +
+ Filename + + Provider + + Dataset / Tags + + Progress + + Status + + Actions +
+
+ {item.original_filename || '(unnamed)'} +
+
+ {item.id.slice(0, 8)} +
+ 		+ {item.source_format === 'audio' ? ( + + + Manual upload + + ) : item.provider || ( + + — + + )} + +
+ {item.dataset ? ( + + {item.dataset} + + ) : ( + + no dataset + + )} + {item.tags.length > 0 && ( +
+ {item.tags.map((tag) => ( + + {tag.name} + + ))} +
+ )} +
+ 		+ + + + e.stopPropagation()} + > +
+ + View + + +
+
+ + {totalPages > 1 && ( +
+

+ Page {page} of {totalPages} +

+
+ + +
+
+ )} +
+ )} +
+ + setShowUpload(false)} /> + setShowAudioUpload(false)} + /> + + { + if (!pendingDelete) return '' + const name = pendingDelete.original_filename || '(unnamed)' + const total = pendingDelete.total_rows + const completed = pendingDelete.completed_rows + const inFlight = + pendingDelete.status === 'pending' || + pendingDelete.status === 'processing' || + pendingDelete.status === 'deleting' + const lines = [ + `“${name}” will be permanently deleted, along with all ${total} row record${total === 1 ? '' : 's'} and ${completed} stored recording${completed === 1 ? '' : 's'} in S3.`, + inFlight + ? 'This batch is still processing — pending tasks will be revoked before deletion.' + : '', + 'This cannot be undone.', + deleteError ? `Error: ${deleteError}` : '', + ] + return lines.filter(Boolean).join('\n\n') + })()} + confirmLabel="Delete" + cancelLabel="Cancel" + variant="danger" + isLoading={deleteMutation.isPending} + onConfirm={() => { + if (pendingDelete) deleteMutation.mutate(pendingDelete.id) + }} + onCancel={() => { + if (deleteMutation.isPending) return + setPendingDelete(null) + setDeleteError(null) + }} + /> +
+ ) +} diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index 91fbc9c5..eb4a9baa 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -1,2132 +1,2165 @@ -// API Types matching the backend schemas - -export type { LLMGenerationConfig } from '../config/llmGenerationParams' -import type { LLMGenerationConfig } from '../config/llmGenerationParams' - -export enum EvaluationType { - ASR = 'asr', - TTS = 'tts', -} - -export enum EvaluationStatus { - PENDING = 'pending', - PROCESSING = 'processing', - COMPLETED = 'completed', - FAILED = 'failed', - CANCELLED = 'cancelled', -} - -export interface AudioFile { - id: string - filename: string - format: string - file_size: number - duration?: number | null - sample_rate?: number | null - channels?: number | null - uploaded_at: string -} - -export interface Evaluation { - id: string - audio_id: string - reference_text?: string | null - evaluation_type: EvaluationType - model_name?: string | null - status: EvaluationStatus - metrics_requested?: string[] | null - created_at: string - started_at?: string | null - completed_at?: string | null - error_message?: string | null -} - -export interface DashboardSummary { - evaluations: { - total: number - completed: number - pending: number - failed: number - } - resources: { - agents: number - personas: number - scenarios: number - integrations: number - voice_bundles: number - } - setup_progress: { - has_integration: boolean - has_voice_bundle: boolean - has_agent: boolean - has_evaluation: boolean - } - metrics: { - total: number - enabled: number - } - call_imports: { - total: number - } - call_import_evaluations: { - total: number - completed: number - running: number - failed: number - } - recent_evaluations: Evaluation[] -} - -export interface ModelConfigEntry { - provider: string - model_type: string - description?: string - featured?: boolean - featured_rank?: number - highlights?: string[] -} - -export interface EvaluationCreate { - audio_id: string - reference_text?: string | null - evaluation_type: EvaluationType - model_name?: string | null - metrics?: string[] -} - -export interface EvaluationResult { - evaluation_id: string - status: EvaluationStatus - transcript?: string | null - metrics: Record - processing_time?: number | null - model_used?: string | null - created_at: string -} - -export interface BatchEvaluationResult { - processed_files: number - failed_files: number - aggregated_metrics?: Record | null - individual_results: EvaluationResult[] -} - -/** Voice agent evaluator run (evaluator_results table). */ -export type EvaluatorResultStatus = - | 'queued' - | 'call_initiating' - | 'call_connecting' - | 'call_in_progress' - | 'call_ended' - | 'transcribing' - | 'evaluating' - | 'fetching_details' - | 'completed' - | 'failed' - -export interface EvaluatorResultMetricScore { - value: unknown - type: string - metric_name: string - parent_metric_id?: string | null -} - -export interface EvaluatorResultRow { - id: string - result_id: string - name: string | null - evaluator_id: string | null - agent_id?: string | null - persona_id?: string | null - scenario_id?: string | null - suite_id?: string | null - timestamp: string - duration_seconds: number | null - status: EvaluatorResultStatus - metric_scores: Record | null - error_message: string | null - agent?: { id: string; name: string } | null - scenario?: { id: string; name: string } | null -} - -export interface EvaluatorResultListResponse { - items: EvaluatorResultRow[] - total: number -} - -export interface EvaluatorResultCounts { - total: number - completed: number - failed: number - in_progress: number - last_run_at?: string | null -} - -export interface EvaluatorResultsScenarioSummary { - scenario_id: string - scenario_name: string - counts: EvaluatorResultCounts -} - -export interface EvaluatorResultsSuiteSummary { - suite_id: string - suite_name?: string | null - agent_id: string - persona_id?: string | null - counts: EvaluatorResultCounts - scenarios?: EvaluatorResultsScenarioSummary[] | null -} - -export interface EvaluatorResultsAgentSummary { - agent_id: string - agent_name: string - counts: EvaluatorResultCounts - suites?: EvaluatorResultsSuiteSummary[] | null -} - -export interface EvaluatorResultsOverviewResponse { - workspace_counts: EvaluatorResultCounts - agents: EvaluatorResultsAgentSummary[] - unassigned: { - counts: EvaluatorResultCounts - recent_result_ids: string[] - } -} - -export interface ListEvaluatorResultsParams { - skip?: number - limit?: number - evaluatorId?: string - agentId?: string - suiteId?: string - scenarioId?: string - status?: 'completed' | 'failed' | 'in_progress' - unassignedOnly?: boolean - playground?: boolean - testAgentsOnly?: boolean -} - -export interface APIKey { - id: string - key: string - name?: string | null - is_active: boolean - created_at: string - last_used?: string | null - message?: string -} - -export interface MessageResponse { - message: string -} - -// IAM & User Types -export enum Role { - READER = 'reader', - WRITER = 'writer', - ADMIN = 'admin', -} - -export enum InvitationStatus { - PENDING = 'pending', - ACCEPTED = 'accepted', - DECLINED = 'declined', - EXPIRED = 'expired', -} - -export interface User { - id: string - email: string - name?: string | null - is_active: boolean - created_at: string -} - -export interface OrganizationMember { - id: string - user_id: string - organization_id: string - role: Role - joined_at: string - user: User -} - -export interface Invitation { - id: string - organization_id: string - email: string - role: Role - status: InvitationStatus - expires_at: string - created_at: string - organization_name?: string | null -} - -export interface InvitationCreate { - email: string - role: Role -} - -export interface RoleUpdate { - role: Role -} - -export interface Profile { - id: string - email: string - name?: string | null - first_name?: string | null - last_name?: string | null - created_at: string - organizations: Array<{ - id: string - name: string - role: string - joined_at: string - }> -} - -export interface UserUpdate { - name?: string | null - first_name?: string | null - last_name?: string | null - email?: string | null -} - -export interface UserPreferences { - theme?: string - notifications_enabled?: boolean - email_notifications?: boolean - default_language?: string - [key: string]: any -} - -export interface UserPreferencesUpdate { - theme?: string - notifications_enabled?: boolean - email_notifications?: boolean - default_language?: string - [key: string]: any -} - -// Integration Types -export enum IntegrationPlatform { - RETELL = 'retell', - VAPI = 'vapi', - CARTESIA = 'cartesia', - ELEVENLABS = 'elevenlabs', - DEEPGRAM = 'deepgram', - MURF = 'murf', - SARVAM = 'sarvam', - VOICEMAKER = 'voicemaker', - SMALLEST = 'smallest', -} - -export enum TelephonyProvider { - PLIVO = 'plivo', - EXOTEL = 'exotel', - VOBIZ = 'vobiz', -} - -export type CredentialRoutingMode = 'inherit' | 'gateway' | 'direct' -export type GatewayInterfaceMode = 'inherit' | 'litellm_shim' | 'native_openai' - -export type EffectiveCredentialRouting = - | 'inherit' - | 'direct' - | 'gateway' - | 'bifrost' - | 'litellm_proxy' - -export interface Integration { - id: string - organization_id: string - platform: IntegrationPlatform - name?: string | null - public_key?: string | null - is_active: boolean - /** True if this row is the default credential for (org, platform). */ - is_default?: boolean - routing_mode?: CredentialRoutingMode - effective_routing?: EffectiveCredentialRouting - created_at: string - updated_at: string - last_tested_at?: string | null -} - -export interface IntegrationCreate { - platform: IntegrationPlatform - api_key: string - public_key?: string - name?: string | null - routing_mode?: CredentialRoutingMode - /** Mark the new credential as the default for (org, platform). */ - is_default?: boolean -} - -// VoiceBundle Types -export enum ModelProvider { - OPENAI = 'openai', - ANTHROPIC = 'anthropic', - GOOGLE = 'google', - XAI = 'xai', - FIREWORKS = 'fireworks', - COHERE = 'cohere', - MISTRAL = 'mistral', - META = 'meta', - TOGETHER = 'together', - PERPLEXITY = 'perplexity', - AZURE = 'azure', - AWS = 'aws', - DEEPGRAM = 'deepgram', - CARTESIA = 'cartesia', - ELEVENLABS = 'elevenlabs', - MURF = 'murf', - CUSTOM = 'custom', - SARVAM = 'sarvam', - VOICEMAKER = 'voicemaker', - SMALLEST = 'smallest', -} - -// AI Provider Types -export interface AIProvider { - id: string - provider: ModelProvider - api_key?: string | null - name?: string | null - endpoint_url?: string | null - is_active: boolean - /** True if this row is the default credential for (org, provider). */ - is_default?: boolean - routing_mode?: CredentialRoutingMode - gateway_model?: string | null - gateway_interface?: GatewayInterfaceMode - gateway_base_url?: string | null - gateway_auth_header?: string | null - gateway_auth_secret_env?: string | null - has_gateway_auth_secret?: boolean - gateway_extra_headers?: Record | null - /** True when provider secrets are resolved by the Bifrost gateway. */ - gateway_managed?: boolean - effective_routing?: EffectiveCredentialRouting - effective_gateway_interface?: 'litellm_shim' | 'native_openai' - created_at: string - updated_at: string - last_tested_at?: string | null -} - -export interface AIProviderCreate { - provider: ModelProvider - api_key?: string | null - name?: string | null - endpoint_url?: string | null - routing_mode?: CredentialRoutingMode - gateway_model?: string | null - gateway_interface?: GatewayInterfaceMode - gateway_base_url?: string | null - gateway_auth_header?: string | null - gateway_auth_secret_env?: string | null - gateway_auth_secret?: string | null - gateway_extra_headers?: Record | null - /** Mark the new credential as the default for (org, provider). */ - is_default?: boolean -} - -export interface AIProviderUpdate { - api_key?: string | null - name?: string | null - endpoint_url?: string | null - is_active?: boolean - routing_mode?: CredentialRoutingMode - gateway_model?: string | null - gateway_interface?: GatewayInterfaceMode - gateway_base_url?: string | null - gateway_auth_header?: string | null - gateway_auth_secret_env?: string | null - gateway_auth_secret?: string | null - clear_gateway_auth_secret?: boolean - gateway_extra_headers?: Record | null -} - -export enum VoiceBundleType { - STT_LLM_TTS = 'stt_llm_tts', - S2S = 's2s', -} - -export interface VoiceBundle { - id: string - name: string - description?: string | null - bundle_type: VoiceBundleType - stt_provider?: ModelProvider | null - stt_model?: string | null - /** - * Optional explicit AIProvider/Integration row id for STT. When null the - * runtime resolver picks the default credential for stt_provider. - */ - stt_credential_id?: string | null - llm_provider?: ModelProvider | null - llm_model?: string | null - llm_temperature?: number | null - llm_max_tokens?: number | null - llm_config?: Record | null - llm_credential_id?: string | null - tts_provider?: ModelProvider | null - tts_model?: string | null - tts_voice?: string | null - tts_config?: Record | null - tts_credential_id?: string | null - s2s_provider?: ModelProvider | null - s2s_model?: string | null - s2s_config?: Record | null - s2s_credential_id?: string | null - extra_metadata?: Record | null - is_active: boolean - created_at: string - updated_at: string - created_by?: string | null -} - -export interface VoiceBundleCreate { - name: string - description?: string | null - bundle_type?: VoiceBundleType - stt_provider?: ModelProvider | null - stt_model?: string | null - stt_credential_id?: string | null - llm_provider?: ModelProvider | null - llm_model?: string | null - llm_temperature?: number | null - llm_max_tokens?: number | null - llm_config?: Record | null - llm_credential_id?: string | null - tts_provider?: ModelProvider | null - tts_model?: string | null - tts_voice?: string | null - tts_config?: Record | null - tts_credential_id?: string | null - s2s_provider?: ModelProvider | null - s2s_model?: string | null - s2s_config?: Record | null - s2s_credential_id?: string | null - extra_metadata?: Record | null -} - -// Test Agent Types -export interface AgentPhoneAssignmentConflict { - agent_id: string - agent_name: string - phone_number: string -} - -export interface AgentPhoneAssignmentCheckResponse { - available: boolean - phone_number?: string | null - conflict?: AgentPhoneAssignmentConflict | null -} - -export interface TestAgent { - id: string - agent_id?: string | null - name: string - phone_number?: string | null - telephony_phone_number_id?: string | null - language: string - description: string | null - prompt_variables?: Record | null - silence_hangup_secs?: number - call_type: string - call_medium: string - voice_bundle_id?: string | null - voice_ai_integration_id?: string | null - voice_ai_agent_id?: string | null - provider_prompt?: string | null - provider_prompt_synced_at?: string | null - created_at: string - updated_at: string -} - -// Test Agent Conversation Types -export interface TestAgentConversation { - id: string - organization_id: string - agent_id: string - persona_id: string - scenario_id: string - voice_bundle_id: string - status: string - live_transcription?: Array<{ - speaker: string - text: string - timestamp: number - audio_segment_key?: string - }> | null - conversation_audio_key?: string | null - full_transcript?: string | null - started_at: string - ended_at?: string | null - duration_seconds?: number | null - conversation_metadata?: Record | null - created_at: string - updated_at: string - created_by?: string | null -} - -export interface TestAgentConversationCreate { - agent_id: string - persona_id: string - scenario_id: string - voice_bundle_id: string - conversation_metadata?: Record | null -} - -export interface TestAgentConversationUpdate { - status?: string | null - live_transcription?: Array> | null - full_transcript?: string | null - conversation_metadata?: Record | null -} - -export interface VoiceBundleUpdate { - name?: string - description?: string | null - stt_provider?: ModelProvider - stt_model?: string - stt_credential_id?: string | null - llm_provider?: ModelProvider - llm_model?: string - llm_temperature?: number | null - llm_max_tokens?: number | null - llm_config?: Record | null - llm_credential_id?: string | null - tts_provider?: ModelProvider - tts_model?: string - tts_voice?: string | null - tts_config?: Record | null - tts_credential_id?: string | null - s2s_provider?: ModelProvider | null - s2s_model?: string | null - s2s_config?: Record | null - s2s_credential_id?: string | null - extra_metadata?: Record | null - is_active?: boolean -} - -// Data Sources Types -export interface S3ConnectionTest { - bucket_name: string - region?: string - access_key_id: string - secret_access_key: string - endpoint_url?: string | null -} - -export interface S3ConnectionTestResponse { - success: boolean - message: string - bucket_name?: string | null -} - -export interface S3FileInfo { - key: string - filename: string - size: number - last_modified: string -} - -export interface S3FolderInfo { - name: string - path: string -} - -export interface S3ListFilesResponse { - files: S3FileInfo[] - total: number - prefix?: string | null -} - -export interface S3BrowseResponse { - folders: S3FolderInfo[] - files: S3FileInfo[] - current_path: string - organization_id: string -} - -export interface S3Status { - enabled: boolean - provider?: 's3' | 'gcs' | string - error?: string | null -} - -// Alert Types -export enum AlertMetricType { - NUMBER_OF_CALLS = 'number_of_calls', - CALL_DURATION = 'call_duration', - ERROR_RATE = 'error_rate', - SUCCESS_RATE = 'success_rate', - LATENCY = 'latency', - CUSTOM = 'custom', -} - -export enum AlertAggregation { - SUM = 'sum', - AVG = 'avg', - COUNT = 'count', - MIN = 'min', - MAX = 'max', -} - -export enum AlertOperator { - GREATER_THAN = '>', - LESS_THAN = '<', - GREATER_THAN_OR_EQUAL = '>=', - LESS_THAN_OR_EQUAL = '<=', - EQUAL = '=', - NOT_EQUAL = '!=', -} - -export enum AlertNotifyFrequency { - IMMEDIATE = 'immediate', - HOURLY = 'hourly', - DAILY = 'daily', - WEEKLY = 'weekly', -} - -export enum AlertStatus { - ACTIVE = 'active', - PAUSED = 'paused', - DISABLED = 'disabled', -} - -export enum AlertHistoryStatus { - TRIGGERED = 'triggered', - NOTIFIED = 'notified', - ACKNOWLEDGED = 'acknowledged', - RESOLVED = 'resolved', -} - -export interface Alert { - id: string - organization_id: string - name: string - description?: string | null - metric_type: AlertMetricType - aggregation: AlertAggregation - operator: AlertOperator - threshold_value: number - time_window_minutes: number - agent_ids?: string[] | null - notify_frequency: AlertNotifyFrequency - notify_emails?: string[] | null - notify_webhooks?: string[] | null - status: AlertStatus - created_at: string - updated_at: string - created_by?: string | null -} - -export interface AlertCreate { - name: string - description?: string | null - metric_type?: AlertMetricType - aggregation?: AlertAggregation - operator?: AlertOperator - threshold_value: number - time_window_minutes?: number - agent_ids?: string[] | null - notify_frequency?: AlertNotifyFrequency - notify_emails?: string[] - notify_webhooks?: string[] -} - -export interface AlertUpdate { - name?: string - description?: string | null - metric_type?: AlertMetricType - aggregation?: AlertAggregation - operator?: AlertOperator - threshold_value?: number - time_window_minutes?: number - agent_ids?: string[] | null - notify_frequency?: AlertNotifyFrequency - notify_emails?: string[] - notify_webhooks?: string[] - status?: AlertStatus -} - -export interface AlertHistoryItem { - id: string - organization_id: string - alert_id: string - triggered_at: string - triggered_value: number - threshold_value: number - status: AlertHistoryStatus - notified_at?: string | null - notification_details?: Record | null - acknowledged_at?: string | null - acknowledged_by?: string | null - resolved_at?: string | null - resolved_by?: string | null - resolution_notes?: string | null - context_data?: Record | null - created_at: string - updated_at: string - alert?: Alert -} - - -// Cron Job Types -export enum CronJobStatus { - ACTIVE = 'active', - PAUSED = 'paused', - COMPLETED = 'completed', -} - -export interface CronJob { - id: string - organization_id: string - name: string - cron_expression: string - timezone: string - max_runs: number - current_runs: number - evaluator_ids: string[] - status: CronJobStatus - next_run_at?: string | null - last_run_at?: string | null - created_at: string - updated_at: string - created_by?: string | null -} - -export interface CronJobCreate { - name: string - cron_expression: string - timezone: string - max_runs: number - evaluator_ids: string[] -} - -export interface CronJobUpdate { - name?: string - cron_expression?: string - timezone?: string - max_runs?: number - evaluator_ids?: string[] - status?: CronJobStatus -} - -// --- Call Imports --- - -/** - * Lifecycle for a call-import batch. - * - * - ``uploaded`` : file landed in S3, no mapping yet. - * - ``mapped`` : user picked a schema + sheet + column mapping; no - * rows materialised yet, no worker enqueued. - * - ``processing`` : rows materialised + workers enqueued. - * - ``pending`` : transient state used by the legacy one-shot - * ``POST /upload`` endpoint just before transitioning - * to ``processing``. - */ -export type CallImportStatus = - | 'pending' - | 'uploaded' - | 'mapped' - | 'processing' - | 'completed' - | 'partial' - | 'failed' - | 'deleting' - -export type CallImportRowStatus = - | 'pending' - | 'processing' - | 'completed' - | 'failed' - -/** Where the value in `transcript` came from. */ -export type CallImportTranscriptSource = - | 'csv' - | 'transcribed' - | 'edited' - | null -/** Lifecycle status for the post-hoc transcription workflow itself. */ -export type CallImportTranscriptStatus = - | 'idle' - | 'pending' - | 'running' - | 'completed' - | 'failed' - | null - -/** - * Which transcript an evaluation run scored against. - * - `production`: the CSV-supplied value on `CallImportRow.transcript`. - * - `diarised`: the worker-produced value on `CallImportRow.diarised_transcript`. - */ -export type CallImportEvaluationTranscriptSource = 'production' | 'diarised' - -/** - * One contiguous turn inside ``CallImportRow.diarised_segments``. - * - * The diarisation worker rewrites each pyannote ``Speaker N`` label - * into ``agent`` / ``user`` (first speaker = agent heuristic). Anything - * beyond two distinct speakers keeps a generic ``speaker_N`` label so - * multi-party recordings still render every voice. - */ -export interface CallImportDiarisedSegment { - speaker: string - text: string - start: number - end: number - /** Original pyannote label (``Speaker 1`` / ``Speaker 2`` / ...). */ - raw_speaker: string -} - -export interface CallImportRow { - id: string - row_index: number - /** Mandatory identifier per row. Renamed from ``external_call_id``. */ - conversation_id: string - recording_url: string | null - recording_date: string | null - /** Production transcript — the value supplied via the CSV upload. */ - transcript: string | null - /** Provenance of the stored production transcript (csv = CSV upload, edited = manual edit). */ - transcript_source: CallImportTranscriptSource - /** Legacy: provider recorded by the original transcription worker before the split. */ - transcript_provider: string | null - transcript_model: string | null - transcript_status: CallImportTranscriptStatus - transcript_error: string | null - transcribed_at: string | null - /** Diarised transcript — produced by the post-hoc diarisation worker. */ - diarised_transcript: string | null - /** Provider used by the diarisation worker (e.g. "deepgram"). */ - diarised_transcript_provider: string | null - diarised_transcript_model: string | null - diarised_transcript_status: CallImportTranscriptStatus - diarised_transcript_error: string | null - diarised_at: string | null - /** - * Structured speaker turns produced by the diarisation worker. Each - * entry is a single contiguous turn shaped as - * `{ speaker: 'agent' | 'user' | 'speaker_N', text, start, end, - * raw_speaker }`. ``diarised_transcript`` is a rendered - * `: ` view of this list with - * ``diarised_speaker_swap`` applied. ``null`` on legacy rows that - * were diarised before structured turns were persisted (or when the - * STT provider didn't surface segments). - */ - diarised_segments: CallImportDiarisedSegment[] | null - /** - * When ``true`` the ``agent`` <-> ``user`` mapping inside - * ``diarised_segments`` is inverted in the rendered transcript / - * CSV export. The worker writes the canonical mapping using a - * "first speaker is the agent" heuristic; reviewers can flip the - * toggle from the row detail panel without re-running diarisation. - */ - diarised_speaker_swap: boolean - /** - * LLM that turned the STT plain-text output into structured - * ``diarised_segments``. NULL on legacy rows (pre-LLM-diariser). - */ - diarised_llm_provider: string | null - diarised_llm_model: string | null - /** - * Exact prompt the LLM diariser ran with. Useful for the modal to - * pre-fill its textarea when the operator wants to iterate on a - * previously-diarised row. - */ - diarised_prompt: string | null - /** - * Diarisation pipeline that produced this row's turns. - * - `stt_llm` (default) — two-stage STT then LLM diariser. - * - `llm_only` — single-stage multimodal LLM (audio in). - * Read-only; written by the worker on each diarisation. - */ - transcribe_mode?: 'stt_llm' | 'llm_only' - /** - * Per-row preservation of the mapped source cells. Values land here - * as whatever type the schema parameter coerced them to — - * strings (text / url / conversation_id / recording_url / - * recording_date / transcript / datetime), numbers, booleans, or - * ``null`` for blanks. Always - * coerce with ``String(value)`` before string operations. - */ - raw_columns: Record | null - status: CallImportRowStatus - recording_s3_key: string | null - recording_content_type: string | null - recording_size_bytes: number | null - error_message: string | null - attempts: number - created_at: string - updated_at: string -} - -export interface CallImportTag { - id: string - name: string - color: string | null - created_at: string - updated_at: string -} - -/** - * Parameter type tag on a Call Import schema parameter. - * - * - ``conversation_id``: mandatory identifier (one per schema). - * - ``recording_url``: feeds ``CallImportRow.recording_url``. - * - ``recording_date``: date-only call recording date used for reports. - * - ``transcript``: feeds ``CallImportRow.transcript``. - * - ``text`` / ``number`` / ``boolean`` / ``datetime`` / ``url``: - * generic typed fields preserved per row in ``raw_columns`` and - * surfaced in the evaluation export under the parameter's name. - */ -export type CallImportSchemaParameterType = - | 'conversation_id' - | 'recording_url' - | 'recording_date' - | 'transcript' - | 'text' - | 'number' - | 'boolean' - | 'datetime' - | 'url' - -export interface CallImportSchemaParameter { - id?: string - name: string - type: CallImportSchemaParameterType - description: string | null - is_required: boolean - ordering?: number -} - -export interface CallImportSchema { - id: string - organization_id: string - workspace_id: string - name: string - description: string | null - parameters: CallImportSchemaParameter[] - /** How many CallImport batches reference this schema. */ - usage_count: number - created_at: string - updated_at: string -} - -export interface CallImportSchemaListResponse { - items: CallImportSchema[] - total: number -} - -export interface CallImportSchemaCreate { - name: string - description?: string | null - parameters: Array> -} - -export interface CallImportSchemaUpdate { - name?: string - description?: string | null - parameters?: Array> -} - -/** - * In-org Workspace - the active workspace scopes call imports and - * metrics in the UI. The org's Default workspace is auto-seeded by - * migration 033 and cannot be deleted. - */ -export interface Workspace { - id: string - organization_id: string - name: string - slug: string - is_default: boolean - created_at: string - updated_at: string - role_id?: string | null - role_name?: string | null - capabilities?: string[] -} - -export interface WorkspaceRole { - id: string - organization_id: string - name: string - description?: string | null - capabilities: string[] - is_system: boolean - created_at: string - updated_at: string -} - -export interface WorkspaceMember { - id: string - workspace_id: string - user_id: string - role_id: string - role_name: string - user_email: string - user_name?: string | null - added_by_user_id?: string | null - created_at: string -} - -export interface CapabilityInfo { - key: string - label: string -} - -export interface CapabilityDomain { - key: string - label: string - capabilities: CapabilityInfo[] -} - -export interface WorkspaceRoleCreate { - name: string - description?: string | null - capabilities: string[] -} - -export interface WorkspaceRoleUpdate { - name?: string - description?: string | null - capabilities?: string[] -} - -export interface CallImportSourceRowSkip { - source_row: number - reason: string - message: string -} - -export interface CallImport { - id: string - organization_id: string - /** Workspace this import belongs to. */ - workspace_id: string - /** - * Telephony provider key. ``null`` until the IMPORT stage in the - * staged flow (which is the first step that knows the provider). - * Always populated on post-import batches and on legacy one-shot - * uploads. - */ - provider: string | null - telephony_integration_id: string | null - original_filename: string | null - /** - * For Excel uploads, which worksheet this batch came from. ``null`` - * for CSV uploads (CSV files have no sheet concept) and for any - * imports created before multi-sheet support landed. - */ - sheet_name: string | null - /** Optional free-text dataset label (high-level segregation filter). */ - dataset: string | null - /** Tags currently attached to this import. Empty array if untagged. */ - tags: CallImportTag[] - /** - * Reusable Input Parameter schema the batch was uploaded against. - * NULL on legacy batches uploaded before the schema-driven flow shipped. - */ - schema_id: string | null - /** - * Schema-driven mapping: ``{parameter_name: csv_header}``. Empty on - * legacy batches; check ``column_mapping`` / ``extra_columns`` / - * ``custom_column_mapping`` instead for those. - */ - parameter_mapping: Record - /** Legacy free-form mapping kept for batches uploaded before schemas. */ - column_mapping: Record - /** Legacy extra-column list kept for backwards-compat. */ - extra_columns: string[] - /** Legacy uploader-named columns kept for backwards-compat. */ - custom_column_mapping: Record - /** - * Source headers the uploader explicitly skipped, captured at the - * MAP stage. Empty for legacy one-shot uploads where the value was - * ephemeral. - */ - skipped_columns: string[] - /** - * Source rows skipped at parse time (missing/invalid conversation ID or URL). - */ - source_row_skips?: CallImportSourceRowSkip[] - /** S3 key for the staged source file. ``null`` on legacy batches. */ - source_s3_key: string | null - /** ``'csv'`` / ``'xlsx'`` for staged files, or ``'audio'`` for manual uploads. */ - source_format: string | null - source_size_bytes: number | null - source_content_type: string | null - /** - * Snapshot of the file's sheets + headers captured at UPLOAD time so - * the MAP UI can render without re-fetching the source from S3. - * ``null`` on legacy batches. - */ - available_sheets: CallImportPreviewSheet[] | null - total_rows: number - completed_rows: number - failed_rows: number - status: CallImportStatus - error_message: string | null - created_at: string - updated_at: string -} - -export interface CallImportDetail extends CallImport { - rows: CallImportRow[] - /** - * Total row count *after* applying the optional ``q`` search filter. - * ``null`` when no filter is active — paginate against ``total_rows`` - * in that case. - */ - filtered_total_rows: number | null - /** - * Batch-wide aggregates of ``CallImportRow.diarised_transcript_status``. - * The ``idle`` bucket (rows never touched by the transcribe/diarise - * worker) is implicit: ``total_rows - (pending + running + completed - * + failed)``. Lets the UI render a transcribe-and-diarise progress - * bar without paginating through every row. - */ - diarised_pending_rows: number - diarised_running_rows: number - diarised_completed_rows: number - diarised_failed_rows: number -} - -export interface CallImportListResponse { - items: CallImport[] - total: number - page: number - page_size: number -} - -export interface CallImportUploadResponse { - id: string - total_rows: number - status: CallImportStatus - dataset: string | null - tags: CallImportTag[] - message: string -} - -/** One worksheet (or one CSV file synthesized as a single sheet). */ -export interface CallImportPreviewSheet { - /** Sheet name for xlsx; filename for csv. */ - name: string - /** Column headers from the first non-empty row. */ - headers: string[] - /** Approximate count of data rows (excludes the header row). */ - row_count: number -} - -/** - * Sheets / headers extracted server-side from an uploaded CSV or Excel - * workbook. Drives the modal's column-mapping UI without forcing the - * frontend to parse the file itself. - */ -export interface CallImportPreviewResponse { - /** ``'csv'`` or ``'xlsx'``. */ - format: 'csv' | 'xlsx' - sheets: CallImportPreviewSheet[] -} - -export type MetricSelectionMode = 'single_choice' | 'multi_label' - -export interface CallImportMetricSummary { - id: string - name: string - metric_type: string | null - description: string | null - parent_metric_id?: string | null - selection_mode?: MetricSelectionMode | null - /** Only meaningful on multi_label parents; gates the Discovered - * Labels panel on the Flow tab. Defaults to false. */ - allow_discovery?: boolean -} - -/** Per-metric LLM override (provider+model+optional credential + generation params). */ -export interface CallImportEvaluationLLMOverride { - provider?: string | null - model?: string | null - credential_id?: string | null - llm_config?: LLMGenerationConfig | null -} - -export interface CallImportEvaluation { - id: string - call_import_id: string - organization_id: string - /** User-supplied label for the run; null when not named. */ - name: string | null - selected_metric_ids: string[] - /** parent_id -> [child_id, ...] snapshot captured at run time. */ - selected_metric_groups?: Record | null - metrics: CallImportMetricSummary[] - status: 'pending' | 'running' | 'completed' | 'partial' | 'failed' - total_rows: number - completed_rows: number - failed_rows: number - error_message: string | null - /** Run-level LLM provider chosen by the user (null = legacy default). */ - llm_provider: string | null - llm_model: string | null - llm_credential_id: string | null - llm_config?: LLMGenerationConfig | null - metric_llm_overrides: Record | null - stt_provider: string | null - stt_model: string | null - stt_credential_id: string | null - /** - * Run-level LLM diariser config used when the worker auto-diarises - * rows that are missing a diarised transcript. - */ - diarisation_llm_provider?: string | null - diarisation_llm_model?: string | null - diarisation_llm_credential_id?: string | null - diarisation_prompt?: string | null - /** - * Diarisation pipeline shape this run was created with. - * - `stt_llm` (default) — STT then an LLM diariser over the text. - * - `llm_only` — audio fed directly to a multimodal diariser LLM. - * Surfaced so the retry / re-run UI can preselect the right mode. - */ - transcribe_mode?: 'stt_llm' | 'llm_only' - /** - * Which transcript column this run scored against. - * Defaults to `production` on legacy runs. - */ - transcript_source: CallImportEvaluationTranscriptSource - /** - * Other evaluation ids created in the same Run Evaluation request. - * Populated only on the POST response when the user ticked both - * Production and Diarised. Empty array on all other reads. - */ - sibling_evaluation_ids: string[] - started_at: string | null - finished_at: string | null - created_at: string - updated_at: string - /** - * Cached LLM-generated TLDR rendered above the Visualizations tab. - * Populated lazily via ``POST /evaluations/{id}/insights``; null on - * runs the user has not summarised yet. - */ - tldr_summary?: EvaluationTldrSummary | null - user_insights?: EvaluationUserInsightsState | null - metric_clusters?: EvaluationMetricClustersState | null - /** - * True when the user opted into top-level metric discovery on the - * Run Evaluation modal. Gates the Discovered metrics panel on the - * evaluation detail Flow tab. - */ - discover_new_metrics?: boolean - /** - * Set while a bulk background operation (abort, force-fail, retry) is - * still running. The UI disables other mutating actions until cleared. - */ - bulk_operation?: 'abort' | 'force_fail_pending' | 'retry' | null -} - -/** - * LLM-generated narrative + bullet patterns for a single evaluation - * run. Cached on the evaluation row so re-opening the Visualizations - * tab doesn't auto-burn LLM tokens. ``is_stale`` is computed by the - * backend at read time when ``completed_rows`` has grown since the - * summary was generated. - */ -export interface EvaluationTldrSummary { - narrative: string - patterns: string[] - metric_insights?: Record - generated_at: string - generated_at_completed_rows: number - provider?: string | null - model?: string | null - is_stale: boolean -} - -export interface UserInsightCategory { - label: string - count: number - share_pct: number -} - -export interface UserInsightEvidenceTurn { - speaker: string - text: string -} - -export interface UserInsightEvidence { - conversation_id?: string | null - quote: string - turns?: UserInsightEvidenceTurn[] -} - -export interface EvaluationUserInsightItem { - id: string - title: string - categories: UserInsightCategory[] - observation: string - evidence: UserInsightEvidence -} - -export interface EvaluationUserInsightsState { - status: 'idle' | 'running' | 'completed' | 'failed' - insights: EvaluationUserInsightItem[] - overview?: string | null - generated_at?: string | null - generated_at_completed_rows: number - progress?: { completed_llm_calls: number; total_llm_calls: number } | null - provider?: string | null - model?: string | null - llm_calls_used: number - max_llm_calls?: number | null - error_message?: string | null - is_stale: boolean -} - -export type MetricClusterGapLabel = - | 'LOGIC_GAP' - | 'UNDERSPEC' - | 'EXISTS_NO_TRIGGER' - | 'MISSING' - -export interface MetricSubCluster { - label: string - count: number - share_pct: number -} - -export interface MetricClusterEvidenceTurn { - speaker: string - text: string -} - -export interface MetricClusterEvidence { - conversation_id?: string | null - evaluation_row_id?: string | null - quote: string - turns?: MetricClusterEvidenceTurn[] -} - -export interface MetricCluster { - id: string - label: string - gap_label: MetricClusterGapLabel - level: number - count: number - share_pct: number - sub_clusters: MetricSubCluster[] - observation: string - failure_reason?: string - evidence: MetricClusterEvidence - is_discovered: boolean -} - -export interface MetricClusterGroup { - metric_id: string - metric_name: string - flagged_count: number - failure_reason?: string - clusters: MetricCluster[] -} - -export interface DiscoveredProblemCluster { - id: string - label: string - gap_label: MetricClusterGapLabel - count: number - share_pct: number - observation: string - failure_reason?: string - evidence: MetricClusterEvidence -} - -export interface RcaRepeatedPatternRow { - metric_id: string - metric_name: string - top_rca_patterns: string - evidence_share_pct: number - evidence_calls: number - evidence_cluster_count?: number - failure_reason: string -} - -export interface RcaMetricHotspotRow { - metric_id: string - metric_name: string - description: string - metric_rate_pct: number - flagged_calls: number -} - -export interface RcaPromptAreaRow { - label: string - share_pct: number - gap_label: MetricClusterGapLabel -} - -export interface MetricClustersRcaSummary { - total_clusters: number - total_clustered_instances: number - total_flagged_instances?: number - analysed_calls: number - repeated_patterns: RcaRepeatedPatternRow[] - metric_hotspots: RcaMetricHotspotRow[] - prompt_areas: RcaPromptAreaRow[] -} - -export interface MetricFailurePolicy { - metric_id: string - failure_values: string[] - failure_child_names?: string[] - numeric_rule?: { op: 'lt' | 'lte' | 'gt' | 'gte'; threshold: number } | null -} - -export interface MetricFailurePolicyValueCount { - label: string - count: number -} - -export interface MetricFailurePolicyMetricPreview { - metric_id: string - metric_name: string - metric_type?: string | null - selection_mode?: string | null - is_multi_label_parent: boolean - value_counts: MetricFailurePolicyValueCount[] - child_names: string[] - row_count_by_value: Record - suggested_policy: MetricFailurePolicy - effective_policy: MetricFailurePolicy -} - -export interface MetricFailurePoliciesResponse { - previews: MetricFailurePolicyMetricPreview[] - policies: Record - source: 'inferred' | 'user' - updated_at?: string | null -} - -export interface MetricClusterEligibleRow { - evaluation_row_id: string - conversation_id?: string | null - row_index?: number | null - flagged_metric_names: string[] -} - -export interface MetricClusterEligibleRowsResponse { - items: MetricClusterEligibleRow[] - total: number -} - -export interface EvaluationMetricClustersState { - status: 'idle' | 'running' | 'completed' | 'failed' | 'cancelled' - groups: MetricClusterGroup[] - discovered_problems: DiscoveredProblemCluster[] - overview?: string | null - generated_at?: string | null - generated_at_completed_rows: number - progress?: { completed_llm_calls: number; total_llm_calls: number } | null - provider?: string | null - model?: string | null - llm_calls_used: number - max_llm_calls?: number | null - error_message?: string | null - is_stale: boolean - selected_evaluation_row_ids?: string[] - failure_policies?: Record - failure_policies_source?: 'inferred' | 'user' - failure_policies_updated_at?: string | null - rca_summary?: MetricClustersRcaSummary | null -} - -export interface AgentFlowNode { - id: string - label: string - node_type: 'start' | 'decision' | 'action' | 'terminal' - position_x?: number | null - position_y?: number | null - prompt_excerpt?: string | null - start_offset?: number | null - end_offset?: number | null -} - -export interface AgentFlowEdge { - source: string - target: string - condition?: string | null -} - -export interface AgentFlowGraph { - nodes: AgentFlowNode[] - edges: AgentFlowEdge[] - generated_at?: string | null - provider?: string | null - model?: string | null - layout_saved_at?: string | null - prompt_content_hash?: string | null - mapping_error?: string | null - generation_error?: string | null -} - -export interface ImportedAgent { - id: string - organization_id: string - name: string - description: string | null - content: string - tags: string[] | null - current_version: number - agent_flowchart?: AgentFlowGraph | null - agent_flowchart_status?: string | null - created_at: string - updated_at: string - created_by: string | null -} - -export interface ImportedAgentDetail extends ImportedAgent { - versions: PromptPartialVersion[] -} - -export interface MetricPartialChild { - name: string - description: string - example: string -} - -export interface MetricPartialContent { - schema_version: 1 - metric_kind: 'single' | 'category' - description: string - children?: MetricPartialChild[] -} - -export interface MetricPartial { - id: string - organization_id: string - name: string - description: string | null - content: string - tags: string[] | null - current_version: number - created_at: string - updated_at: string - created_by: string | null -} - -export interface MetricPartialDetail extends MetricPartial { - versions: PromptPartialVersion[] -} - -export interface PromptPartialVersion { - id: string - prompt_partial_id: string - version: number - content: string - change_summary: string | null - created_at: string - created_by: string | null -} - -export interface PromptImprovementSuggestion { - id: string - metric_id: string - metric_name: string - cluster_id: string - cluster_label: string - gap_label: MetricClusterGapLabel - share_pct: number - priority: 'high' | 'medium' | 'low' - change_type?: 'edit' | 'add' - target_section: string - anchor_excerpt?: string - current_gap: string - suggested_text: string - rationale: string - flow_node_id?: string - flow_node_label?: string -} - -export interface EvaluationPromptImprovementsState { - status: 'idle' | 'running' | 'completed' | 'failed' - imported_agent_id?: string | null - imported_agent_name?: string | null - suggestions: PromptImprovementSuggestion[] - overview?: string | null - generated_at?: string | null - generated_at_completed_rows: number - provider?: string | null - model?: string | null - error_message?: string | null - is_stale: boolean -} - -export interface MetricPeriodDelta { - label: string - detail: string - why?: string | null -} - -export interface CallImportEvaluationListResponse { - items: CallImportEvaluation[] - total: number -} - -export interface CallImportEvaluationBaselineCandidate { - evaluation_id: string - name: string - dataset: string - period_label: string | null - period_start: string | null - period_end: string | null - period_display: string - completed_rows: number - created_at: string - is_default: boolean -} - -export interface CallImportEvaluationBaselineCandidatesResponse { - items: CallImportEvaluationBaselineCandidate[] - default_evaluation_id: string | null -} - -export interface CallImportEvaluationRow { - id: string - evaluation_id: string - call_import_row_id: string - row_index: number | null - /** Mandatory identifier from the source batch (renamed from ``external_call_id``). */ - conversation_id: string | null - transcript: string | null - raw_columns: Record | null - recording_url: string | null - recording_date: string | null - /** - * S3 object key for the downloaded recording. Prefer this over - * ``recording_url`` for playback — we resolve it to a presigned URL - * so audio plays from our storage instead of the (often expired) - * provider URL. - */ - recording_s3_key: string | null - diarised_transcript_status?: string | null - diarised_transcript_error?: string | null - status: 'pending' | 'running' | 'completed' | 'failed' | 'skipped' - metric_scores: Record - error_message: string | null - started_at: string | null - finished_at: string | null - created_at: string - updated_at: string -} - -export interface CallImportEvaluationRowListResponse { - items: CallImportEvaluationRow[] - total: number - page: number - page_size: number -} - -// --- Retry (re-enqueue failed rows on an existing evaluation run) --- - -export interface CallImportEvaluationRetryRequest { - /** - * Restrict the retry to a specific subset of evaluation rows. - * When omitted, every row with status='failed' in this run is - * re-enqueued. - */ - eval_row_ids?: string[] - - /** - * Optional LLM overrides. When provided, persisted onto the run so - * future retries default to the new config. ``llm_provider`` and - * ``llm_model`` must be sent together — the backend 400s on - * half-configured input. - */ - llm_provider?: string - llm_model?: string - llm_credential_id?: string | null - - /** - * Optional STT overrides (only meaningful when the run scores the - * diarised transcript). Same paired-field rule as LLM. - */ - stt_provider?: string - stt_model?: string - stt_credential_id?: string | null - - /** - * When true, wipe the diarised transcript on every retried row so - * the (possibly new) STT runs from scratch. Only takes effect for - * diarised runs that have STT config. - */ - transcribe_overwrite?: boolean -} - -export interface CallImportEvaluationRetrySkippedItem { - eval_row_id: string - /** - * Why this row was not re-enqueued. Known values: - * - 'unknown' (id not in this run) - * - 'in_progress' (status is pending/running) - * - 'completed' (already successful) - * - 'source_row_missing' - */ - reason: 'unknown' | 'in_progress' | 'completed' | 'source_row_missing' -} - -export interface CallImportEvaluationRetryResponse { - requeued: number - /** - * Of those, how many were chained through a diarisation task first - * because the diarised transcript was missing. - */ - transcribe_requeued: number - skipped: CallImportEvaluationRetrySkippedItem[] -} - -export interface CallImportEvaluationBulkActionResponse { - accepted: boolean - target_count: number - evaluation_id: string -} - -// --- Diarization / transcription --- - -export interface CallImportTranscribeRequest { - /** - * Diarisation pipeline shape. - * - `stt_llm` (default) — STT produces plain text, then an LLM - * diariser splits it into agent/user turns. STT fields required. - * - `llm_only` — skip STT entirely and feed the audio bytes - * directly to a multimodal `diarization_llm_*` model along with - * `diarization_prompt`. STT fields MUST be omitted in this mode. - */ - mode?: 'stt_llm' | 'llm_only' - /** Required when `mode === 'stt_llm'`; must be null/omitted in `llm_only`. */ - stt_provider?: string | null - /** Required when `mode === 'stt_llm'`; must be null/omitted in `llm_only`. */ - stt_model?: string | null - credential_id?: string | null - language?: string | null - only_missing?: boolean - overwrite_existing?: boolean - row_ids?: string[] - /** - * LLM diariser. In `stt_llm` mode it splits the STT plain-text into - * agent/user turns; in `llm_only` mode it directly receives the - * audio along with `diarization_prompt`. Always required. - */ - diarization_llm_provider: string - diarization_llm_model: string - diarization_llm_credential_id?: string | null - /** - * Operator-supplied system prompt for the diariser LLM. NULL/empty - * means "fall back to the canonical default" (see - * ``getDiarisationDefaultPrompt``). - */ - diarization_prompt?: string | null -} - -export interface CallImportTranscribeResponse { - queued: number - skipped_rows: number - skipped_reason_counts: Record - accepted?: boolean -} - -export interface CallImportRowBulkDeleteResponse { - deleted: number - status?: 'completed' | 'accepted' -} - -export interface CallImportRetryFailedRowsResponse { - requeued: number - enqueue_failed: number - skipped: number -} - -export interface CallImportDiarisationPromptDefaultResponse { - prompt: string -} - -// --- Aggregation / visualization payloads --- - -export interface CallImportMetricHistogramBucket { - x0: number - x1: number - count: number -} - -export interface CallImportMetricValueCount { - label: string - count: number -} - -/** - * One unordered pair-count cell from a multi-label parent's - * co-occurrence matrix. ``a`` and ``b`` are child label names with - * ``a < b`` lexicographically; ``count`` is the number of rows on - * which both labels fired together. - */ -export interface CallImportMetricLabelPair { - a: string - b: string - count: number -} - -export interface CallImportMetricAggregate { - metric_id: string - metric_name: string - metric_type: string | null - metric_category?: 'quality' | 'user_insight' | string - /** - * True when the metric is a multi-label classifier parent. - * ``value_counts`` then lists per-child label tallies and one row - * may contribute to several labels, so the chart layout has to - * ignore the pie toggle (slices wouldn't sum to 100%) and the - * n-badge represents rows scored, not label occurrences. - */ - is_multi_label_parent?: boolean - count: number - skipped_count: number - error_count: number - mean: number | null - median: number | null - p25: number | null - p75: number | null - p95: number | null - min: number | null - max: number | null - stddev: number | null - histogram_buckets: CallImportMetricHistogramBucket[] - value_counts: CallImportMetricValueCount[] - /** - * Pairwise label intersections for multi-label parent metrics. - * Empty for everything else. The Visualizations tab reconstructs - * a square symmetric matrix from these unordered pairs to render - * the co-occurrence heatmap chart type. - */ - co_occurrence?: CallImportMetricLabelPair[] -} - -export interface CallImportEvaluationAggregateResponse { - evaluation_id: string - total_rows: number - completed_rows: number - failed_rows: number - metrics: CallImportMetricAggregate[] - period_deltas?: Record - baseline_evaluation_id?: string | null - failure_policies_source?: 'inferred' | 'user' | null -} - -export interface EvaluatorResultsAggregateResponse { - scope: string - suite_id?: string | null - agent_id?: string | null - scenario_id?: string | null - total_rows: number - completed_rows: number - failed_rows: number - metrics: CallImportMetricAggregate[] -} - -export interface CallImportInsightsRunPoint { - evaluation_id: string - name: string | null - created_at: string - mean: number | null - completed_rows: number -} - -export interface CallImportInsightsMetric { - metric_id: string - metric_name: string - metric_type: string | null - latest: CallImportMetricAggregate | null - trend: CallImportInsightsRunPoint[] -} - -export interface CallImportInsightsResponse { - call_import_id: string - total_rows: number - rows_with_transcript: number - rows_without_transcript: number - transcript_source_counts: Record - evaluation_count: number - metrics: CallImportInsightsMetric[] -} - -// --- Metrics hierarchy + flow visualization --- - -export interface MetricSummary { - id: string - organization_id: string - name: string - description: string | null - metric_type: string - metric_category?: 'quality' | 'user_insight' | string - trigger: string - enabled: boolean - is_default: boolean - metric_origin: string - supported_surfaces: string[] - enabled_surfaces: string[] - custom_data_type: string | null - custom_config: Record | null - tags: string[] | null - capture_rationale: boolean - parent_metric_id: string | null - selection_mode: MetricSelectionMode | null - allow_discovery?: boolean - /** - * When true, this metric is a "transcript-compare judge": at - * call-import evaluation time the worker feeds BOTH the production - * transcript and the diarised transcript to the LLM as a labeled - * pair, and the run's transcript_source toggle is ignored for this - * metric. Mutually exclusive with parent_metric_id and selection_mode - * — comparison metrics stay standalone. - */ - compare_transcripts?: boolean - children?: MetricSummary[] - created_at: string - updated_at: string - created_by: string | null -} - -export interface MetricChildDraft { - name: string - description?: string | null - enabled?: boolean - capture_rationale?: boolean | null - tags?: string[] | null -} - -export interface MetricCreateWithChildrenPayload { - name: string - description?: string | null - selection_mode: MetricSelectionMode - enabled?: boolean - supported_surfaces?: string[] - enabled_surfaces?: string[] - tags?: string[] | null - allow_discovery?: boolean - children: MetricChildDraft[] -} - -export interface MetricFlowNode { - id: string - label: string - count: number - is_terminal: boolean - is_discovered?: boolean -} - -export interface MetricFlowEdge { - source: string - target: string - count: number -} - -export interface MetricFlowResponse { - parent_metric_id: string - parent_metric_name: string - selection_mode: MetricSelectionMode | null - nodes: MetricFlowNode[] - edges: MetricFlowEdge[] - total_rows: number - rows_with_sequence: number -} - -export interface DiscoveredLabel { - key: string - name: string - description?: string | null - sample_rationale?: string | null - /** - * Up to 3 distinct LLM rationales captured for this candidate - * across rows. The Discovered Labels promote flow surfaces the - * first 2 as an ``Examples:`` block on the new sub-metric's - * rubric so the user starts with concrete cases in the prompt. - */ - examples?: string[] - count: number -} - -export interface DiscoveredLabelsResponse { - parent_metric_id: string - items: DiscoveredLabel[] -} - -/** - * One LLM-discovered candidate TOP-LEVEL metric aggregated across all - * rows of an evaluation. Mirrors :class:`DiscoveredLabel` but adds a - * ``suggested_type`` field — the LLM's guess at the best shape for - * the new metric — that the promote modal can pre-fill the type radio - * with. - */ -export interface DiscoveredMetric { - key: string - name: string - description?: string | null - suggested_type: 'boolean' | 'rating' | 'category' - sample_rationale?: string | null - examples?: string[] - count: number -} - -export interface DiscoveredMetricsResponse { - evaluation_id: string - items: DiscoveredMetric[] -} - -export interface ObservabilityCallAgent { - id: string - agent_id?: string | null - name: string -} - -export interface ObservabilityCallData { - startedAt?: string - started_at?: string - endedAt?: string - ended_at?: string - from_phone_number?: string - to_phone_number?: string - endedReason?: string - recording_s3_key?: string - recording_url?: string - duration_seconds?: number - agent_name?: string - _agent_ref?: string | number - direction?: string - messages?: Array<{ role: string; content: string; start_time?: number; end_time?: number }> - live_transcript?: Array<{ role: string; content: string; timestamp?: string; start_time?: number }> - metadata?: Record - call_short_id?: string -} - -export interface ObservabilityCall { - id: string - call_short_id: string - status?: string | null - call_event?: string | null - is_live?: boolean - direction?: string | null - source?: string | null - provider_platform?: string | null - provider_call_id?: string | null - agent_id?: string | null - agent?: ObservabilityCallAgent | null - created_at?: string | null - updated_at?: string | null - call_data?: ObservabilityCallData | null - live_transcript?: Array<{ role: string; content: string; timestamp?: string }> -} +// API Types matching the backend schemas + +export type { LLMGenerationConfig } from '../config/llmGenerationParams' +import type { LLMGenerationConfig } from '../config/llmGenerationParams' + +export enum EvaluationType { + ASR = 'asr', + TTS = 'tts', +} + +export enum EvaluationStatus { + PENDING = 'pending', + PROCESSING = 'processing', + COMPLETED = 'completed', + FAILED = 'failed', + CANCELLED = 'cancelled', +} + +export interface AudioFile { + id: string + filename: string + format: string + file_size: number + duration?: number | null + sample_rate?: number | null + channels?: number | null + uploaded_at: string +} + +export interface Evaluation { + id: string + audio_id: string + reference_text?: string | null + evaluation_type: EvaluationType + model_name?: string | null + status: EvaluationStatus + metrics_requested?: string[] | null + created_at: string + started_at?: string | null + completed_at?: string | null + error_message?: string | null +} + +export interface DashboardSummary { + evaluations: { + total: number + completed: number + pending: number + failed: number + } + resources: { + agents: number + personas: number + scenarios: number + integrations: number + voice_bundles: number + } + setup_progress: { + has_integration: boolean + has_voice_bundle: boolean + has_agent: boolean + has_evaluation: boolean + } + metrics: { + total: number + enabled: number + } + call_imports: { + total: number + } + call_import_evaluations: { + total: number + completed: number + running: number + failed: number + } + recent_evaluations: Evaluation[] +} + +export interface ModelConfigEntry { + provider: string + model_type: string + description?: string + featured?: boolean + featured_rank?: number + highlights?: string[] +} + +export interface EvaluationCreate { + audio_id: string + reference_text?: string | null + evaluation_type: EvaluationType + model_name?: string | null + metrics?: string[] +} + +export interface EvaluationResult { + evaluation_id: string + status: EvaluationStatus + transcript?: string | null + metrics: Record + processing_time?: number | null + model_used?: string | null + created_at: string +} + +export interface BatchEvaluationResult { + processed_files: number + failed_files: number + aggregated_metrics?: Record | null + individual_results: EvaluationResult[] +} + +/** Voice agent evaluator run (evaluator_results table). */ +export type EvaluatorResultStatus = + | 'queued' + | 'call_initiating' + | 'call_connecting' + | 'call_in_progress' + | 'call_ended' + | 'transcribing' + | 'evaluating' + | 'fetching_details' + | 'completed' + | 'failed' + +export interface EvaluatorResultMetricScore { + value: unknown + type: string + metric_name: string + parent_metric_id?: string | null +} + +export interface EvaluatorResultRow { + id: string + result_id: string + name: string | null + evaluator_id: string | null + agent_id?: string | null + persona_id?: string | null + scenario_id?: string | null + suite_id?: string | null + timestamp: string + duration_seconds: number | null + status: EvaluatorResultStatus + metric_scores: Record | null + error_message: string | null + agent?: { id: string; name: string } | null + scenario?: { id: string; name: string } | null +} + +export interface EvaluatorResultListResponse { + items: EvaluatorResultRow[] + total: number +} + +export interface EvaluatorResultCounts { + total: number + completed: number + failed: number + in_progress: number + last_run_at?: string | null +} + +export interface EvaluatorResultsScenarioSummary { + scenario_id: string + scenario_name: string + counts: EvaluatorResultCounts +} + +export interface EvaluatorResultsSuiteSummary { + suite_id: string + suite_name?: string | null + agent_id: string + persona_id?: string | null + counts: EvaluatorResultCounts + scenarios?: EvaluatorResultsScenarioSummary[] | null +} + +export interface EvaluatorResultsAgentSummary { + agent_id: string + agent_name: string + counts: EvaluatorResultCounts + suites?: EvaluatorResultsSuiteSummary[] | null +} + +export interface EvaluatorResultsOverviewResponse { + workspace_counts: EvaluatorResultCounts + agents: EvaluatorResultsAgentSummary[] + unassigned: { + counts: EvaluatorResultCounts + recent_result_ids: string[] + } +} + +export interface ListEvaluatorResultsParams { + skip?: number + limit?: number + evaluatorId?: string + agentId?: string + suiteId?: string + scenarioId?: string + status?: 'completed' | 'failed' | 'in_progress' + unassignedOnly?: boolean + playground?: boolean + testAgentsOnly?: boolean +} + +export interface APIKey { + id: string + key: string + name?: string | null + is_active: boolean + created_at: string + last_used?: string | null + message?: string +} + +export interface MessageResponse { + message: string +} + +// IAM & User Types +export enum Role { + READER = 'reader', + WRITER = 'writer', + ADMIN = 'admin', +} + +export enum InvitationStatus { + PENDING = 'pending', + ACCEPTED = 'accepted', + DECLINED = 'declined', + EXPIRED = 'expired', +} + +export interface User { + id: string + email: string + name?: string | null + is_active: boolean + created_at: string +} + +export interface OrganizationMember { + id: string + user_id: string + organization_id: string + role: Role + joined_at: string + user: User +} + +export interface Invitation { + id: string + organization_id: string + email: string + role: Role + status: InvitationStatus + expires_at: string + created_at: string + organization_name?: string | null +} + +export interface InvitationCreate { + email: string + role: Role +} + +export interface RoleUpdate { + role: Role +} + +export interface Profile { + id: string + email: string + name?: string | null + first_name?: string | null + last_name?: string | null + created_at: string + organizations: Array<{ + id: string + name: string + role: string + joined_at: string + }> +} + +export interface UserUpdate { + name?: string | null + first_name?: string | null + last_name?: string | null + email?: string | null +} + +export interface UserPreferences { + theme?: string + notifications_enabled?: boolean + email_notifications?: boolean + default_language?: string + [key: string]: any +} + +export interface UserPreferencesUpdate { + theme?: string + notifications_enabled?: boolean + email_notifications?: boolean + default_language?: string + [key: string]: any +} + +// Integration Types +export enum IntegrationPlatform { + RETELL = 'retell', + VAPI = 'vapi', + CARTESIA = 'cartesia', + ELEVENLABS = 'elevenlabs', + DEEPGRAM = 'deepgram', + MURF = 'murf', + SARVAM = 'sarvam', + VOICEMAKER = 'voicemaker', + SMALLEST = 'smallest', +} + +export enum TelephonyProvider { + PLIVO = 'plivo', + EXOTEL = 'exotel', + VOBIZ = 'vobiz', +} + +export type CredentialRoutingMode = 'inherit' | 'gateway' | 'direct' +export type GatewayInterfaceMode = 'inherit' | 'litellm_shim' | 'native_openai' + +export type EffectiveCredentialRouting = + | 'inherit' + | 'direct' + | 'gateway' + | 'bifrost' + | 'litellm_proxy' + +export interface Integration { + id: string + organization_id: string + platform: IntegrationPlatform + name?: string | null + public_key?: string | null + is_active: boolean + /** True if this row is the default credential for (org, platform). */ + is_default?: boolean + routing_mode?: CredentialRoutingMode + effective_routing?: EffectiveCredentialRouting + created_at: string + updated_at: string + last_tested_at?: string | null +} + +export interface IntegrationCreate { + platform: IntegrationPlatform + api_key: string + public_key?: string + name?: string | null + routing_mode?: CredentialRoutingMode + /** Mark the new credential as the default for (org, platform). */ + is_default?: boolean +} + +// VoiceBundle Types +export enum ModelProvider { + OPENAI = 'openai', + ANTHROPIC = 'anthropic', + GOOGLE = 'google', + XAI = 'xai', + FIREWORKS = 'fireworks', + COHERE = 'cohere', + MISTRAL = 'mistral', + META = 'meta', + TOGETHER = 'together', + PERPLEXITY = 'perplexity', + AZURE = 'azure', + AWS = 'aws', + DEEPGRAM = 'deepgram', + CARTESIA = 'cartesia', + ELEVENLABS = 'elevenlabs', + MURF = 'murf', + CUSTOM = 'custom', + SARVAM = 'sarvam', + VOICEMAKER = 'voicemaker', + SMALLEST = 'smallest', +} + +// AI Provider Types +export interface AIProvider { + id: string + provider: ModelProvider + api_key?: string | null + name?: string | null + endpoint_url?: string | null + is_active: boolean + /** True if this row is the default credential for (org, provider). */ + is_default?: boolean + routing_mode?: CredentialRoutingMode + gateway_model?: string | null + gateway_interface?: GatewayInterfaceMode + gateway_base_url?: string | null + gateway_auth_header?: string | null + gateway_auth_secret_env?: string | null + has_gateway_auth_secret?: boolean + gateway_extra_headers?: Record | null + /** True when provider secrets are resolved by the Bifrost gateway. */ + gateway_managed?: boolean + effective_routing?: EffectiveCredentialRouting + effective_gateway_interface?: 'litellm_shim' | 'native_openai' + created_at: string + updated_at: string + last_tested_at?: string | null +} + +export interface AIProviderCreate { + provider: ModelProvider + api_key?: string | null + name?: string | null + endpoint_url?: string | null + routing_mode?: CredentialRoutingMode + gateway_model?: string | null + gateway_interface?: GatewayInterfaceMode + gateway_base_url?: string | null + gateway_auth_header?: string | null + gateway_auth_secret_env?: string | null + gateway_auth_secret?: string | null + gateway_extra_headers?: Record | null + /** Mark the new credential as the default for (org, provider). */ + is_default?: boolean +} + +export interface AIProviderUpdate { + api_key?: string | null + name?: string | null + endpoint_url?: string | null + is_active?: boolean + routing_mode?: CredentialRoutingMode + gateway_model?: string | null + gateway_interface?: GatewayInterfaceMode + gateway_base_url?: string | null + gateway_auth_header?: string | null + gateway_auth_secret_env?: string | null + gateway_auth_secret?: string | null + clear_gateway_auth_secret?: boolean + gateway_extra_headers?: Record | null +} + +export enum VoiceBundleType { + STT_LLM_TTS = 'stt_llm_tts', + S2S = 's2s', +} + +export interface VoiceBundle { + id: string + name: string + description?: string | null + bundle_type: VoiceBundleType + stt_provider?: ModelProvider | null + stt_model?: string | null + /** + * Optional explicit AIProvider/Integration row id for STT. When null the + * runtime resolver picks the default credential for stt_provider. + */ + stt_credential_id?: string | null + llm_provider?: ModelProvider | null + llm_model?: string | null + llm_temperature?: number | null + llm_max_tokens?: number | null + llm_config?: Record | null + llm_credential_id?: string | null + tts_provider?: ModelProvider | null + tts_model?: string | null + tts_voice?: string | null + tts_config?: Record | null + tts_credential_id?: string | null + s2s_provider?: ModelProvider | null + s2s_model?: string | null + s2s_config?: Record | null + s2s_credential_id?: string | null + extra_metadata?: Record | null + is_active: boolean + created_at: string + updated_at: string + created_by?: string | null +} + +export interface VoiceBundleCreate { + name: string + description?: string | null + bundle_type?: VoiceBundleType + stt_provider?: ModelProvider | null + stt_model?: string | null + stt_credential_id?: string | null + llm_provider?: ModelProvider | null + llm_model?: string | null + llm_temperature?: number | null + llm_max_tokens?: number | null + llm_config?: Record | null + llm_credential_id?: string | null + tts_provider?: ModelProvider | null + tts_model?: string | null + tts_voice?: string | null + tts_config?: Record | null + tts_credential_id?: string | null + s2s_provider?: ModelProvider | null + s2s_model?: string | null + s2s_config?: Record | null + s2s_credential_id?: string | null + extra_metadata?: Record | null +} + +// Test Agent Types +export interface AgentPhoneAssignmentConflict { + agent_id: string + agent_name: string + phone_number: string +} + +export interface AgentPhoneAssignmentCheckResponse { + available: boolean + phone_number?: string | null + conflict?: AgentPhoneAssignmentConflict | null +} + +export interface TestAgent { + id: string + agent_id?: string | null + name: string + phone_number?: string | null + telephony_phone_number_id?: string | null + language: string + description: string | null + prompt_variables?: Record | null + silence_hangup_secs?: number + call_type: string + call_medium: string + voice_bundle_id?: string | null + voice_ai_integration_id?: string | null + voice_ai_agent_id?: string | null + provider_prompt?: string | null + provider_prompt_synced_at?: string | null + created_at: string + updated_at: string +} + +// Test Agent Conversation Types +export interface TestAgentConversation { + id: string + organization_id: string + agent_id: string + persona_id: string + scenario_id: string + voice_bundle_id: string + status: string + live_transcription?: Array<{ + speaker: string + text: string + timestamp: number + audio_segment_key?: string + }> | null + conversation_audio_key?: string | null + full_transcript?: string | null + started_at: string + ended_at?: string | null + duration_seconds?: number | null + conversation_metadata?: Record | null + created_at: string + updated_at: string + created_by?: string | null +} + +export interface TestAgentConversationCreate { + agent_id: string + persona_id: string + scenario_id: string + voice_bundle_id: string + conversation_metadata?: Record | null +} + +export interface TestAgentConversationUpdate { + status?: string | null + live_transcription?: Array> | null + full_transcript?: string | null + conversation_metadata?: Record | null +} + +export interface VoiceBundleUpdate { + name?: string + description?: string | null + stt_provider?: ModelProvider + stt_model?: string + stt_credential_id?: string | null + llm_provider?: ModelProvider + llm_model?: string + llm_temperature?: number | null + llm_max_tokens?: number | null + llm_config?: Record | null + llm_credential_id?: string | null + tts_provider?: ModelProvider + tts_model?: string + tts_voice?: string | null + tts_config?: Record | null + tts_credential_id?: string | null + s2s_provider?: ModelProvider | null + s2s_model?: string | null + s2s_config?: Record | null + s2s_credential_id?: string | null + extra_metadata?: Record | null + is_active?: boolean +} + +// Data Sources Types +export interface S3ConnectionTest { + bucket_name: string + region?: string + access_key_id: string + secret_access_key: string + endpoint_url?: string | null +} + +export interface S3ConnectionTestResponse { + success: boolean + message: string + bucket_name?: string | null +} + +export interface S3FileInfo { + key: string + filename: string + size: number + last_modified: string +} + +export interface S3FolderInfo { + name: string + path: string +} + +export interface S3ListFilesResponse { + files: S3FileInfo[] + total: number + prefix?: string | null +} + +export interface S3BrowseResponse { + folders: S3FolderInfo[] + files: S3FileInfo[] + current_path: string + organization_id: string +} + +export interface S3Status { + enabled: boolean + provider?: 's3' | 'gcs' | string + error?: string | null +} + +// Alert Types +export enum AlertMetricType { + NUMBER_OF_CALLS = 'number_of_calls', + CALL_DURATION = 'call_duration', + ERROR_RATE = 'error_rate', + SUCCESS_RATE = 'success_rate', + LATENCY = 'latency', + CUSTOM = 'custom', +} + +export enum AlertAggregation { + SUM = 'sum', + AVG = 'avg', + COUNT = 'count', + MIN = 'min', + MAX = 'max', +} + +export enum AlertOperator { + GREATER_THAN = '>', + LESS_THAN = '<', + GREATER_THAN_OR_EQUAL = '>=', + LESS_THAN_OR_EQUAL = '<=', + EQUAL = '=', + NOT_EQUAL = '!=', +} + +export enum AlertNotifyFrequency { + IMMEDIATE = 'immediate', + HOURLY = 'hourly', + DAILY = 'daily', + WEEKLY = 'weekly', +} + +export enum AlertStatus { + ACTIVE = 'active', + PAUSED = 'paused', + DISABLED = 'disabled', +} + +export enum AlertHistoryStatus { + TRIGGERED = 'triggered', + NOTIFIED = 'notified', + ACKNOWLEDGED = 'acknowledged', + RESOLVED = 'resolved', +} + +export interface Alert { + id: string + organization_id: string + name: string + description?: string | null + metric_type: AlertMetricType + aggregation: AlertAggregation + operator: AlertOperator + threshold_value: number + time_window_minutes: number + agent_ids?: string[] | null + notify_frequency: AlertNotifyFrequency + notify_emails?: string[] | null + notify_webhooks?: string[] | null + status: AlertStatus + created_at: string + updated_at: string + created_by?: string | null +} + +export interface AlertCreate { + name: string + description?: string | null + metric_type?: AlertMetricType + aggregation?: AlertAggregation + operator?: AlertOperator + threshold_value: number + time_window_minutes?: number + agent_ids?: string[] | null + notify_frequency?: AlertNotifyFrequency + notify_emails?: string[] + notify_webhooks?: string[] +} + +export interface AlertUpdate { + name?: string + description?: string | null + metric_type?: AlertMetricType + aggregation?: AlertAggregation + operator?: AlertOperator + threshold_value?: number + time_window_minutes?: number + agent_ids?: string[] | null + notify_frequency?: AlertNotifyFrequency + notify_emails?: string[] + notify_webhooks?: string[] + status?: AlertStatus +} + +export interface AlertHistoryItem { + id: string + organization_id: string + alert_id: string + triggered_at: string + triggered_value: number + threshold_value: number + status: AlertHistoryStatus + notified_at?: string | null + notification_details?: Record | null + acknowledged_at?: string | null + acknowledged_by?: string | null + resolved_at?: string | null + resolved_by?: string | null + resolution_notes?: string | null + context_data?: Record | null + created_at: string + updated_at: string + alert?: Alert +} + + +// Cron Job Types +export enum CronJobStatus { + ACTIVE = 'active', + PAUSED = 'paused', + COMPLETED = 'completed', +} + +export interface CronJob { + id: string + organization_id: string + name: string + cron_expression: string + timezone: string + max_runs: number + current_runs: number + evaluator_ids: string[] + status: CronJobStatus + next_run_at?: string | null + last_run_at?: string | null + created_at: string + updated_at: string + created_by?: string | null +} + +export interface CronJobCreate { + name: string + cron_expression: string + timezone: string + max_runs: number + evaluator_ids: string[] +} + +export interface CronJobUpdate { + name?: string + cron_expression?: string + timezone?: string + max_runs?: number + evaluator_ids?: string[] + status?: CronJobStatus +} + +// --- Call Imports --- + +/** + * Lifecycle for a call-import batch. + * + * - ``uploaded`` : file landed in S3, no mapping yet. + * - ``mapped`` : user picked a schema + sheet + column mapping; no + * rows materialised yet, no worker enqueued. + * - ``processing`` : rows materialised + workers enqueued. + * - ``pending`` : transient state used by the legacy one-shot + * ``POST /upload`` endpoint just before transitioning + * to ``processing``. + */ +export type CallImportStatus = + | 'pending' + | 'uploaded' + | 'mapped' + | 'processing' + | 'completed' + | 'partial' + | 'failed' + | 'deleting' + +export type CallImportRowStatus = + | 'pending' + | 'processing' + | 'completed' + | 'failed' + +/** Where the value in `transcript` came from. */ +export type CallImportTranscriptSource = + | 'csv' + | 'transcribed' + | 'edited' + | null +/** Lifecycle status for the post-hoc transcription workflow itself. */ +export type CallImportTranscriptStatus = + | 'idle' + | 'pending' + | 'running' + | 'completed' + | 'failed' + | null + +/** + * Which transcript an evaluation run scored against. + * - `production`: the CSV-supplied value on `CallImportRow.transcript`. + * - `diarised`: the worker-produced value on `CallImportRow.diarised_transcript`. + */ +export type CallImportEvaluationTranscriptSource = 'production' | 'diarised' + +/** + * One contiguous turn inside ``CallImportRow.diarised_segments``. + * + * The diarisation worker rewrites each pyannote ``Speaker N`` label + * into ``agent`` / ``user`` (first speaker = agent heuristic). Anything + * beyond two distinct speakers keeps a generic ``speaker_N`` label so + * multi-party recordings still render every voice. + */ +export interface CallImportDiarisedSegment { + speaker: string + text: string + start: number + end: number + /** Original pyannote label (``Speaker 1`` / ``Speaker 2`` / ...). */ + raw_speaker: string +} + +export interface CallImportRow { + id: string + row_index: number + /** Mandatory identifier per row. Renamed from ``external_call_id``. */ + conversation_id: string + recording_url: string | null + recording_date: string | null + /** Production transcript — the value supplied via the CSV upload. */ + transcript: string | null + /** Provenance of the stored production transcript (csv = CSV upload, edited = manual edit). */ + transcript_source: CallImportTranscriptSource + /** Legacy: provider recorded by the original transcription worker before the split. */ + transcript_provider: string | null + transcript_model: string | null + transcript_status: CallImportTranscriptStatus + transcript_error: string | null + transcribed_at: string | null + /** Diarised transcript — produced by the post-hoc diarisation worker. */ + diarised_transcript: string | null + /** Provider used by the diarisation worker (e.g. "deepgram"). */ + diarised_transcript_provider: string | null + diarised_transcript_model: string | null + diarised_transcript_status: CallImportTranscriptStatus + diarised_transcript_error: string | null + diarised_at: string | null + /** + * Structured speaker turns produced by the diarisation worker. Each + * entry is a single contiguous turn shaped as + * `{ speaker: 'agent' | 'user' | 'speaker_N', text, start, end, + * raw_speaker }`. ``diarised_transcript`` is a rendered + * `: ` view of this list with + * ``diarised_speaker_swap`` applied. ``null`` on legacy rows that + * were diarised before structured turns were persisted (or when the + * STT provider didn't surface segments). + */ + diarised_segments: CallImportDiarisedSegment[] | null + /** + * When ``true`` the ``agent`` <-> ``user`` mapping inside + * ``diarised_segments`` is inverted in the rendered transcript / + * CSV export. The worker writes the canonical mapping using a + * "first speaker is the agent" heuristic; reviewers can flip the + * toggle from the row detail panel without re-running diarisation. + */ + diarised_speaker_swap: boolean + /** + * LLM that turned the STT plain-text output into structured + * ``diarised_segments``. NULL on legacy rows (pre-LLM-diariser). + */ + diarised_llm_provider: string | null + diarised_llm_model: string | null + /** + * Exact prompt the LLM diariser ran with. Useful for the modal to + * pre-fill its textarea when the operator wants to iterate on a + * previously-diarised row. + */ + diarised_prompt: string | null + /** + * Diarisation pipeline that produced this row's turns. + * - `stt_llm` (default) — two-stage STT then LLM diariser. + * - `llm_only` — single-stage multimodal LLM (audio in). + * Read-only; written by the worker on each diarisation. + */ + transcribe_mode?: 'stt_llm' | 'llm_only' + /** + * Per-row preservation of the mapped source cells. Values land here + * as whatever type the schema parameter coerced them to — + * strings (text / url / conversation_id / recording_url / + * recording_date / transcript / datetime), numbers, booleans, or + * ``null`` for blanks. Always + * coerce with ``String(value)`` before string operations. + */ + raw_columns: Record | null + status: CallImportRowStatus + recording_s3_key: string | null + recording_content_type: string | null + recording_size_bytes: number | null + error_message: string | null + attempts: number + created_at: string + updated_at: string +} + +export interface CallImportTag { + id: string + name: string + color: string | null + created_at: string + updated_at: string +} + +/** + * Parameter type tag on a Call Import schema parameter. + * + * - ``conversation_id``: mandatory identifier (one per schema). + * - ``recording_url``: feeds ``CallImportRow.recording_url``. + * - ``recording_date``: date-only call recording date used for reports. + * - ``transcript``: feeds ``CallImportRow.transcript``. + * - ``text`` / ``number`` / ``boolean`` / ``datetime`` / ``url``: + * generic typed fields preserved per row in ``raw_columns`` and + * surfaced in the evaluation export under the parameter's name. + */ +export type CallImportSchemaParameterType = + | 'conversation_id' + | 'recording_url' + | 'recording_date' + | 'transcript' + | 'text' + | 'number' + | 'boolean' + | 'datetime' + | 'url' + +export interface CallImportSchemaParameter { + id?: string + name: string + type: CallImportSchemaParameterType + description: string | null + is_required: boolean + ordering?: number +} + +export interface CallImportSchema { + id: string + organization_id: string + workspace_id: string + name: string + description: string | null + parameters: CallImportSchemaParameter[] + /** How many CallImport batches reference this schema. */ + usage_count: number + created_at: string + updated_at: string +} + +export interface CallImportSchemaListResponse { + items: CallImportSchema[] + total: number +} + +export interface CallImportSchemaCreate { + name: string + description?: string | null + parameters: Array> +} + +export interface CallImportSchemaUpdate { + name?: string + description?: string | null + parameters?: Array> +} + +/** + * In-org Workspace - the active workspace scopes call imports and + * metrics in the UI. The org's Default workspace is auto-seeded by + * migration 033 and cannot be deleted. + */ +export interface Workspace { + id: string + organization_id: string + name: string + slug: string + is_default: boolean + created_at: string + updated_at: string + role_id?: string | null + role_name?: string | null + capabilities?: string[] +} + +export interface WorkspaceRole { + id: string + organization_id: string + name: string + description?: string | null + capabilities: string[] + is_system: boolean + created_at: string + updated_at: string +} + +export interface WorkspaceMember { + id: string + workspace_id: string + user_id: string + role_id: string + role_name: string + user_email: string + user_name?: string | null + added_by_user_id?: string | null + created_at: string +} + +export interface CapabilityInfo { + key: string + label: string +} + +export interface CapabilityDomain { + key: string + label: string + capabilities: CapabilityInfo[] +} + +export interface WorkspaceRoleCreate { + name: string + description?: string | null + capabilities: string[] +} + +export interface WorkspaceRoleUpdate { + name?: string + description?: string | null + capabilities?: string[] +} + +export interface CallImportSourceRowSkip { + source_row: number + reason: string + message: string +} + +export interface CallImport { + id: string + organization_id: string + /** Workspace this import belongs to. */ + workspace_id: string + /** + * Telephony provider key. ``null`` until the IMPORT stage in the + * staged flow (which is the first step that knows the provider). + * Always populated on post-import batches and on legacy one-shot + * uploads. + */ + provider: string | null + telephony_integration_id: string | null + original_filename: string | null + /** + * For Excel uploads, which worksheet this batch came from. ``null`` + * for CSV uploads (CSV files have no sheet concept) and for any + * imports created before multi-sheet support landed. + */ + sheet_name: string | null + /** Optional free-text dataset label (high-level segregation filter). */ + dataset: string | null + /** Tags currently attached to this import. Empty array if untagged. */ + tags: CallImportTag[] + /** + * Reusable Input Parameter schema the batch was uploaded against. + * NULL on legacy batches uploaded before the schema-driven flow shipped. + */ + schema_id: string | null + /** + * Schema-driven mapping: ``{parameter_name: csv_header}``. Empty on + * legacy batches; check ``column_mapping`` / ``extra_columns`` / + * ``custom_column_mapping`` instead for those. + */ + parameter_mapping: Record + /** Legacy free-form mapping kept for batches uploaded before schemas. */ + column_mapping: Record + /** Legacy extra-column list kept for backwards-compat. */ + extra_columns: string[] + /** Legacy uploader-named columns kept for backwards-compat. */ + custom_column_mapping: Record + /** + * Source headers the uploader explicitly skipped, captured at the + * MAP stage. Empty for legacy one-shot uploads where the value was + * ephemeral. + */ + skipped_columns: string[] + /** + * Source rows skipped at parse time (missing/invalid conversation ID or URL). + */ + source_row_skips?: CallImportSourceRowSkip[] + /** S3 key for the staged source file. ``null`` on legacy batches. */ + source_s3_key: string | null + /** ``'csv'`` / ``'xlsx'`` for staged files, or ``'audio'`` for manual uploads. */ + source_format: string | null + source_size_bytes: number | null + source_content_type: string | null + /** + * Snapshot of the file's sheets + headers captured at UPLOAD time so + * the MAP UI can render without re-fetching the source from S3. + * ``null`` on legacy batches. + */ + available_sheets: CallImportPreviewSheet[] | null + total_rows: number + completed_rows: number + failed_rows: number + status: CallImportStatus + error_message: string | null + created_at: string + updated_at: string + created_by_email?: string | null + last_updated_by_email?: string | null +} + +export interface CallImportDetail extends CallImport { + rows: CallImportRow[] + /** + * Total row count *after* applying the optional ``q`` search filter. + * ``null`` when no filter is active — paginate against ``total_rows`` + * in that case. + */ + filtered_total_rows: number | null + /** + * Batch-wide aggregates of ``CallImportRow.diarised_transcript_status``. + * The ``idle`` bucket (rows never touched by the transcribe/diarise + * worker) is implicit: ``total_rows - (pending + running + completed + * + failed)``. Lets the UI render a transcribe-and-diarise progress + * bar without paginating through every row. + */ + diarised_pending_rows: number + diarised_running_rows: number + diarised_completed_rows: number + diarised_failed_rows: number +} + +export interface CallImportListResponse { + items: CallImport[] + total: number + page: number + page_size: number +} + +export interface CallImportUploadResponse { + id: string + total_rows: number + status: CallImportStatus + dataset: string | null + tags: CallImportTag[] + message: string +} + +/** One worksheet (or one CSV file synthesized as a single sheet). */ +export interface CallImportPreviewSheet { + /** Sheet name for xlsx; filename for csv. */ + name: string + /** Column headers from the first non-empty row. */ + headers: string[] + /** Approximate count of data rows (excludes the header row). */ + row_count: number +} + +/** + * Sheets / headers extracted server-side from an uploaded CSV or Excel + * workbook. Drives the modal's column-mapping UI without forcing the + * frontend to parse the file itself. + */ +export interface CallImportPreviewResponse { + /** ``'csv'`` or ``'xlsx'``. */ + format: 'csv' | 'xlsx' + sheets: CallImportPreviewSheet[] +} + +export type MetricSelectionMode = 'single_choice' | 'multi_label' + +export interface CallImportMetricSummary { + id: string + name: string + metric_type: string | null + description: string | null + parent_metric_id?: string | null + selection_mode?: MetricSelectionMode | null + /** Only meaningful on multi_label parents; gates the Discovered + * Labels panel on the Flow tab. Defaults to false. */ + allow_discovery?: boolean +} + +/** Per-metric LLM override (provider+model+optional credential + generation params). */ +export interface CallImportEvaluationLLMOverride { + provider?: string | null + model?: string | null + credential_id?: string | null + llm_config?: LLMGenerationConfig | null +} + +export interface CallImportEvaluation { + id: string + call_import_id: string + organization_id: string + /** User-supplied label for the run; null when not named. */ + name: string | null + selected_metric_ids: string[] + /** parent_id -> [child_id, ...] snapshot captured at run time. */ + selected_metric_groups?: Record | null + metrics: CallImportMetricSummary[] + status: 'pending' | 'running' | 'completed' | 'partial' | 'failed' + total_rows: number + completed_rows: number + failed_rows: number + error_message: string | null + /** Run-level LLM provider chosen by the user (null = legacy default). */ + llm_provider: string | null + llm_model: string | null + llm_credential_id: string | null + llm_config?: LLMGenerationConfig | null + metric_llm_overrides: Record | null + stt_provider: string | null + stt_model: string | null + stt_credential_id: string | null + /** + * Run-level LLM diariser config used when the worker auto-diarises + * rows that are missing a diarised transcript. + */ + diarisation_llm_provider?: string | null + diarisation_llm_model?: string | null + diarisation_llm_credential_id?: string | null + diarisation_prompt?: string | null + /** + * Diarisation pipeline shape this run was created with. + * - `stt_llm` (default) — STT then an LLM diariser over the text. + * - `llm_only` — audio fed directly to a multimodal diariser LLM. + * Surfaced so the retry / re-run UI can preselect the right mode. + */ + transcribe_mode?: 'stt_llm' | 'llm_only' + /** + * Which transcript column this run scored against. + * Defaults to `production` on legacy runs. + */ + transcript_source: CallImportEvaluationTranscriptSource + /** + * Other evaluation ids created in the same Run Evaluation request. + * Populated only on the POST response when the user ticked both + * Production and Diarised. Empty array on all other reads. + */ + sibling_evaluation_ids: string[] + started_at: string | null + finished_at: string | null + created_at: string + updated_at: string + created_by_email?: string | null + last_updated_by_email?: string | null + /** + * Cached LLM-generated TLDR rendered above the Visualizations tab. + * Populated lazily via ``POST /evaluations/{id}/insights``; null on + * runs the user has not summarised yet. + */ + tldr_summary?: EvaluationTldrSummary | null + user_insights?: EvaluationUserInsightsState | null + metric_clusters?: EvaluationMetricClustersState | null + /** + * True when the user opted into top-level metric discovery on the + * Run Evaluation modal. Gates the Discovered metrics panel on the + * evaluation detail Flow tab. + */ + discover_new_metrics?: boolean + /** + * Set while a bulk background operation (abort, force-fail, retry) is + * still running. The UI disables other mutating actions until cleared. + */ + bulk_operation?: 'abort' | 'force_fail_pending' | 'retry' | null +} + +/** + * LLM-generated narrative + bullet patterns for a single evaluation + * run. Cached on the evaluation row so re-opening the Visualizations + * tab doesn't auto-burn LLM tokens. ``is_stale`` is computed by the + * backend at read time when ``completed_rows`` has grown since the + * summary was generated. + */ +export interface EvaluationTldrSummary { + narrative: string + patterns: string[] + metric_insights?: Record + generated_at: string + generated_at_completed_rows: number + provider?: string | null + model?: string | null + is_stale: boolean +} + +export interface UserInsightCategory { + label: string + count: number + share_pct: number +} + +export interface UserInsightEvidenceTurn { + speaker: string + text: string +} + +export interface UserInsightEvidence { + conversation_id?: string | null + quote: string + turns?: UserInsightEvidenceTurn[] +} + +export interface EvaluationUserInsightItem { + id: string + title: string + categories: UserInsightCategory[] + observation: string + evidence: UserInsightEvidence +} + +export interface EvaluationUserInsightsState { + status: 'idle' | 'running' | 'completed' | 'failed' + insights: EvaluationUserInsightItem[] + overview?: string | null + generated_at?: string | null + generated_at_completed_rows: number + progress?: { completed_llm_calls: number; total_llm_calls: number } | null + provider?: string | null + model?: string | null + llm_calls_used: number + max_llm_calls?: number | null + error_message?: string | null + is_stale: boolean +} + +export type MetricClusterGapLabel = + | 'LOGIC_GAP' + | 'UNDERSPEC' + | 'EXISTS_NO_TRIGGER' + | 'MISSING' + +export interface MetricSubCluster { + label: string + count: number + share_pct: number +} + +export interface MetricClusterEvidenceTurn { + speaker: string + text: string +} + +export interface MetricClusterEvidence { + conversation_id?: string | null + evaluation_row_id?: string | null + quote: string + turns?: MetricClusterEvidenceTurn[] +} + +export interface MetricCluster { + id: string + label: string + gap_label: MetricClusterGapLabel + level: number + count: number + share_pct: number + sub_clusters: MetricSubCluster[] + observation: string + failure_reason?: string + evidence: MetricClusterEvidence + is_discovered: boolean +} + +export interface MetricClusterGroup { + metric_id: string + metric_name: string + flagged_count: number + failure_reason?: string + clusters: MetricCluster[] +} + +export interface DiscoveredProblemCluster { + id: string + label: string + gap_label: MetricClusterGapLabel + count: number + share_pct: number + observation: string + failure_reason?: string + evidence: MetricClusterEvidence +} + +export interface RcaRepeatedPatternRow { + metric_id: string + metric_name: string + top_rca_patterns: string + evidence_share_pct: number + evidence_calls: number + evidence_cluster_count?: number + failure_reason: string +} + +export interface RcaMetricHotspotRow { + metric_id: string + metric_name: string + description: string + metric_rate_pct: number + flagged_calls: number +} + +export interface RcaPromptAreaRow { + label: string + share_pct: number + gap_label: MetricClusterGapLabel +} + +export interface MetricClustersRcaSummary { + total_clusters: number + total_clustered_instances: number + total_flagged_instances?: number + analysed_calls: number + repeated_patterns: RcaRepeatedPatternRow[] + metric_hotspots: RcaMetricHotspotRow[] + prompt_areas: RcaPromptAreaRow[] +} + +export interface MetricFailurePolicy { + metric_id: string + failure_values: string[] + failure_child_names?: string[] + numeric_rule?: { op: 'lt' | 'lte' | 'gt' | 'gte'; threshold: number } | null +} + +export interface MetricFailurePolicyValueCount { + label: string + count: number +} + +export interface MetricFailurePolicyMetricPreview { + metric_id: string + metric_name: string + metric_type?: string | null + selection_mode?: string | null + is_multi_label_parent: boolean + value_counts: MetricFailurePolicyValueCount[] + child_names: string[] + row_count_by_value: Record + suggested_policy: MetricFailurePolicy + effective_policy: MetricFailurePolicy +} + +export interface MetricFailurePoliciesResponse { + previews: MetricFailurePolicyMetricPreview[] + policies: Record + source: 'inferred' | 'user' + updated_at?: string | null +} + +export interface MetricClusterEligibleRow { + evaluation_row_id: string + conversation_id?: string | null + row_index?: number | null + flagged_metric_names: string[] +} + +export interface MetricClusterEligibleRowsResponse { + items: MetricClusterEligibleRow[] + total: number +} + +export interface EvaluationMetricClustersState { + status: 'idle' | 'running' | 'completed' | 'failed' | 'cancelled' + groups: MetricClusterGroup[] + discovered_problems: DiscoveredProblemCluster[] + overview?: string | null + generated_at?: string | null + generated_at_completed_rows: number + progress?: { completed_llm_calls: number; total_llm_calls: number } | null + provider?: string | null + model?: string | null + llm_calls_used: number + max_llm_calls?: number | null + error_message?: string | null + is_stale: boolean + selected_evaluation_row_ids?: string[] + failure_policies?: Record + failure_policies_source?: 'inferred' | 'user' + failure_policies_updated_at?: string | null + rca_summary?: MetricClustersRcaSummary | null +} + +export interface AgentFlowNode { + id: string + label: string + node_type: 'start' | 'decision' | 'action' | 'terminal' + position_x?: number | null + position_y?: number | null + prompt_excerpt?: string | null + start_offset?: number | null + end_offset?: number | null +} + +export interface AgentFlowEdge { + source: string + target: string + condition?: string | null +} + +export interface AgentFlowGraph { + nodes: AgentFlowNode[] + edges: AgentFlowEdge[] + generated_at?: string | null + provider?: string | null + model?: string | null + layout_saved_at?: string | null + prompt_content_hash?: string | null + mapping_error?: string | null + generation_error?: string | null +} + +export interface ImportedAgent { + id: string + organization_id: string + name: string + description: string | null + content: string + tags: string[] | null + current_version: number + agent_flowchart?: AgentFlowGraph | null + agent_flowchart_status?: string | null + created_at: string + updated_at: string + created_by: string | null +} + +export interface ImportedAgentDetail extends ImportedAgent { + versions: PromptPartialVersion[] +} + +export interface MetricPartialChild { + name: string + description: string + example: string +} + +export interface MetricPartialContent { + schema_version: 1 + metric_kind: 'single' | 'category' + description: string + children?: MetricPartialChild[] +} + +export interface MetricPartial { + id: string + organization_id: string + name: string + description: string | null + content: string + tags: string[] | null + current_version: number + created_at: string + updated_at: string + created_by: string | null +} + +export interface MetricPartialDetail extends MetricPartial { + versions: PromptPartialVersion[] +} + +export interface PromptPartialVersion { + id: string + prompt_partial_id: string + version: number + content: string + change_summary: string | null + created_at: string + created_by: string | null +} + +export interface PromptImprovementSuggestion { + id: string + metric_id: string + metric_name: string + cluster_id: string + cluster_label: string + gap_label: MetricClusterGapLabel + share_pct: number + priority: 'high' | 'medium' | 'low' + change_type?: 'edit' | 'add' + target_section: string + anchor_excerpt?: string + current_gap: string + suggested_text: string + rationale: string + flow_node_id?: string + flow_node_label?: string +} + +export interface EvaluationPromptImprovementsState { + status: 'idle' | 'running' | 'completed' | 'failed' + imported_agent_id?: string | null + imported_agent_name?: string | null + suggestions: PromptImprovementSuggestion[] + overview?: string | null + generated_at?: string | null + generated_at_completed_rows: number + provider?: string | null + model?: string | null + error_message?: string | null + is_stale: boolean +} + +export interface MetricPeriodDelta { + label: string + detail: string + why?: string | null +} + +export interface CallImportEvaluationListResponse { + items: CallImportEvaluation[] + total: number +} + +export interface CallImportEvaluationBaselineCandidate { + evaluation_id: string + name: string + dataset: string + period_label: string | null + period_start: string | null + period_end: string | null + period_display: string + completed_rows: number + created_at: string + is_default: boolean +} + +export interface CallImportEvaluationBaselineCandidatesResponse { + items: CallImportEvaluationBaselineCandidate[] + default_evaluation_id: string | null +} + +export interface CallImportEvaluationPdfReport { + id: string + filename: string + preview_url?: string | null + download_url?: string | null + created_at: string + created_by?: string | null + report_type: string + vendor_name: string + config_summary?: string | null + storage_available?: boolean + cache_hit?: boolean +} + +export interface CallImportEvaluationPdfReportListItem { + id: string + filename?: string | null + vendor_name: string + report_type: string + created_by?: string | null + created_at: string + config_summary?: string | null + cache_fingerprint?: string | null +} + +export interface CallImportEvaluationPdfReportListResponse { + items: CallImportEvaluationPdfReportListItem[] +} + +export interface CallImportEvaluationRow { + id: string + evaluation_id: string + call_import_row_id: string + row_index: number | null + /** Mandatory identifier from the source batch (renamed from ``external_call_id``). */ + conversation_id: string | null + transcript: string | null + raw_columns: Record | null + recording_url: string | null + recording_date: string | null + /** + * S3 object key for the downloaded recording. Prefer this over + * ``recording_url`` for playback — we resolve it to a presigned URL + * so audio plays from our storage instead of the (often expired) + * provider URL. + */ + recording_s3_key: string | null + diarised_transcript_status?: string | null + diarised_transcript_error?: string | null + status: 'pending' | 'running' | 'completed' | 'failed' | 'skipped' + metric_scores: Record + error_message: string | null + started_at: string | null + finished_at: string | null + created_at: string + updated_at: string +} + +export interface CallImportEvaluationRowListResponse { + items: CallImportEvaluationRow[] + total: number + page: number + page_size: number +} + +// --- Retry (re-enqueue failed rows on an existing evaluation run) --- + +export interface CallImportEvaluationRetryRequest { + /** + * Restrict the retry to a specific subset of evaluation rows. + * When omitted, every row with status='failed' in this run is + * re-enqueued. + */ + eval_row_ids?: string[] + + /** + * Optional LLM overrides. When provided, persisted onto the run so + * future retries default to the new config. ``llm_provider`` and + * ``llm_model`` must be sent together — the backend 400s on + * half-configured input. + */ + llm_provider?: string + llm_model?: string + llm_credential_id?: string | null + + /** + * Optional STT overrides (only meaningful when the run scores the + * diarised transcript). Same paired-field rule as LLM. + */ + stt_provider?: string + stt_model?: string + stt_credential_id?: string | null + + /** + * When true, wipe the diarised transcript on every retried row so + * the (possibly new) STT runs from scratch. Only takes effect for + * diarised runs that have STT config. + */ + transcribe_overwrite?: boolean +} + +export interface CallImportEvaluationRetrySkippedItem { + eval_row_id: string + /** + * Why this row was not re-enqueued. Known values: + * - 'unknown' (id not in this run) + * - 'in_progress' (status is pending/running) + * - 'completed' (already successful) + * - 'source_row_missing' + */ + reason: 'unknown' | 'in_progress' | 'completed' | 'source_row_missing' +} + +export interface CallImportEvaluationRetryResponse { + requeued: number + /** + * Of those, how many were chained through a diarisation task first + * because the diarised transcript was missing. + */ + transcribe_requeued: number + skipped: CallImportEvaluationRetrySkippedItem[] +} + +export interface CallImportEvaluationBulkActionResponse { + accepted: boolean + target_count: number + evaluation_id: string +} + +// --- Diarization / transcription --- + +export interface CallImportTranscribeRequest { + /** + * Diarisation pipeline shape. + * - `stt_llm` (default) — STT produces plain text, then an LLM + * diariser splits it into agent/user turns. STT fields required. + * - `llm_only` — skip STT entirely and feed the audio bytes + * directly to a multimodal `diarization_llm_*` model along with + * `diarization_prompt`. STT fields MUST be omitted in this mode. + */ + mode?: 'stt_llm' | 'llm_only' + /** Required when `mode === 'stt_llm'`; must be null/omitted in `llm_only`. */ + stt_provider?: string | null + /** Required when `mode === 'stt_llm'`; must be null/omitted in `llm_only`. */ + stt_model?: string | null + credential_id?: string | null + language?: string | null + only_missing?: boolean + overwrite_existing?: boolean + row_ids?: string[] + /** + * LLM diariser. In `stt_llm` mode it splits the STT plain-text into + * agent/user turns; in `llm_only` mode it directly receives the + * audio along with `diarization_prompt`. Always required. + */ + diarization_llm_provider: string + diarization_llm_model: string + diarization_llm_credential_id?: string | null + /** + * Operator-supplied system prompt for the diariser LLM. NULL/empty + * means "fall back to the canonical default" (see + * ``getDiarisationDefaultPrompt``). + */ + diarization_prompt?: string | null +} + +export interface CallImportTranscribeResponse { + queued: number + skipped_rows: number + skipped_reason_counts: Record + accepted?: boolean +} + +export interface CallImportRowBulkDeleteResponse { + deleted: number + status?: 'completed' | 'accepted' +} + +export interface CallImportRetryFailedRowsResponse { + requeued: number + enqueue_failed: number + skipped: number +} + +export interface CallImportDiarisationPromptDefaultResponse { + prompt: string +} + +// --- Aggregation / visualization payloads --- + +export interface CallImportMetricHistogramBucket { + x0: number + x1: number + count: number +} + +export interface CallImportMetricValueCount { + label: string + count: number +} + +/** + * One unordered pair-count cell from a multi-label parent's + * co-occurrence matrix. ``a`` and ``b`` are child label names with + * ``a < b`` lexicographically; ``count`` is the number of rows on + * which both labels fired together. + */ +export interface CallImportMetricLabelPair { + a: string + b: string + count: number +} + +export interface CallImportMetricAggregate { + metric_id: string + metric_name: string + metric_type: string | null + metric_category?: 'quality' | 'user_insight' | string + /** + * True when the metric is a multi-label classifier parent. + * ``value_counts`` then lists per-child label tallies and one row + * may contribute to several labels, so the chart layout has to + * ignore the pie toggle (slices wouldn't sum to 100%) and the + * n-badge represents rows scored, not label occurrences. + */ + is_multi_label_parent?: boolean + count: number + skipped_count: number + error_count: number + mean: number | null + median: number | null + p25: number | null + p75: number | null + p95: number | null + min: number | null + max: number | null + stddev: number | null + histogram_buckets: CallImportMetricHistogramBucket[] + value_counts: CallImportMetricValueCount[] + /** + * Pairwise label intersections for multi-label parent metrics. + * Empty for everything else. The Visualizations tab reconstructs + * a square symmetric matrix from these unordered pairs to render + * the co-occurrence heatmap chart type. + */ + co_occurrence?: CallImportMetricLabelPair[] +} + +export interface CallImportEvaluationAggregateResponse { + evaluation_id: string + total_rows: number + completed_rows: number + failed_rows: number + metrics: CallImportMetricAggregate[] + period_deltas?: Record + baseline_evaluation_id?: string | null + failure_policies_source?: 'inferred' | 'user' | null +} + +export interface EvaluatorResultsAggregateResponse { + scope: string + suite_id?: string | null + agent_id?: string | null + scenario_id?: string | null + total_rows: number + completed_rows: number + failed_rows: number + metrics: CallImportMetricAggregate[] +} + +export interface CallImportInsightsRunPoint { + evaluation_id: string + name: string | null + created_at: string + mean: number | null + completed_rows: number +} + +export interface CallImportInsightsMetric { + metric_id: string + metric_name: string + metric_type: string | null + latest: CallImportMetricAggregate | null + trend: CallImportInsightsRunPoint[] +} + +export interface CallImportInsightsResponse { + call_import_id: string + total_rows: number + rows_with_transcript: number + rows_without_transcript: number + transcript_source_counts: Record + evaluation_count: number + metrics: CallImportInsightsMetric[] +} + +// --- Metrics hierarchy + flow visualization --- + +export interface MetricSummary { + id: string + organization_id: string + name: string + description: string | null + metric_type: string + metric_category?: 'quality' | 'user_insight' | string + trigger: string + enabled: boolean + is_default: boolean + metric_origin: string + supported_surfaces: string[] + enabled_surfaces: string[] + custom_data_type: string | null + custom_config: Record | null + tags: string[] | null + capture_rationale: boolean + parent_metric_id: string | null + selection_mode: MetricSelectionMode | null + allow_discovery?: boolean + /** + * When true, this metric is a "transcript-compare judge": at + * call-import evaluation time the worker feeds BOTH the production + * transcript and the diarised transcript to the LLM as a labeled + * pair, and the run's transcript_source toggle is ignored for this + * metric. Mutually exclusive with parent_metric_id and selection_mode + * — comparison metrics stay standalone. + */ + compare_transcripts?: boolean + children?: MetricSummary[] + created_at: string + updated_at: string + created_by: string | null +} + +export interface MetricChildDraft { + name: string + description?: string | null + enabled?: boolean + capture_rationale?: boolean | null + tags?: string[] | null +} + +export interface MetricCreateWithChildrenPayload { + name: string + description?: string | null + selection_mode: MetricSelectionMode + enabled?: boolean + supported_surfaces?: string[] + enabled_surfaces?: string[] + tags?: string[] | null + allow_discovery?: boolean + children: MetricChildDraft[] +} + +export interface MetricFlowNode { + id: string + label: string + count: number + is_terminal: boolean + is_discovered?: boolean +} + +export interface MetricFlowEdge { + source: string + target: string + count: number +} + +export interface MetricFlowResponse { + parent_metric_id: string + parent_metric_name: string + selection_mode: MetricSelectionMode | null + nodes: MetricFlowNode[] + edges: MetricFlowEdge[] + total_rows: number + rows_with_sequence: number +} + +export interface DiscoveredLabel { + key: string + name: string + description?: string | null + sample_rationale?: string | null + /** + * Up to 3 distinct LLM rationales captured for this candidate + * across rows. The Discovered Labels promote flow surfaces the + * first 2 as an ``Examples:`` block on the new sub-metric's + * rubric so the user starts with concrete cases in the prompt. + */ + examples?: string[] + count: number +} + +export interface DiscoveredLabelsResponse { + parent_metric_id: string + items: DiscoveredLabel[] +} + +/** + * One LLM-discovered candidate TOP-LEVEL metric aggregated across all + * rows of an evaluation. Mirrors :class:`DiscoveredLabel` but adds a + * ``suggested_type`` field — the LLM's guess at the best shape for + * the new metric — that the promote modal can pre-fill the type radio + * with. + */ +export interface DiscoveredMetric { + key: string + name: string + description?: string | null + suggested_type: 'boolean' | 'rating' | 'category' + sample_rationale?: string | null + examples?: string[] + count: number +} + +export interface DiscoveredMetricsResponse { + evaluation_id: string + items: DiscoveredMetric[] +} + +export interface ObservabilityCallAgent { + id: string + agent_id?: string | null + name: string +} + +export interface ObservabilityCallData { + startedAt?: string + started_at?: string + endedAt?: string + ended_at?: string + from_phone_number?: string + to_phone_number?: string + endedReason?: string + recording_s3_key?: string + recording_url?: string + duration_seconds?: number + agent_name?: string + _agent_ref?: string | number + direction?: string + messages?: Array<{ role: string; content: string; start_time?: number; end_time?: number }> + live_transcript?: Array<{ role: string; content: string; timestamp?: string; start_time?: number }> + metadata?: Record + call_short_id?: string +} + +export interface ObservabilityCall { + id: string + call_short_id: string + status?: string | null + call_event?: string | null + is_live?: boolean + direction?: string | null + source?: string | null + provider_platform?: string | null + provider_call_id?: string | null + agent_id?: string | null + agent?: ObservabilityCallAgent | null + created_at?: string | null + updated_at?: string | null + call_data?: ObservabilityCallData | null + live_transcript?: Array<{ role: string; content: string; timestamp?: string }> +} diff --git a/tests/test_api/test_call_import_audit.py b/tests/test_api/test_call_import_audit.py new file mode 100644 index 00000000..60592ff0 --- /dev/null +++ b/tests/test_api/test_call_import_audit.py @@ -0,0 +1,78 @@ +"""Audit fields (created_by / last_updated_by email) on call imports.""" + +from uuid import uuid4 + +from app.models.database import CallImport, Workspace +from app.models.enums import CallImportStatus + + +def _ensure_default_workspace(db_session, org_id): + ws = ( + db_session.query(Workspace) + .filter(Workspace.organization_id == org_id, Workspace.is_default.is_(True)) + .first() + ) + if ws is None: + ws = Workspace( + organization_id=org_id, name="Default", slug="default", is_default=True + ) + db_session.add(ws) + db_session.commit() + return ws + + +def test_update_call_import_metadata_stamps_actor_emails( + authenticated_client, db_session, org_id, seed_org +): + workspace = _ensure_default_workspace(db_session, org_id) + call_import = CallImport( + id=uuid4(), + organization_id=org_id, + workspace_id=workspace.id, + provider="exotel", + original_filename="batch.csv", + total_rows=0, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.COMPLETED, + dataset="before", + ) + db_session.add(call_import) + db_session.commit() + + response = authenticated_client.patch( + f"/api/v1/call-imports/{call_import.id}", + json={"dataset": "after"}, + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["dataset"] == "after" + assert body["created_by_email"] is None + assert body["last_updated_by_email"] == "owner@example.com" + + +def test_list_call_imports_includes_actor_emails( + authenticated_client, db_session, org_id, seed_org +): + workspace = _ensure_default_workspace(db_session, org_id) + call_import = CallImport( + id=uuid4(), + organization_id=org_id, + workspace_id=workspace.id, + provider=None, + original_filename="listed.csv", + total_rows=0, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.UPLOADED, + ) + db_session.add(call_import) + db_session.commit() + + listing = authenticated_client.get("/api/v1/call-imports") + assert listing.status_code == 200, listing.text + items = listing.json()["items"] + match = [item for item in items if item["id"] == str(call_import.id)] + assert len(match) == 1 + assert match[0]["created_by_email"] is None + assert match[0]["last_updated_by_email"] is None diff --git a/tests/test_api/test_call_import_evaluation_pdf_report.py b/tests/test_api/test_call_import_evaluation_pdf_report.py index 9b2f316b..d24bd739 100644 --- a/tests/test_api/test_call_import_evaluation_pdf_report.py +++ b/tests/test_api/test_call_import_evaluation_pdf_report.py @@ -516,6 +516,7 @@ def test_pdf_report_uses_saved_report_branding_logo( fake_s3 = SimpleNamespace( download_file_by_key=lambda key: b"custom-logo", + is_enabled=lambda: False, ) monkeypatch.setattr(s3_module, "s3_service", fake_s3) captured: dict[str, object] = {} @@ -1496,3 +1497,134 @@ def _override_db(): assert response.status_code == 403 assert "Editor role" in response.json()["detail"] assert "Viewer" in response.json()["detail"] + + +def _enable_mock_blob_storage(monkeypatch): + from app.services.storage.blob_storage_service import blob_storage_service + + stored: dict[str, bytes] = {} + + def upload_file_by_key(file_content, key, content_type="audio/mpeg"): + stored[key] = file_content + return key + + monkeypatch.setattr(blob_storage_service._s3, "is_enabled", lambda: True) + monkeypatch.setattr(blob_storage_service, "upload_file_by_key", upload_file_by_key) + monkeypatch.setattr( + blob_storage_service, + "generate_presigned_url_by_key", + lambda key, expiration=3600, **kwargs: f"https://storage.example/{key}", + ) + return stored + + +def test_pdf_report_returns_json_and_stores_when_storage_enabled( + authenticated_client, db_session, org_id, seed_org, monkeypatch +): + from app.models.database import CallImportEvaluationPdfReport + + monkeypatch.setattr( + call_import_evaluation_pdf_report_service, + "_render_weasyprint", + lambda _html, **_kwargs: None, + ) + stored = _enable_mock_blob_storage(monkeypatch) + call_import, evaluation = _seed_completed_evaluation(db_session, org_id) + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations/{evaluation.id}/pdf-report", + json={"vendor_name": "Acme Vendor", "report_type": "external"}, + ) + + assert response.status_code == 200, response.text + body = response.json() + assert body["vendor_name"] == "Acme Vendor" + assert body["report_type"] == "external" + assert body["preview_url"].startswith("https://storage.example/") + assert body["download_url"].startswith("https://storage.example/") + + row = ( + db_session.query(CallImportEvaluationPdfReport) + .filter(CallImportEvaluationPdfReport.evaluation_id == evaluation.id) + .one() + ) + assert row.s3_key + assert row.s3_key in stored + assert stored[row.s3_key].startswith(b"%PDF") + + +def test_pdf_report_reuses_cached_report_when_unchanged( + authenticated_client, db_session, org_id, seed_org, monkeypatch +): + render_calls = {"count": 0} + + def counting_render(*_args, **_kwargs): + render_calls["count"] += 1 + return None + + monkeypatch.setattr( + call_import_evaluation_pdf_report_service, + "_render_weasyprint", + counting_render, + ) + stored = _enable_mock_blob_storage(monkeypatch) + call_import, evaluation = _seed_completed_evaluation(db_session, org_id) + base = f"/api/v1/call-imports/{call_import.id}/evaluations/{evaluation.id}" + payload = {"vendor_name": "Acme Vendor", "report_type": "external"} + + first = authenticated_client.post(f"{base}/pdf-report", json=payload) + second = authenticated_client.post(f"{base}/pdf-report", json=payload) + + assert first.status_code == 200, first.text + assert second.status_code == 200, second.text + assert first.json()["id"] == second.json()["id"] + assert second.json().get("cache_hit") is True + assert render_calls["count"] == 1 + assert len(stored) == 1 + + +def test_pdf_report_list_and_get_stored_versions( + authenticated_client, db_session, org_id, seed_org, monkeypatch +): + monkeypatch.setattr( + call_import_evaluation_pdf_report_service, + "_render_weasyprint", + lambda _html, **_kwargs: None, + ) + _enable_mock_blob_storage(monkeypatch) + call_import, evaluation = _seed_completed_evaluation(db_session, org_id) + base = f"/api/v1/call-imports/{call_import.id}/evaluations/{evaluation.id}" + + first = authenticated_client.post( + f"{base}/pdf-report", + json={ + "vendor_name": "Acme Vendor", + "report_type": "external", + "report_config": {"quality_metric_ids": ["a"]}, + }, + ) + second = authenticated_client.post( + f"{base}/pdf-report", + json={ + "vendor_name": "Acme Vendor", + "report_type": "internal", + "report_config": {"quality_metric_ids": ["b"]}, + }, + ) + assert first.status_code == 200 + assert second.status_code == 200 + first_id = first.json()["id"] + second_id = second.json()["id"] + assert first_id != second_id + + listed = authenticated_client.get(f"{base}/pdf-reports") + assert listed.status_code == 200 + items = listed.json()["items"] + assert len(items) == 2 + listed_ids = {item["id"] for item in items} + assert listed_ids == {first_id, second_id} + + detail = authenticated_client.get(f"{base}/pdf-reports/{first_id}") + assert detail.status_code == 200 + assert detail.json()["id"] == first_id + assert detail.json()["preview_url"].startswith("https://storage.example/") diff --git a/tests/test_api/test_call_import_evaluation_prompt_improvements.py b/tests/test_api/test_call_import_evaluation_prompt_improvements.py new file mode 100644 index 00000000..ecce8a00 --- /dev/null +++ b/tests/test_api/test_call_import_evaluation_prompt_improvements.py @@ -0,0 +1,66 @@ +"""API tests for evaluation prompt-improvements enqueue and audit stamping.""" + +from __future__ import annotations + +import types + +from app.models.database import PromptPartial +from app.services.imported_agent_constants import IMPORTED_AGENT_TAG +from tests.test_api.test_call_import_evaluation_insights import _seed_eval_with_data + + +def test_post_prompt_improvements_stamps_last_updated_by_email( + authenticated_client, + db_session, + org_id, + seed_org, + make_ai_provider, + monkeypatch, +): + make_ai_provider(provider="openai", is_active=True) + call_import, evaluation, _ = _seed_eval_with_data(db_session, org_id) + + evaluation.metric_clusters = { + "status": "completed", + "groups": [], + "overview": "done", + "generated_at": "2026-01-01T00:00:00+00:00", + "generated_at_completed_rows": 1, + } + evaluation.last_updated_by_user_id = None + db_session.flush() + + agent = PromptPartial( + organization_id=org_id, + workspace_id=call_import.workspace_id, + name="Imported agent", + content="You are a helpful agent.", + tags=[IMPORTED_AGENT_TAG], + ) + db_session.add(agent) + db_session.commit() + + def fake_apply_async(*, kwargs=None, **_kw): + return types.SimpleNamespace(id="prompt-improvements-task-1") + + monkeypatch.setattr( + "app.workers.tasks.generate_evaluation_prompt_improvements.generate_evaluation_prompt_improvements_task.apply_async", + fake_apply_async, + ) + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations/{evaluation.id}/prompt-improvements", + json={ + "imported_agent_id": str(agent.id), + "regenerate": True, + "force": True, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["status"] == "running" + + detail = authenticated_client.get( + f"/api/v1/call-imports/{call_import.id}/evaluations/{evaluation.id}" + ) + assert detail.status_code == 200, detail.text + assert detail.json()["last_updated_by_email"] == "owner@example.com" diff --git a/tests/test_api/test_call_import_evaluation_serialize_progress.py b/tests/test_api/test_call_import_evaluation_serialize_progress.py index f6c42aad..430dd244 100644 --- a/tests/test_api/test_call_import_evaluation_serialize_progress.py +++ b/tests/test_api/test_call_import_evaluation_serialize_progress.py @@ -40,6 +40,8 @@ def test_serialize_eval_merges_redis_deltas_without_clearing(db_session, org_id, "finished_at": None, "created_at": now, "updated_at": now, + "created_by_user_id": None, + "last_updated_by_user_id": None, }, )() diff --git a/tests/test_api/test_call_import_evaluation_user_insights.py b/tests/test_api/test_call_import_evaluation_user_insights.py index 1fd76744..4d9a1336 100644 --- a/tests/test_api/test_call_import_evaluation_user_insights.py +++ b/tests/test_api/test_call_import_evaluation_user_insights.py @@ -191,6 +191,32 @@ def test_post_user_insights_enqueues_task( assert response.json()["max_llm_calls"] == 100 +def test_post_user_insights_stamps_last_updated_by_email( + authenticated_client, + db_session, + org_id, + seed_org, + make_ai_provider, + stub_user_insights_worker, +): + make_ai_provider(provider="openai", is_active=True) + call_import, evaluation = _seed_evaluation(db_session, org_id) + evaluation.last_updated_by_user_id = None + db_session.commit() + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations/{evaluation.id}/user-insights", + json={"regenerate": True, "force": True}, + ) + assert response.status_code == 200, response.text + + detail = authenticated_client.get( + f"/api/v1/call-imports/{call_import.id}/evaluations/{evaluation.id}" + ) + assert detail.status_code == 200, detail.text + assert detail.json()["last_updated_by_email"] == "owner@example.com" + + def test_post_user_insights_requires_completed_rows( authenticated_client, db_session, org_id, seed_org ): diff --git a/tests/test_api/test_call_import_evaluations.py b/tests/test_api/test_call_import_evaluations.py index faf3ff9d..43d9230b 100644 --- a/tests/test_api/test_call_import_evaluations.py +++ b/tests/test_api/test_call_import_evaluations.py @@ -1152,3 +1152,43 @@ def test_evaluation_retry_can_override_telephony_credentials( db_session.refresh(call_import) assert call_import.telephony_integration_id == right_integration.id assert call_import.provider == "exotel" + + +def test_create_evaluation_sets_actor_emails( + authenticated_client, db_session, org_id, seed_org +): + metric = _make_metric(db_session, org_id) + call_import, _rows = _make_call_import(db_session, org_id, rows=2) + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id]), + ) + assert response.status_code == 202, response.text + body = response.json() + assert body["created_by_email"] == "owner@example.com" + assert body["last_updated_by_email"] == "owner@example.com" + + +def test_update_evaluation_name_stamps_last_updated_by_email( + authenticated_client, db_session, org_id, seed_org +): + metric = _make_metric(db_session, org_id) + call_import, _rows = _make_call_import(db_session, org_id, rows=1) + + created = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id]), + ) + assert created.status_code == 202, created.text + eval_id = created.json()["id"] + + patched = authenticated_client.patch( + f"/api/v1/call-imports/{call_import.id}/evaluations/{eval_id}", + json={"name": "Renamed run"}, + ) + assert patched.status_code == 200, patched.text + body = patched.json() + assert body["name"] == "Renamed run" + assert body["created_by_email"] == "owner@example.com" + assert body["last_updated_by_email"] == "owner@example.com" diff --git a/tests/test_api/test_call_import_evaluations_mapped_async.py b/tests/test_api/test_call_import_evaluations_mapped_async.py index ac3035f9..f072dd68 100644 --- a/tests/test_api/test_call_import_evaluations_mapped_async.py +++ b/tests/test_api/test_call_import_evaluations_mapped_async.py @@ -117,3 +117,48 @@ def test_create_evaluation_from_mapped_enqueues_async_materialization( .first() ) assert refreshed_import.status == CallImportStatus.PROCESSING + + +def test_create_evaluation_from_mapped_stamps_parent_last_updated_by( + authenticated_client, + db_session, + org_id, + seed_org, + monkeypatch, +): + from tests.test_api.test_call_import_evaluations import ( + _eval_body, + _make_metric, + ) + + monkeypatch.setattr( + "app.api.v1.routes.call_imports._ensure_blob_storage_enabled", + lambda: None, + ) + + metric = _make_metric(db_session, org_id) + workspace = metric.workspace_id + call_import = _make_mapped_call_import(db_session, org_id, workspace) + call_import.last_updated_by_user_id = None + db_session.commit() + + delay_mock = MagicMock(return_value=MagicMock(id="async-task")) + fake_bulk_ops = types.ModuleType("app.workers.tasks.call_import_bulk_ops") + fake_bulk_ops.materialize_mapped_call_import_evaluation_task = MagicMock( + delay=delay_mock, + ) + monkeypatch.setitem( + sys.modules, + "app.workers.tasks.call_import_bulk_ops", + fake_bulk_ops, + ) + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id]), + ) + assert response.status_code == 202, response.text + + detail = authenticated_client.get(f"/api/v1/call-imports/{call_import.id}") + assert detail.status_code == 200, detail.text + assert detail.json()["last_updated_by_email"] == "owner@example.com" diff --git a/tests/test_api/test_call_import_metric_clusters_rows.py b/tests/test_api/test_call_import_metric_clusters_rows.py index a1d1bd31..6595b5ab 100644 --- a/tests/test_api/test_call_import_metric_clusters_rows.py +++ b/tests/test_api/test_call_import_metric_clusters_rows.py @@ -147,6 +147,46 @@ def fake_apply_async(*, kwargs=None, **_kw): assert len(captured["evaluation_row_ids"]) == 2 +def test_generate_metric_clusters_stamps_last_updated_by_email( + authenticated_client, + db_session, + org_id, + seed_org, + make_ai_provider, + monkeypatch, +): + make_ai_provider(provider="openai", is_active=True) + call_import, evaluation, _ = _seed_eval_with_rows( + db_session, + org_id, + rows=[ + {"conversation_id": "c0", "status": "completed", "score_value": 0.2}, + ], + ) + evaluation.last_updated_by_user_id = None + db_session.commit() + + def fake_apply_async(*, kwargs=None, **_kw): + return types.SimpleNamespace(id="cluster-task-1") + + monkeypatch.setattr( + "app.workers.tasks.generate_evaluation_metric_clusters.generate_evaluation_metric_clusters_task.apply_async", + fake_apply_async, + ) + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations/{evaluation.id}/metric-clusters", + json={"row_limit": 1}, + ) + assert response.status_code == 200, response.text + + detail = authenticated_client.get( + f"/api/v1/call-imports/{call_import.id}/evaluations/{evaluation.id}" + ) + assert detail.status_code == 200, detail.text + assert detail.json()["last_updated_by_email"] == "owner@example.com" + + def test_cancel_preserves_selected_row_ids_in_state( authenticated_client, db_session, org_id, seed_org, make_ai_provider, monkeypatch ): diff --git a/tests/test_api/test_call_import_sharded_row_mutations.py b/tests/test_api/test_call_import_sharded_row_mutations.py index 6bc8b1ad..dde88d01 100644 --- a/tests/test_api/test_call_import_sharded_row_mutations.py +++ b/tests/test_api/test_call_import_sharded_row_mutations.py @@ -15,6 +15,7 @@ import pytest from fastapi import HTTPException +from app.core.auth.principal import AuthMethod, Principal from app.api.v1.routes.call_imports import ( cancel_call_import_row_diarisation, toggle_call_import_row_speaker_swap, @@ -32,6 +33,14 @@ def _fake_catalog_db(call_import): return catalog_db +def _test_principal(organization_id): + return Principal( + organization_id=organization_id, + auth_method=AuthMethod.LOCAL_PASSWORD, + user_id=uuid4(), + ) + + def _fake_shard_row( *, call_import_id, @@ -97,6 +106,7 @@ def fake_locate(_row_id): row_id=row_id, api_key="", organization_id=organization_id, + principal=_test_principal(organization_id), db=_fake_catalog_db(_fake_call_import(call_import_id, organization_id)), ) @@ -128,6 +138,7 @@ def fake_locate(_row_id): row_id=row_id, api_key="", organization_id=organization_id, + principal=_test_principal(organization_id), db=_fake_catalog_db(_fake_call_import(call_import_id, organization_id)), ) @@ -175,6 +186,7 @@ async def test_cancel_diarisation_commits_on_shard_session(monkeypatch): row_id=row_id, api_key="", organization_id=organization_id, + principal=_test_principal(organization_id), db=_fake_catalog_db(_fake_call_import(call_import_id, organization_id)), ) diff --git a/tests/test_workers/test_call_import_unified_dispatch.py b/tests/test_workers/test_call_import_unified_dispatch.py index d923ee4a..32c5ab22 100644 --- a/tests/test_workers/test_call_import_unified_dispatch.py +++ b/tests/test_workers/test_call_import_unified_dispatch.py @@ -1,16 +1,16 @@ """Unit tests for the unified call-import eval dispatch pipeline.""" from types import SimpleNamespace +from unittest.mock import MagicMock, patch from uuid import uuid4 -import pytest - from app.models.enums import CallImportRowStatus from app.workers.concurrency.eval_dispatch import ( EvalDispatchOutcome, _needs_import_for_eval, _needs_transcribe_for_eval, _try_dispatch_single_row, + build_eval_chain_import_apply_async, ) @@ -99,11 +99,51 @@ def test_needs_transcribe_after_recording_ready(): ) -def test_try_dispatch_enqueues_import_for_pending_row(monkeypatch): +@patch("app.workers.tasks.process_call_import_row.process_call_import_row_task") +def test_build_eval_chain_import_apply_async_passes_run_eval_row_id(mock_task): + source_row = _source_row() + eval_row = SimpleNamespace(id=uuid4()) + reserved_task_id = "reserved-import-task" + + async_result = MagicMock() + async_result.id = reserved_task_id + mock_task.apply_async.return_value = async_result + + result = build_eval_chain_import_apply_async( + source_row=source_row, + eval_row=eval_row, + reserved_task_id=reserved_task_id, + ) + + assert result is async_result + mock_task.apply_async.assert_called_once_with( + args=(str(source_row.id),), + kwargs={ + "_eval_slot_task_id": reserved_task_id, + "run_eval_row_id": str(eval_row.id), + }, + queue="imports", + task_id=reserved_task_id, + ) + + +@patch("app.workers.concurrency.eval_dispatch.build_eval_chain_import_apply_async") +def test_try_dispatch_enqueues_import_for_pending_row( + mock_build_import_apply_async, monkeypatch +): monkeypatch.setattr( "app.db_sharding.sessions.is_sharding_enabled", lambda: False, ) + monkeypatch.setattr( + "app.services.call_imports.evaluation_bulk_op.get_evaluation_bulk_operation", + lambda _evaluation_id: None, + ) + monkeypatch.setattr( + "app.workers.concurrency.import_dispatch._peek_authenticated_import_credit", + lambda **kwargs: None, + ) + evaluation = _evaluation() eval_row = SimpleNamespace(id=uuid4(), celery_task_id=None, status="pending") call_import = SimpleNamespace( @@ -116,15 +156,10 @@ def test_try_dispatch_enqueues_import_for_pending_row(monkeypatch): source_row = _source_row( call_import_id=evaluation.call_import_id, ) - captured = {} - class _AsyncResult: - id = "import-task-123" - - monkeypatch.setattr( - "app.workers.tasks.process_call_import_row.process_call_import_row_task.apply_async", - lambda *a, **kw: captured.update({"apply_async_kwargs": kw}) or _AsyncResult(), - ) + async_result = MagicMock() + async_result.id = "import-task-123" + mock_build_import_apply_async.return_value = async_result def fake_reserve(**kwargs): kwargs["enqueue_fn"]("reserved-id") @@ -144,7 +179,8 @@ def fake_reserve(**kwargs): ) assert result == EvalDispatchOutcome("dispatched") - assert captured["apply_async_kwargs"]["kwargs"]["run_eval_row_id"] == str( - eval_row.id + mock_build_import_apply_async.assert_called_once_with( + source_row=source_row, + eval_row=eval_row, + reserved_task_id="reserved-id", ) - assert captured["apply_async_kwargs"]["queue"] == "imports" diff --git a/tests/test_workers/test_process_call_import_row.py b/tests/test_workers/test_process_call_import_row.py index e9f56f09..0dcdbc22 100644 --- a/tests/test_workers/test_process_call_import_row.py +++ b/tests/test_workers/test_process_call_import_row.py @@ -226,6 +226,9 @@ def run(self, row_id, *args, **kwargs): def retry(self, exc=None, countdown=None): raise RetryCalled((exc, countdown)) + def apply_async(self, *args, **kwargs): + return types.SimpleNamespace(id="test-task-id") + return _FakeBindTask() class _CeleryDelegate: