test: run the full unit suite in ci and fix revealed failures

.github/workflows/ci.yml

@@ -46,8 +46,8 @@ jobs:
           fi
           uv run pre-commit run --from-ref "$BASE" --to-ref HEAD
 
-  python:
-    name: python checks
+  lint-python:
+    name: python lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
@@ -67,8 +67,39 @@ jobs:
       - name: Focused Python lint
         run: make lint-python
 
-      - name: Focused Python tests
-        run: make test-python
+  unit:
+    name: unit (py${{ matrix.python }} / ${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python: ["3.12"]
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.2.0
+        with:
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+
+      - name: Set up Python
+        run: uv python install ${{ matrix.python }}
+
+      # Everything the suite imports, minus `retrieval`: torch/transformers have
+      # no consumer in raven/, and their native runtimes segfault at interpreter
+      # finalization (see the same hazard noted in cli/agent_commands.py).
+      - name: Install dependencies
+        run: uv sync --locked --extra channels --extra tools --extra sandbox --dev
+
+      - name: Unit tests (integration + e2e deselected by norecursedirs + addopts)
+        # TERM=dumb: typer's rich_utils force-enables colour when GITHUB_ACTIONS is
+        # set, and the highlighter then splits option names with escape codes, so
+        # `"--flag" in result.stdout` assertions fail only on CI.
+        env:
+          TERM: dumb
+        run: uv run --extra channels --extra tools --extra sandbox pytest -q
 
   windows-upgrade:
     name: Windows self-upgrade
@@ -83,7 +114,10 @@ jobs:
         run: uv python install 3.12
 
       - name: Test real uv self-upgrade
-        run: uvx --python 3.12 --from "pytest>=8,<10" pytest --noconftest tests/integration/test_cli_upgrade_real_uv.py -q
+        # pytest-asyncio is needed even though this file has no async test: the
+        # isolated interpreter still reads the project's pytest config, whose
+        # --strict-config turns the then-unknown asyncio_mode into a hard error.
+        run: uvx --python 3.12 --from "pytest>=8,<10" --with pytest-asyncio pytest --noconftest tests/integration/test_cli_upgrade_real_uv.py -q
 
   tui:
     name: TUI checks

Makefile

@@ -46,7 +46,7 @@ lint-bridge:
 test: test-python test-tui
 
 test-python:
-	uv run --extra dev pytest tests/test_commit_lint.py tests/test_large_file_check.py tests/test_cli_smoke.py tests/test_litellm_setup.py -q
+	uv run --extra channels --extra tools --extra sandbox pytest -q
 
 test-tui:
 	npm test --prefix ui-tui

pyproject.toml

@@ -255,9 +255,19 @@ ignore = ["E501"]
 "benchmarks/proactivity_eval/runners/synthesizers/keyword.py" = ["E741"]
 
 [tool.pytest.ini_options]
-asyncio_mode = "auto"
 testpaths = ["tests"]
+norecursedirs = ["tests/integration", "tests/data", "*.egg", ".*", "build", "dist"]
+addopts = [
+    "-ra",
+    "--strict-markers",
+    "--strict-config",
+    "-m",
+    "not integration and not e2e",
+]
+asyncio_mode = "auto"
 markers = [
+    "integration: needs a real external resource (LLM / RPC subprocess / VM / network); deselected by default, run with '-m integration' or by pointing at tests/integration",
+    "e2e: spawns real subprocesses / drives the full stack (slow); deselected by default, run with '-m e2e' (e.g. tests/tui/autotest)",
     "real_llm: hits a real everos runtime (LLM + embedding); skipped unless EVEROS_LLM__API_KEY / EVEROS_EMBEDDING__MODEL are configured",
     "llm_judge: uses an LLM as a grader for extraction quality (extra cost / non-determinism); opt-in",
 ]

tests/conftest.py

@@ -9,6 +9,34 @@
 import pytest
 
 
+def pytest_unconfigure(config: pytest.Config) -> None:
+    """On CI, hard-exit past interpreter finalization once the run is over.
+
+    Native runtimes pulled in by the suite (lancedb's Rust/tokio thread, asyncio
+    subprocess transports finalized during GC) segfault Py_FinalizeEx on Linux,
+    turning a fully green run into exit 139. raven.cli._exit guards the CLI the
+    same way; the pytest process needs its own guard because it finalizes with
+    those runtimes live.
+
+    Local runs keep normal semantics so nothing masks an exit-time error, and
+    the recorded status is preserved either way -- a failing run still exits
+    non-zero.
+    """
+    import os
+
+    if not os.environ.get("CI"):
+        return
+
+    from raven.cli._exit import flush_and_hard_exit
+
+    flush_and_hard_exit(int(getattr(config, "_raven_exitstatus", 0)))
+
+
+def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
+    """Stash the real exit status so pytest_unconfigure can preserve it."""
+    session.config._raven_exitstatus = int(exitstatus)  # type: ignore[attr-defined]
+
+
 @pytest.fixture(autouse=True)
 def _restore_loguru_enabled_state():
     """Undo any ``loguru.logger.disable("raven")`` left over from a

tests/test_token_wise_agentloop_experiment.py → tests/integration/test_token_wise_ablation_real_llm.py

@@ -41,8 +41,10 @@
 from raven.token_wise.registry import StrategyRegistry
 from raven.token_wise.usage_tracker import UsageTracker
 
-KEY_FILE = Path(__file__).resolve().parent.parent / "raven" / "key.env"
-REPORT_PATH = Path(__file__).resolve().parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT.md"
+pytestmark = pytest.mark.real_llm
+
+KEY_FILE = Path(__file__).resolve().parent.parent.parent / "raven" / "key.env"
+REPORT_PATH = Path(__file__).resolve().parent.parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT.md"
 MODEL = "anthropic/claude-sonnet-4-5"
 TURNS = 6
 COST_GUARD_USD = 0.50

tests/test_token_wise_hermes_vs_raven.py → tests/integration/test_token_wise_hermes_vs_raven_real_llm.py

@@ -45,8 +45,12 @@
 from raven.token_wise.system_and_tail_cache import SystemAndTailCacheStrategy
 from raven.token_wise.usage_tracker import UsageTracker
 
-KEY_FILE = Path(__file__).resolve().parent.parent / "raven" / "key.env"
-REPORT_PATH = Path(__file__).resolve().parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT_HERMES_VS_RAVEN.md"
+pytestmark = pytest.mark.real_llm
+
+KEY_FILE = Path(__file__).resolve().parent.parent.parent / "raven" / "key.env"
+REPORT_PATH = (
+    Path(__file__).resolve().parent.parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT_HERMES_VS_RAVEN.md"
+)
 MODEL = "anthropic/claude-sonnet-4-5"
 COST_GUARD_USD = 2.00
 _OPENROUTER_PIN = {"provider": {"order": ["Anthropic"], "allow_fallbacks": False}}

tests/test_token_wise_integration_openrouter.py → tests/integration/test_token_wise_openrouter_real_llm.py

@@ -34,8 +34,10 @@
 from raven.providers.litellm_provider import LiteLLMProvider
 from raven.token_wise.registry import StrategyRegistry
 
-KEY_FILE = Path(__file__).resolve().parent.parent / "raven" / "key.env"
-REPORT_PATH = Path(__file__).resolve().parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT.md"
+pytestmark = pytest.mark.real_llm
+
+KEY_FILE = Path(__file__).resolve().parent.parent.parent / "raven" / "key.env"
+REPORT_PATH = Path(__file__).resolve().parent.parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT.md"
 MODEL = "anthropic/claude-sonnet-4-5"
 TURNS = 6
 COST_GUARD_USD = 0.50  # hard cap; abort if we go over

tests/test_token_wise_workload_scenarios.py → tests/integration/test_token_wise_workload_real_llm.py

@@ -47,8 +47,10 @@
 from raven.token_wise.registry import StrategyRegistry
 from raven.token_wise.usage_tracker import UsageTracker
 
-KEY_FILE = Path(__file__).resolve().parent.parent / "raven" / "key.env"
-REPORT_PATH = Path(__file__).resolve().parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT_WORKLOADS.md"
+pytestmark = pytest.mark.real_llm
+
+KEY_FILE = Path(__file__).resolve().parent.parent.parent / "raven" / "key.env"
+REPORT_PATH = Path(__file__).resolve().parent.parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT_WORKLOADS.md"
 MODEL = "anthropic/claude-sonnet-4-5"
 COST_GUARD_USD = 1.50
 _OPENROUTER_PIN = {"provider": {"order": ["Anthropic"], "allow_fallbacks": False}}

tests/test_atomic_io.py

@@ -38,7 +38,10 @@ def test_locked_append_concurrent_writers_lose_nothing(tmp_path: Path):
     """Two processes appending concurrently: every line lands, and the
     lines of one locked_append call stay contiguous (turn-block invariant)."""
     path = tmp_path / "s.jsonl"
-    procs = [multiprocessing.Process(target=_append_worker, args=(str(path), w)) for w in range(WRITERS)]
+    # spawn, not the Linux default fork: pytest leaves the parent multi-threaded,
+    # and forking from there segfaults the interpreter at exit.
+    ctx = multiprocessing.get_context("spawn")
+    procs = [ctx.Process(target=_append_worker, args=(str(path), w)) for w in range(WRITERS)]
     for p in procs:
         p.start()
     for p in procs:

tests/test_channels_dingtalk.py

@@ -8,6 +8,9 @@
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
+
+pytest.importorskip("dingtalk_stream")
+
 from dingtalk_stream import AckMessage
 
 from raven.channels.adapters.dingtalk import parsing as p

tests/test_channels_feishu.py

@@ -8,6 +8,8 @@
 
 import pytest
 
+pytest.importorskip("lark_oapi")
+
 from raven.channels.adapters.feishu import cards, content
 from raven.channels.adapters.feishu.channel import FeishuChannel
 

tests/test_channels_matrix.py

@@ -9,6 +9,8 @@
 
 import pytest
 
+pytest.importorskip("nio")
+
 from raven.channels.adapters.matrix import content
 from raven.channels.adapters.matrix.channel import MatrixChannel
 

tests/test_channels_qq.py

@@ -11,6 +11,10 @@
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock
 
+import pytest
+
+pytest.importorskip("botpy")
+
 from raven.channels.adapters.qq import parsing as qp
 from raven.channels.adapters.qq.channel import QQChannel
 

tests/test_channels_slack.py

@@ -11,6 +11,10 @@
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock
 
+import pytest
+
+pytest.importorskip("slack_sdk")
+
 from raven.channels.adapters.slack import parsing as sp
 from raven.channels.adapters.slack.channel import SlackChannel
 

tests/test_channels_telegram.py

@@ -8,6 +8,8 @@
 
 import pytest
 
+pytest.importorskip("telegram")
+
 from raven.channels.adapters.telegram.channel import (
     TelegramChannel,
     _inbound_ext,

tests/test_channels_wecom.py

@@ -6,6 +6,10 @@
 from types import SimpleNamespace
 from unittest.mock import AsyncMock
 
+import pytest
+
+pytest.importorskip("wecom_aibot_sdk")
+
 from raven.channels.adapters.wecom.channel import WecomChannel
 
 

tests/test_cli_cron_handler.py

@@ -498,7 +498,7 @@ async def test_repl_assembly_cron_renders_once_via_clioutlet_not_broadcast(fake_
     patch_cron_config(["*"])
 
     class _CronEchoLoop:
-        async def run_turn(self, req, emit, drain, *, stream) -> TurnOutcome:
+        async def run_turn(self, req, emit, drain, *, stream, inline_tool_stream=False) -> TurnOutcome:
             await emit(Text(content=f"cron-reply<{req.conversation}>", source=req.source))
             return TurnOutcome(usage=Usage(0, 0, 0), explicit_reply=True)
 

tests/test_cli_tui_logging_isolation.py

@@ -65,6 +65,14 @@ def _isolate_logging(tmp_path, monkeypatch):
     saved_named_handlers = {name: list(logging.getLogger(name).handlers) for name in _KNOWN_TTY_LEAKING_LOGGERS}
     saved_disable = logging.root.manager.disable
 
+    # litellm's stderr handlers are global and installed once at litellm import; a
+    # prior test that called redirect_loguru_to_file may have stripped them. Ensure
+    # the precondition (a stderr handler exists to be stripped) holds regardless of
+    # suite order — the finally-block restores the snapshot, dropping what we add.
+    for name in _KNOWN_TTY_LEAKING_LOGGERS:
+        if not _tty_stream_handlers(name):
+            logging.getLogger(name).addHandler(logging.StreamHandler(sys.stderr))
+
     try:
         yield
     finally:

tests/test_currently_focused.py

@@ -90,7 +90,7 @@ def _seed_episodes(
 
 
 def _run(coro):
-    return asyncio.get_event_loop().run_until_complete(coro)
+    return asyncio.run(coro)
 
 
 # ===========================================================================

tests/test_daily_analysis.py

@@ -78,7 +78,7 @@ def config() -> DailyAnalysisConfig:
 
 
 def _run(coro):
-    return asyncio.get_event_loop().run_until_complete(coro)
+    return asyncio.run(coro)
 
 
 def _provider_with_result(payload: dict) -> MagicMock:

tests/test_default_context_engine.py

@@ -195,8 +195,8 @@ async def test_skill_and_memory_run_concurrently(
         self,
         builder: ContextBuilder,
     ) -> None:
-        slow_source = _StubSource("local", hits=[], delay_s=0.10)
-        slow_backend = _StubBackend(recall_response=[], delay_s=0.10)
+        slow_source = _StubSource("local", hits=[], delay_s=0.20)
+        slow_backend = _StubBackend(recall_response=[], delay_s=0.20)
         eng = _engine(
             builder,
             router=SkillForgeRouter([slow_source]),
@@ -210,8 +210,8 @@ async def test_skill_and_memory_run_concurrently(
             turn=_turn(),
         )
         elapsed = time.monotonic() - t0
-        # Serial would be ~0.20 s. Concurrent ~0.10 s. Loose bound 0.15.
-        assert elapsed < 0.15
+        # Serial would be ~0.40 s. Concurrent ~0.20 s. Loose bound 0.30.
+        assert elapsed < 0.30
 
     async def test_track_ids_passed_to_recall(
         self,

tests/test_memory_locking.py

@@ -5,9 +5,10 @@
 
 from __future__ import annotations
 
+import multiprocessing
 import sys
 import time
-from multiprocessing import Process, Value
+from multiprocessing import Value
 from pathlib import Path
 
 import pytest
@@ -72,9 +73,12 @@ def test_lock_serializes_across_processes(tmp_path: Path) -> None:
     store = MemoryStore(tmp_path)
     store.write_long_term("init\n")
 
-    ready = Value("i", 0)
-    done = Value("i", 0)
-    p = Process(
+    # spawn, not the Linux default fork: pytest leaves the parent multi-threaded,
+    # and forking from there segfaults the interpreter at exit.
+    ctx = multiprocessing.get_context("spawn")
+    ready = ctx.Value("i", 0)
+    done = ctx.Value("i", 0)
+    p = ctx.Process(
         target=_worker_acquire_and_hold,
         args=(str(tmp_path), 0.3, ready, done),
     )

tests/test_sentinel_observations_producer.py

@@ -119,7 +119,7 @@ def _seed_topic_fires(
 
 
 def _run(updater: AttentionUpdater):
-    return asyncio.get_event_loop().run_until_complete(updater.update())
+    return asyncio.run(updater.update())
 
 
 # ===========================================================================

tests/test_session_manager.py

@@ -245,7 +245,10 @@ def test_concurrent_writers_lose_no_turns(tmp_path: Path):
     """Two processes saving the same session: both turn blocks land,
     each block's messages contiguous (tool_call/result adjacency)."""
     key = "tui:race01"
-    procs = [multiprocessing.Process(target=_turn_worker, args=(str(tmp_path), key, w)) for w in range(2)]
+    # spawn, not the Linux default fork: pytest leaves the parent multi-threaded,
+    # and forking from there segfaults the interpreter at exit.
+    ctx = multiprocessing.get_context("spawn")
+    procs = [ctx.Process(target=_turn_worker, args=(str(tmp_path), key, w)) for w in range(2)]
     for p in procs:
         p.start()
     for p in procs:

tests/test_tui_commands_error_codes.py

@@ -19,6 +19,20 @@
 # ---------------------------------------------------------------------------
 
 
+@pytest.fixture(autouse=True)
+def _hermetic_preagentloop(monkeypatch):
+    """Isolate the AgentLoop-ctor error path from ambient config.
+
+    _build_tui_agent_loop builds a lazy provider before constructing AgentLoop;
+    with no provider configured (clean CI, or a prior test that reset config
+    state) make_lazy_provider raises typer.Exit(1) before AgentLoop is reached,
+    so the ctor-error classification under test never runs. Stub it so these
+    tests exercise only what they claim to."""
+    from unittest.mock import MagicMock
+
+    monkeypatch.setattr("raven.cli._helpers.make_lazy_provider", lambda *a, **k: MagicMock())
+
+
 def _patch_agent_loop_to_raise(monkeypatch: pytest.MonkeyPatch, exc: BaseException) -> None:
     """Replace ``AgentLoop`` constructor so that calling it raises ``exc``.