diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0d2c9697..b4f071f43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,8 +46,8 @@ jobs: fi uv run pre-commit run --from-ref "$BASE" --to-ref HEAD - python: - name: python checks + lint-python: + name: python lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -67,8 +67,39 @@ jobs: - name: Focused Python lint run: make lint-python - - name: Focused Python tests - run: make test-python + unit: + name: unit (py${{ matrix.python }} / ${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + python: ["3.12"] + steps: + - uses: actions/checkout@v7 + + - name: Install uv + uses: astral-sh/setup-uv@v8.2.0 + with: + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Set up Python + run: uv python install ${{ matrix.python }} + + # Everything the suite imports, minus `retrieval`: torch/transformers have + # no consumer in raven/, and their native runtimes segfault at interpreter + # finalization (see the same hazard noted in cli/agent_commands.py). + - name: Install dependencies + run: uv sync --locked --extra channels --extra tools --extra sandbox --dev + + - name: Unit tests (integration + e2e deselected by norecursedirs + addopts) + # TERM=dumb: typer's rich_utils force-enables colour when GITHUB_ACTIONS is + # set, and the highlighter then splits option names with escape codes, so + # `"--flag" in result.stdout` assertions fail only on CI. + env: + TERM: dumb + run: uv run --extra channels --extra tools --extra sandbox pytest -q windows-upgrade: name: Windows self-upgrade @@ -83,7 +114,10 @@ jobs: run: uv python install 3.12 - name: Test real uv self-upgrade - run: uvx --python 3.12 --from "pytest>=8,<10" pytest --noconftest tests/integration/test_cli_upgrade_real_uv.py -q + # pytest-asyncio is needed even though this file has no async test: the + # isolated interpreter still reads the project's pytest config, whose + # --strict-config turns the then-unknown asyncio_mode into a hard error. + run: uvx --python 3.12 --from "pytest>=8,<10" --with pytest-asyncio pytest --noconftest tests/integration/test_cli_upgrade_real_uv.py -q tui: name: TUI checks diff --git a/Makefile b/Makefile index 6742f9eb7..8c689eb0f 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ lint-bridge: test: test-python test-tui test-python: - uv run --extra dev pytest tests/test_commit_lint.py tests/test_large_file_check.py tests/test_cli_smoke.py tests/test_litellm_setup.py -q + uv run --extra channels --extra tools --extra sandbox pytest -q test-tui: npm test --prefix ui-tui diff --git a/pyproject.toml b/pyproject.toml index f273aaac3..eb54f691e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -255,9 +255,19 @@ ignore = ["E501"] "benchmarks/proactivity_eval/runners/synthesizers/keyword.py" = ["E741"] [tool.pytest.ini_options] -asyncio_mode = "auto" testpaths = ["tests"] +norecursedirs = ["tests/integration", "tests/data", "*.egg", ".*", "build", "dist"] +addopts = [ + "-ra", + "--strict-markers", + "--strict-config", + "-m", + "not integration and not e2e", +] +asyncio_mode = "auto" markers = [ + "integration: needs a real external resource (LLM / RPC subprocess / VM / network); deselected by default, run with '-m integration' or by pointing at tests/integration", + "e2e: spawns real subprocesses / drives the full stack (slow); deselected by default, run with '-m e2e' (e.g. tests/tui/autotest)", "real_llm: hits a real everos runtime (LLM + embedding); skipped unless EVEROS_LLM__API_KEY / EVEROS_EMBEDDING__MODEL are configured", "llm_judge: uses an LLM as a grader for extraction quality (extra cost / non-determinism); opt-in", ] diff --git a/tests/conftest.py b/tests/conftest.py index 9e82742a6..994463e8a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,34 @@ import pytest +def pytest_unconfigure(config: pytest.Config) -> None: + """On CI, hard-exit past interpreter finalization once the run is over. + + Native runtimes pulled in by the suite (lancedb's Rust/tokio thread, asyncio + subprocess transports finalized during GC) segfault Py_FinalizeEx on Linux, + turning a fully green run into exit 139. raven.cli._exit guards the CLI the + same way; the pytest process needs its own guard because it finalizes with + those runtimes live. + + Local runs keep normal semantics so nothing masks an exit-time error, and + the recorded status is preserved either way -- a failing run still exits + non-zero. + """ + import os + + if not os.environ.get("CI"): + return + + from raven.cli._exit import flush_and_hard_exit + + flush_and_hard_exit(int(getattr(config, "_raven_exitstatus", 0))) + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + """Stash the real exit status so pytest_unconfigure can preserve it.""" + session.config._raven_exitstatus = int(exitstatus) # type: ignore[attr-defined] + + @pytest.fixture(autouse=True) def _restore_loguru_enabled_state(): """Undo any ``loguru.logger.disable("raven")`` left over from a diff --git a/tests/test_evolver_launch_e2e.py b/tests/integration/test_evolver_launch_e2e.py similarity index 100% rename from tests/test_evolver_launch_e2e.py rename to tests/integration/test_evolver_launch_e2e.py diff --git a/tests/test_sandbox_cli_integration.py b/tests/integration/test_sandbox_cli_integration.py similarity index 100% rename from tests/test_sandbox_cli_integration.py rename to tests/integration/test_sandbox_cli_integration.py diff --git a/tests/test_sandbox_cli_real_vm.py b/tests/integration/test_sandbox_cli_real_vm.py similarity index 100% rename from tests/test_sandbox_cli_real_vm.py rename to tests/integration/test_sandbox_cli_real_vm.py diff --git a/tests/test_sandbox_integration.py b/tests/integration/test_sandbox_integration.py similarity index 100% rename from tests/test_sandbox_integration.py rename to tests/integration/test_sandbox_integration.py diff --git a/tests/test_sentinel_e2e.py b/tests/integration/test_sentinel_e2e.py similarity index 100% rename from tests/test_sentinel_e2e.py rename to tests/integration/test_sentinel_e2e.py diff --git a/tests/test_token_wise_agentloop_experiment.py b/tests/integration/test_token_wise_ablation_real_llm.py similarity index 98% rename from tests/test_token_wise_agentloop_experiment.py rename to tests/integration/test_token_wise_ablation_real_llm.py index 22ea11c62..deced2827 100644 --- a/tests/test_token_wise_agentloop_experiment.py +++ b/tests/integration/test_token_wise_ablation_real_llm.py @@ -41,8 +41,10 @@ from raven.token_wise.registry import StrategyRegistry from raven.token_wise.usage_tracker import UsageTracker -KEY_FILE = Path(__file__).resolve().parent.parent / "raven" / "key.env" -REPORT_PATH = Path(__file__).resolve().parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT.md" +pytestmark = pytest.mark.real_llm + +KEY_FILE = Path(__file__).resolve().parent.parent.parent / "raven" / "key.env" +REPORT_PATH = Path(__file__).resolve().parent.parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT.md" MODEL = "anthropic/claude-sonnet-4-5" TURNS = 6 COST_GUARD_USD = 0.50 diff --git a/tests/test_token_wise_hermes_vs_raven.py b/tests/integration/test_token_wise_hermes_vs_raven_real_llm.py similarity index 98% rename from tests/test_token_wise_hermes_vs_raven.py rename to tests/integration/test_token_wise_hermes_vs_raven_real_llm.py index d07689a69..c4cdc5242 100644 --- a/tests/test_token_wise_hermes_vs_raven.py +++ b/tests/integration/test_token_wise_hermes_vs_raven_real_llm.py @@ -45,8 +45,12 @@ from raven.token_wise.system_and_tail_cache import SystemAndTailCacheStrategy from raven.token_wise.usage_tracker import UsageTracker -KEY_FILE = Path(__file__).resolve().parent.parent / "raven" / "key.env" -REPORT_PATH = Path(__file__).resolve().parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT_HERMES_VS_RAVEN.md" +pytestmark = pytest.mark.real_llm + +KEY_FILE = Path(__file__).resolve().parent.parent.parent / "raven" / "key.env" +REPORT_PATH = ( + Path(__file__).resolve().parent.parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT_HERMES_VS_RAVEN.md" +) MODEL = "anthropic/claude-sonnet-4-5" COST_GUARD_USD = 2.00 _OPENROUTER_PIN = {"provider": {"order": ["Anthropic"], "allow_fallbacks": False}} diff --git a/tests/test_token_wise_integration_openrouter.py b/tests/integration/test_token_wise_openrouter_real_llm.py similarity index 98% rename from tests/test_token_wise_integration_openrouter.py rename to tests/integration/test_token_wise_openrouter_real_llm.py index 0dca7582a..13db0c2ab 100644 --- a/tests/test_token_wise_integration_openrouter.py +++ b/tests/integration/test_token_wise_openrouter_real_llm.py @@ -34,8 +34,10 @@ from raven.providers.litellm_provider import LiteLLMProvider from raven.token_wise.registry import StrategyRegistry -KEY_FILE = Path(__file__).resolve().parent.parent / "raven" / "key.env" -REPORT_PATH = Path(__file__).resolve().parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT.md" +pytestmark = pytest.mark.real_llm + +KEY_FILE = Path(__file__).resolve().parent.parent.parent / "raven" / "key.env" +REPORT_PATH = Path(__file__).resolve().parent.parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT.md" MODEL = "anthropic/claude-sonnet-4-5" TURNS = 6 COST_GUARD_USD = 0.50 # hard cap; abort if we go over diff --git a/tests/test_token_wise_workload_scenarios.py b/tests/integration/test_token_wise_workload_real_llm.py similarity index 99% rename from tests/test_token_wise_workload_scenarios.py rename to tests/integration/test_token_wise_workload_real_llm.py index 979add2e4..ded57d347 100644 --- a/tests/test_token_wise_workload_scenarios.py +++ b/tests/integration/test_token_wise_workload_real_llm.py @@ -47,8 +47,10 @@ from raven.token_wise.registry import StrategyRegistry from raven.token_wise.usage_tracker import UsageTracker -KEY_FILE = Path(__file__).resolve().parent.parent / "raven" / "key.env" -REPORT_PATH = Path(__file__).resolve().parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT_WORKLOADS.md" +pytestmark = pytest.mark.real_llm + +KEY_FILE = Path(__file__).resolve().parent.parent.parent / "raven" / "key.env" +REPORT_PATH = Path(__file__).resolve().parent.parent.parent / "raven" / "token_wise" / "EXPERIMENT_REPORT_WORKLOADS.md" MODEL = "anthropic/claude-sonnet-4-5" COST_GUARD_USD = 1.50 _OPENROUTER_PIN = {"provider": {"order": ["Anthropic"], "allow_fallbacks": False}} diff --git a/tests/test_atomic_io.py b/tests/test_atomic_io.py index e3b18d778..3a45e2091 100644 --- a/tests/test_atomic_io.py +++ b/tests/test_atomic_io.py @@ -38,7 +38,10 @@ def test_locked_append_concurrent_writers_lose_nothing(tmp_path: Path): """Two processes appending concurrently: every line lands, and the lines of one locked_append call stay contiguous (turn-block invariant).""" path = tmp_path / "s.jsonl" - procs = [multiprocessing.Process(target=_append_worker, args=(str(path), w)) for w in range(WRITERS)] + # spawn, not the Linux default fork: pytest leaves the parent multi-threaded, + # and forking from there segfaults the interpreter at exit. + ctx = multiprocessing.get_context("spawn") + procs = [ctx.Process(target=_append_worker, args=(str(path), w)) for w in range(WRITERS)] for p in procs: p.start() for p in procs: diff --git a/tests/test_channels_dingtalk.py b/tests/test_channels_dingtalk.py index f2d12449e..7df29f438 100644 --- a/tests/test_channels_dingtalk.py +++ b/tests/test_channels_dingtalk.py @@ -8,6 +8,9 @@ from unittest.mock import AsyncMock, MagicMock import pytest + +pytest.importorskip("dingtalk_stream") + from dingtalk_stream import AckMessage from raven.channels.adapters.dingtalk import parsing as p diff --git a/tests/test_channels_feishu.py b/tests/test_channels_feishu.py index 8b3484128..b6694115c 100644 --- a/tests/test_channels_feishu.py +++ b/tests/test_channels_feishu.py @@ -8,6 +8,8 @@ import pytest +pytest.importorskip("lark_oapi") + from raven.channels.adapters.feishu import cards, content from raven.channels.adapters.feishu.channel import FeishuChannel diff --git a/tests/test_channels_matrix.py b/tests/test_channels_matrix.py index 1a0566b52..0b847b071 100644 --- a/tests/test_channels_matrix.py +++ b/tests/test_channels_matrix.py @@ -9,6 +9,8 @@ import pytest +pytest.importorskip("nio") + from raven.channels.adapters.matrix import content from raven.channels.adapters.matrix.channel import MatrixChannel diff --git a/tests/test_channels_qq.py b/tests/test_channels_qq.py index 2ebe75ac8..81429542c 100644 --- a/tests/test_channels_qq.py +++ b/tests/test_channels_qq.py @@ -11,6 +11,10 @@ from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock +import pytest + +pytest.importorskip("botpy") + from raven.channels.adapters.qq import parsing as qp from raven.channels.adapters.qq.channel import QQChannel diff --git a/tests/test_channels_slack.py b/tests/test_channels_slack.py index 4ed8f993f..3f888b996 100644 --- a/tests/test_channels_slack.py +++ b/tests/test_channels_slack.py @@ -11,6 +11,10 @@ from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock +import pytest + +pytest.importorskip("slack_sdk") + from raven.channels.adapters.slack import parsing as sp from raven.channels.adapters.slack.channel import SlackChannel diff --git a/tests/test_channels_telegram.py b/tests/test_channels_telegram.py index a48df2ccc..5dce98619 100644 --- a/tests/test_channels_telegram.py +++ b/tests/test_channels_telegram.py @@ -8,6 +8,8 @@ import pytest +pytest.importorskip("telegram") + from raven.channels.adapters.telegram.channel import ( TelegramChannel, _inbound_ext, diff --git a/tests/test_channels_wecom.py b/tests/test_channels_wecom.py index c5e7f5345..7d3086e92 100644 --- a/tests/test_channels_wecom.py +++ b/tests/test_channels_wecom.py @@ -6,6 +6,10 @@ from types import SimpleNamespace from unittest.mock import AsyncMock +import pytest + +pytest.importorskip("wecom_aibot_sdk") + from raven.channels.adapters.wecom.channel import WecomChannel diff --git a/tests/test_cli_cron_handler.py b/tests/test_cli_cron_handler.py index b73524ecb..6e4706273 100644 --- a/tests/test_cli_cron_handler.py +++ b/tests/test_cli_cron_handler.py @@ -498,7 +498,7 @@ async def test_repl_assembly_cron_renders_once_via_clioutlet_not_broadcast(fake_ patch_cron_config(["*"]) class _CronEchoLoop: - async def run_turn(self, req, emit, drain, *, stream) -> TurnOutcome: + async def run_turn(self, req, emit, drain, *, stream, inline_tool_stream=False) -> TurnOutcome: await emit(Text(content=f"cron-reply<{req.conversation}>", source=req.source)) return TurnOutcome(usage=Usage(0, 0, 0), explicit_reply=True) diff --git a/tests/test_cli_tui_logging_isolation.py b/tests/test_cli_tui_logging_isolation.py index de1f99624..fdc3963f4 100644 --- a/tests/test_cli_tui_logging_isolation.py +++ b/tests/test_cli_tui_logging_isolation.py @@ -65,6 +65,14 @@ def _isolate_logging(tmp_path, monkeypatch): saved_named_handlers = {name: list(logging.getLogger(name).handlers) for name in _KNOWN_TTY_LEAKING_LOGGERS} saved_disable = logging.root.manager.disable + # litellm's stderr handlers are global and installed once at litellm import; a + # prior test that called redirect_loguru_to_file may have stripped them. Ensure + # the precondition (a stderr handler exists to be stripped) holds regardless of + # suite order — the finally-block restores the snapshot, dropping what we add. + for name in _KNOWN_TTY_LEAKING_LOGGERS: + if not _tty_stream_handlers(name): + logging.getLogger(name).addHandler(logging.StreamHandler(sys.stderr)) + try: yield finally: diff --git a/tests/test_currently_focused.py b/tests/test_currently_focused.py index 86b12459e..2536e0900 100644 --- a/tests/test_currently_focused.py +++ b/tests/test_currently_focused.py @@ -90,7 +90,7 @@ def _seed_episodes( def _run(coro): - return asyncio.get_event_loop().run_until_complete(coro) + return asyncio.run(coro) # =========================================================================== diff --git a/tests/test_daily_analysis.py b/tests/test_daily_analysis.py index 73bb7c85c..839d07f7e 100644 --- a/tests/test_daily_analysis.py +++ b/tests/test_daily_analysis.py @@ -78,7 +78,7 @@ def config() -> DailyAnalysisConfig: def _run(coro): - return asyncio.get_event_loop().run_until_complete(coro) + return asyncio.run(coro) def _provider_with_result(payload: dict) -> MagicMock: diff --git a/tests/test_default_context_engine.py b/tests/test_default_context_engine.py index f792ce4da..3b92e5714 100644 --- a/tests/test_default_context_engine.py +++ b/tests/test_default_context_engine.py @@ -195,8 +195,8 @@ async def test_skill_and_memory_run_concurrently( self, builder: ContextBuilder, ) -> None: - slow_source = _StubSource("local", hits=[], delay_s=0.10) - slow_backend = _StubBackend(recall_response=[], delay_s=0.10) + slow_source = _StubSource("local", hits=[], delay_s=0.20) + slow_backend = _StubBackend(recall_response=[], delay_s=0.20) eng = _engine( builder, router=SkillForgeRouter([slow_source]), @@ -210,8 +210,8 @@ async def test_skill_and_memory_run_concurrently( turn=_turn(), ) elapsed = time.monotonic() - t0 - # Serial would be ~0.20 s. Concurrent ~0.10 s. Loose bound 0.15. - assert elapsed < 0.15 + # Serial would be ~0.40 s. Concurrent ~0.20 s. Loose bound 0.30. + assert elapsed < 0.30 async def test_track_ids_passed_to_recall( self, diff --git a/tests/test_memory_locking.py b/tests/test_memory_locking.py index 98078694b..0c9aac7b1 100644 --- a/tests/test_memory_locking.py +++ b/tests/test_memory_locking.py @@ -5,9 +5,10 @@ from __future__ import annotations +import multiprocessing import sys import time -from multiprocessing import Process, Value +from multiprocessing import Value from pathlib import Path import pytest @@ -72,9 +73,12 @@ def test_lock_serializes_across_processes(tmp_path: Path) -> None: store = MemoryStore(tmp_path) store.write_long_term("init\n") - ready = Value("i", 0) - done = Value("i", 0) - p = Process( + # spawn, not the Linux default fork: pytest leaves the parent multi-threaded, + # and forking from there segfaults the interpreter at exit. + ctx = multiprocessing.get_context("spawn") + ready = ctx.Value("i", 0) + done = ctx.Value("i", 0) + p = ctx.Process( target=_worker_acquire_and_hold, args=(str(tmp_path), 0.3, ready, done), ) diff --git a/tests/test_sentinel_observations_producer.py b/tests/test_sentinel_observations_producer.py index 0143e9bb9..738494bb4 100644 --- a/tests/test_sentinel_observations_producer.py +++ b/tests/test_sentinel_observations_producer.py @@ -119,7 +119,7 @@ def _seed_topic_fires( def _run(updater: AttentionUpdater): - return asyncio.get_event_loop().run_until_complete(updater.update()) + return asyncio.run(updater.update()) # =========================================================================== diff --git a/tests/test_session_manager.py b/tests/test_session_manager.py index c82e69760..6db033e81 100644 --- a/tests/test_session_manager.py +++ b/tests/test_session_manager.py @@ -245,7 +245,10 @@ def test_concurrent_writers_lose_no_turns(tmp_path: Path): """Two processes saving the same session: both turn blocks land, each block's messages contiguous (tool_call/result adjacency).""" key = "tui:race01" - procs = [multiprocessing.Process(target=_turn_worker, args=(str(tmp_path), key, w)) for w in range(2)] + # spawn, not the Linux default fork: pytest leaves the parent multi-threaded, + # and forking from there segfaults the interpreter at exit. + ctx = multiprocessing.get_context("spawn") + procs = [ctx.Process(target=_turn_worker, args=(str(tmp_path), key, w)) for w in range(2)] for p in procs: p.start() for p in procs: diff --git a/tests/test_tui_commands_error_codes.py b/tests/test_tui_commands_error_codes.py index 488ecc069..e33756aae 100644 --- a/tests/test_tui_commands_error_codes.py +++ b/tests/test_tui_commands_error_codes.py @@ -19,6 +19,20 @@ # --------------------------------------------------------------------------- +@pytest.fixture(autouse=True) +def _hermetic_preagentloop(monkeypatch): + """Isolate the AgentLoop-ctor error path from ambient config. + + _build_tui_agent_loop builds a lazy provider before constructing AgentLoop; + with no provider configured (clean CI, or a prior test that reset config + state) make_lazy_provider raises typer.Exit(1) before AgentLoop is reached, + so the ctor-error classification under test never runs. Stub it so these + tests exercise only what they claim to.""" + from unittest.mock import MagicMock + + monkeypatch.setattr("raven.cli._helpers.make_lazy_provider", lambda *a, **k: MagicMock()) + + def _patch_agent_loop_to_raise(monkeypatch: pytest.MonkeyPatch, exc: BaseException) -> None: """Replace ``AgentLoop`` constructor so that calling it raises ``exc``. diff --git a/tests/test_tui_rpc_model.py b/tests/test_tui_rpc_model.py index 72197f597..fe3128419 100644 --- a/tests/test_tui_rpc_model.py +++ b/tests/test_tui_rpc_model.py @@ -27,6 +27,9 @@ @pytest.fixture def fake_home(monkeypatch, tmp_path) -> Path: monkeypatch.setattr(Path, "home", lambda: tmp_path) + # Clear any process-wide config-path override a prior test left set, so + # get_config_path() falls back to the patched Path.home (monkeypatch restores it). + monkeypatch.setattr("raven.config.loader._current_config_path", None) return tmp_path diff --git a/tests/test_turn_wire_shape_conformance.py b/tests/test_turn_wire_shape_conformance.py index 2f67f3215..47eff934d 100644 --- a/tests/test_turn_wire_shape_conformance.py +++ b/tests/test_turn_wire_shape_conformance.py @@ -85,7 +85,9 @@ async def test_message_complete_payload_has_turn_id_and_usage_only() -> None: class _StubAgent: tools: dict = {} - async def run_turn(self, req, emit, drain, *, stream, usage_sink=None, text_sink=None): + async def run_turn( + self, req, emit, drain, *, stream, inline_tool_stream=False, usage_sink=None, text_sink=None + ): from raven.spine import Text, TurnOutcome, Usage await emit(Text(content="stub content (must not appear on wire)", source=req.source))