From 19ab3df5fc1318f7c52d050af876ea8c12ccc8b1 Mon Sep 17 00:00:00 2001 From: Perry Mosbacher Date: Wed, 2 Sep 2026 23:14:35 -0400 Subject: [PATCH] test: stop the suite killing itself part-way through `RepeaterDaemon._shutdown` arms an exit watchdog -- a `threading.Timer` that calls `os._exit(0)` after `SHUTDOWN_EXIT_GRACE_S` (5 s). That is right in production: it guarantees SIGTERM is honoured. `test_run_starts_http_and_handles_dispatcher_cancelled_gracefully` awaits `daemon.run()`, whose `finally` reaches `_shutdown()`, so it armed a real watchdog and returned. Five seconds later the timer fired and took the whole pytest process down. The failure mode is what makes this expensive: `os._exit(0)` exits with **status zero**. pytest produces no summary line and a truncated progress bar, so the run looks like it simply ended -- CI reads exit 0 as green on a run that never finished, and any failure after that point is invisible. Locally the suite appeared to hang at ~77% with no failures reported; it was actually being killed there. It also misattributes itself. The test blamed is whichever happened to be running five seconds later, so it moves between runs (`test_route_login_server_companion_only_skips_login_helper` and `test_route_login_server_hash_collision_offers_to_both` on two runs here) and never points at the cause. Finding it needed wrapping `_arm_exit_watchdog` to log the arming test. Two changes: - Patch `_arm_exit_watchdog` for that test, matching the guard the shutdown tests in `test_main_py_coverage.py` already use. - Add `tests/conftest.py` with an autouse fixture that fails any test leaving a live `shutdown-watchdog` timer, and cancels it. This blames the responsible test at the moment it leaks, rather than an unrelated one later. Verified it catches the original bug: reverting the one-line fix turns the silent process-kill into a teardown error naming the right test. `test_exit_watchdog_is_a_daemon_timer_that_forces_exit` is unaffected -- it sets a 0.05 s grace and asserts on a patched `os._exit` before returning, so it leaves nothing behind. With this the suite runs to completion: 3 failed, 1580 passed. Those 3 are pre-existing on `dev` and unrelated. Co-Authored-By: Claude Opus 5 --- tests/conftest.py | 55 ++++++++++++++++++++++++++++++++++++++ tests/test_main_py_more.py | 6 +++++ 2 files changed, 61 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..90b24a3 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,55 @@ +"""Shared test guards. + +`RepeaterDaemon._shutdown` arms an exit watchdog: a `threading.Timer` that calls +`os._exit(0)` once `SHUTDOWN_EXIT_GRACE_S` has elapsed. That is correct in +production -- it guarantees SIGTERM is honoured -- but a test that reaches +`_shutdown` without patching it leaves a live timer behind. When it fires, five +seconds later, it takes the whole pytest process down **with status 0**, part +way through an unrelated test. + +The damage is that it looks like success: pytest exits 0 with no summary line +and a truncated progress bar, so CI reports green on a run that never finished +and any failure after that point is invisible. The test blamed in the output is +whichever happened to be running when the timer fired, so it moves between runs +and does not point at the cause. + +The autouse fixture below fails the *responsible* test instead, at the moment it +leaks the timer. +""" +import threading + +import pytest + +_WATCHDOG_NAME = "shutdown-watchdog" + + +def _live_watchdogs(): + return [t for t in threading.enumerate() if t.name == _WATCHDOG_NAME and t.is_alive()] + + +@pytest.fixture(autouse=True) +def no_leaked_exit_watchdog(): + """Fail any test that leaves a live shutdown-exit watchdog behind. + + Patch it out when the code under test reaches `_shutdown`: + + with patch.object(daemon, "_arm_exit_watchdog"): + await daemon.run() + + or, when the watchdog itself is what you are testing, keep the grace short + and assert on a patched `repeater.main.os._exit` before the test returns. + """ + yield + + leaked = _live_watchdogs() + for timer in leaked: + timer.cancel() + + if leaked: + pytest.fail( + "test left %d live '%s' timer(s) running. Each one calls os._exit(0) " + "when it fires and will kill the pytest process mid-run, with exit " + "status 0 and no summary. Patch _arm_exit_watchdog for this test." + % (len(leaked), _WATCHDOG_NAME), + pytrace=False, + ) diff --git a/tests/test_main_py_more.py b/tests/test_main_py_more.py index 08631ac..5bcfe62 100644 --- a/tests/test_main_py_more.py +++ b/tests/test_main_py_more.py @@ -76,6 +76,12 @@ async def test_run_starts_http_and_handles_dispatcher_cancelled_gracefully(): patch("asyncio.get_running_loop", return_value=fake_loop_for_signals), patch("repeater.main.HTTPStatsServer", return_value=fake_http_instance), patch("os.path.exists", return_value=False), + # run()'s finally reaches _shutdown(), which arms the real exit + # watchdog: a threading.Timer that calls os._exit(0) after + # SHUTDOWN_EXIT_GRACE_S (5 s). It would fire long after this test + # returns and take the whole pytest process down. Same guard the + # shutdown tests in test_main_py_coverage.py already use. + patch.object(daemon, "_arm_exit_watchdog"), ): await daemon.run()