Recover stalled database connections and skip unreachable chat recipients

src/core/health.py

@@ -1,6 +1,8 @@
 """Health reflects a live event loop and a usable database connection."""
 import asyncio
+import logging
 import os
+import threading
 import time
 from pathlib import Path
 
@@ -13,9 +15,12 @@ HEARTBEAT = Path(os.getenv("HEARTBEAT_FILE", "/tmp/lottery-heartbeat"))
 
 async def heartbeat():
     while True:
-        async with async_session_maker() as session:
-            await session.execute(text("SELECT 1"))
-        HEARTBEAT.write_text(str(time.time()), encoding="ascii")
+        async with asyncio.timeout(20):
+            async with async_session_maker() as session:
+                await session.execute(text("SELECT 1"))
+        pending = HEARTBEAT.with_name(HEARTBEAT.name + ".tmp")
+        pending.write_text(str(time.time()), encoding="ascii")
+        pending.replace(HEARTBEAT)
         await asyncio.sleep(10)
 
 
@@ -26,5 +31,20 @@ def healthy():
         return False
 
 
+def start_watchdog(grace_seconds=75, interval=10):
+    """Exit even when the event loop or driver cancellation itself is stuck."""
+    stopped = threading.Event()
+    started = time.monotonic()
+
+    def watch():
+        while not stopped.wait(interval):
+            if time.monotonic() - started >= grace_seconds and not healthy():
+                logging.getLogger(__name__).critical("Heartbeat expired; restarting the application")
+                os._exit(1)
+
+    threading.Thread(target=watch, name="heartbeat-watchdog", daemon=True).start()
+    return stopped
+
+
 if __name__ == "__main__":
     raise SystemExit(0 if healthy() else 1)