51 lines
1.5 KiB
Python
51 lines
1.5 KiB
Python
"""Health reflects a live event loop and a usable database connection."""
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import text
|
|
|
|
from .database import async_session_maker
|
|
|
|
HEARTBEAT = Path(os.getenv("HEARTBEAT_FILE", "/tmp/lottery-heartbeat"))
|
|
|
|
|
|
async def heartbeat():
|
|
while True:
|
|
async with asyncio.timeout(20):
|
|
async with async_session_maker() as session:
|
|
await session.execute(text("SELECT 1"))
|
|
pending = HEARTBEAT.with_name(HEARTBEAT.name + ".tmp")
|
|
pending.write_text(str(time.time()), encoding="ascii")
|
|
pending.replace(HEARTBEAT)
|
|
await asyncio.sleep(10)
|
|
|
|
|
|
def healthy():
|
|
try:
|
|
return 0 <= time.time() - float(HEARTBEAT.read_text(encoding="ascii")) < 60
|
|
except (OSError, ValueError):
|
|
return False
|
|
|
|
|
|
def start_watchdog(grace_seconds=75, interval=10):
|
|
"""Exit even when the event loop or driver cancellation itself is stuck."""
|
|
stopped = threading.Event()
|
|
started = time.monotonic()
|
|
|
|
def watch():
|
|
while not stopped.wait(interval):
|
|
if time.monotonic() - started >= grace_seconds and not healthy():
|
|
logging.getLogger(__name__).critical("Heartbeat expired; restarting the application")
|
|
os._exit(1)
|
|
|
|
threading.Thread(target=watch, name="heartbeat-watchdog", daemon=True).start()
|
|
return stopped
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(0 if healthy() else 1)
|