Recover stalled database connections and skip unreachable chat recipients
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
2026-09-13 20:21:20 +09:00
parent 8094cbb8f7
commit d5a9c13b68
10 changed files with 122 additions and 14 deletions

View File

@@ -133,7 +133,7 @@ class ActivityService:
.where(
and_(
BlockedUser.telegram_id == telegram_id,
BlockedUser.error_type == 'inactive',
BlockedUser.error_type.in_(['inactive', 'blocked_bot', 'deactivated', 'not_found', 'chat_not_found']),
BlockedUser.is_active == True
)
)

View File

@@ -1,6 +1,8 @@
"""Health reflects a live event loop and a usable database connection."""
import asyncio
import logging
import os
import threading
import time
from pathlib import Path
@@ -13,9 +15,12 @@ HEARTBEAT = Path(os.getenv("HEARTBEAT_FILE", "/tmp/lottery-heartbeat"))
async def heartbeat():
while True:
async with async_session_maker() as session:
await session.execute(text("SELECT 1"))
HEARTBEAT.write_text(str(time.time()), encoding="ascii")
async with asyncio.timeout(20):
async with async_session_maker() as session:
await session.execute(text("SELECT 1"))
pending = HEARTBEAT.with_name(HEARTBEAT.name + ".tmp")
pending.write_text(str(time.time()), encoding="ascii")
pending.replace(HEARTBEAT)
await asyncio.sleep(10)
@@ -26,5 +31,20 @@ def healthy():
return False
def start_watchdog(grace_seconds=75, interval=10):
"""Exit even when the event loop or driver cancellation itself is stuck."""
stopped = threading.Event()
started = time.monotonic()
def watch():
while not stopped.wait(interval):
if time.monotonic() - started >= grace_seconds and not healthy():
logging.getLogger(__name__).critical("Heartbeat expired; restarting the application")
os._exit(1)
threading.Thread(target=watch, name="heartbeat-watchdog", daemon=True).start()
return stopped
if __name__ == "__main__":
raise SystemExit(0 if healthy() else 1)

View File

@@ -3,6 +3,7 @@ from src.utils.errors import public_error
from src.core.access import is_admin
from src.utils.delivery import background_delivery
from aiogram import Router, F
from aiogram.exceptions import TelegramBadRequest, TelegramForbiddenError
from aiogram.types import Message, CallbackQuery, InlineKeyboardMarkup, InlineKeyboardButton
from aiogram.fsm.context import FSMContext
from aiogram.fsm.state import State, StatesGroup
@@ -384,10 +385,10 @@ def _is_message_processed(chat_id: int, message_id: int) -> bool:
async def get_all_active_users(session: AsyncSession) -> List:
"""Получить всех пользователей для рассылки (всем, кто когда-либо общался с ботом)"""
users = await UserService.get_all_users(session)
# Рассылаем всем пользователям - и зарегистрированным, и незарегистрированным
# Они все имеют право общаться в чате (главное - что они вошли в чат)
return users
from sqlalchemy import select
from src.core.models import BlockedUser, User
unavailable = select(BlockedUser.telegram_id).where(BlockedUser.is_active.is_(True))
return list((await session.scalars(select(User).where(User.telegram_id.not_in(unavailable)))).all())
async def broadcast_message_with_scheduler(
@@ -415,6 +416,7 @@ async def broadcast_message_with_scheduler(
async with async_session_maker() as session:
users = await get_all_active_users(session)
users = [user for user in users if user.telegram_id != message.bot.id]
logger.info(f"[CHAT] broadcast_message_with_scheduler: всего пользователей для рассылки: {len(users)}")
@@ -502,6 +504,20 @@ async def _copy_with_sender(message: Message, recipient_id: int, sender_info: st
try:
sent = await copy_preserving_entities(message, recipient_id, sender_info or "Участник")
return sent.message_id
except (TelegramForbiddenError, TelegramBadRequest) as error:
error_text = str(error).lower()
unavailable = isinstance(error, TelegramForbiddenError) or any(
reason in error_text for reason in ("chat not found", "user not found", "user is deactivated")
)
if unavailable:
from src.core.broadcast_services import BroadcastService
reason = "blocked_bot" if isinstance(error, TelegramForbiddenError) else "chat_not_found"
async with async_session_maker() as session:
await BroadcastService().mark_user_blocked(session, recipient_id, reason, str(error))
return None
import logging
logging.getLogger(__name__).warning("Telegram rejected chat message content: %s", error)
return None
except Exception:
import logging
logging.getLogger(__name__).exception("Chat delivery failed")