harden deploy reports and admin alerts

app/api/admin.py

@@ -36,8 +36,9 @@ from app.services.admin_notifications import (
     create_admin_notification,
     dismiss_admin_notification,
     mark_admin_notification_read,
+    retry_admin_telegram_notifications,
 )
-from app.services.notifications import notify_user
+from app.services.notifications import notify_user, process_notification_queue
 
 router = APIRouter(prefix="/admin", tags=["admin"])
 
@@ -456,6 +457,35 @@ async def read_all_admin_notifications(
     return {"updated": len(rows)}
 
 
+@router.post("/notifications/retry")
+async def retry_notifications(
+    limit: int = 50,
+    session: AsyncSession = Depends(get_session),
+    current_user: User = Depends(get_current_telegram_user),
+) -> dict[str, Any]:
+    require_admin_access(current_user, FULL_ADMIN_ROLES | {"support"})
+    limit = min(max(limit, 1), 200)
+    service_delivered = await process_notification_queue(session, limit=limit)
+    admin_delivered = await retry_admin_telegram_notifications(session, limit=limit)
+    await log_audit(
+        session,
+        actor=current_user,
+        action="admin.notifications.retry",
+        target_type="notifications",
+        metadata={
+            "limit": limit,
+            "service_delivered": service_delivered,
+            "admin_delivered": admin_delivered,
+        },
+    )
+    await session.commit()
+    return {
+        "service_delivered": service_delivered,
+        "admin_delivered": admin_delivered,
+        "limit": limit,
+    }
+
+
 @router.post("/notifications/{notification_id}/dismiss")
 async def dismiss_notification(
     notification_id: int,

app/api/ocr.py

@@ -1,14 +1,16 @@
 import re
+import time
 from datetime import date
 from decimal import Decimal
 
-from fastapi import APIRouter, Depends, File, Request, UploadFile
+from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
 from pydantic import BaseModel
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.api.deps import get_current_telegram_user
 from app.db.session import get_session
 from app.models.user import User
+from app.services.admin_notifications import create_admin_notification
 from app.services.ocr_provider import get_ocr_provider
 from app.services.rate_limit import check_rate_limit
 from app.services.uploads import SAFE_IMAGE_TYPES, SAFE_TEXT_TYPES, validate_upload
@@ -40,6 +42,72 @@ class OCRResultRead(BaseModel):
     provider: str = "heuristic"
 
 
+async def validate_ocr_upload(
+    *,
+    session: AsyncSession,
+    current_user: User,
+    content: bytes,
+    filename: str | None,
+    content_type: str | None,
+) -> str:
+    try:
+        return validate_upload(
+            content=content,
+            filename=filename,
+            content_type=content_type,
+            max_bytes=MAX_OCR_FILE_BYTES,
+            allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES,
+        )
+    except HTTPException as exc:
+        await create_admin_notification(
+            session,
+            event_type="upload_blocked",
+            title="Upload blocked",
+            body=f"OCR upload blocked: {filename or '-'}\nReason: {exc.detail}",
+            entity_type="user",
+            entity_id=current_user.id,
+            severity="warning",
+            idempotency_key=(
+                f"upload_blocked:{current_user.id}:{filename or 'upload'}:{exc.status_code}:"
+                f"{int(time.time() // 60)}"
+            ),
+            metadata={
+                "filename": filename,
+                "content_type": content_type,
+                "status_code": exc.status_code,
+                "detail": exc.detail,
+            },
+        )
+        await session.commit()
+        raise
+
+
+async def recognize_with_alert(
+    *,
+    session: AsyncSession,
+    current_user: User,
+    content: bytes,
+    filename: str | None,
+    scope: str,
+):
+    try:
+        return await get_ocr_provider().recognize(content, filename)
+    except Exception as exc:  # noqa: BLE001 - OCR must fail gracefully and alert admins
+        await create_admin_notification(
+            session,
+            event_type="ocr_failed",
+            title="OCR provider failed",
+            body=f"Scope: {scope}\nFile: {filename or '-'}\nError: {type(exc).__name__}",
+            entity_type="user",
+            entity_id=current_user.id,
+            severity="error",
+            idempotency_key=f"ocr_failed:{scope}:{current_user.id}:{int(time.time() // 60)}",
+            metadata={"scope": scope, "filename": filename, "error_type": type(exc).__name__},
+        )
+        await session.commit()
+        return None
+
+
 @router.post("/parse-text-receipt", response_model=ReceiptSuggestion)
 async def parse_text_receipt(
     request: Request,
@@ -49,17 +117,23 @@ async def parse_text_receipt(
 ) -> ReceiptSuggestion:
     await check_rate_limit(scope="ocr", limit=10, window_seconds=60, request=request, user=current_user, session=session)
     content = await file.read()
-    validate_upload(
-        content=content,
+    await validate_ocr_upload(
+        session=session,
+        current_user=current_user,
         filename=file.filename,
         content_type=file.content_type,
-        max_bytes=MAX_OCR_FILE_BYTES,
-        allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES,
+        content=content,
     )
     content_type = (file.content_type or "").lower()
     if content_type.startswith("image/") or content_type == "application/pdf":
-        result = await get_ocr_provider().recognize(content, file.filename)
-        if not result.recognized_text:
+        result = await recognize_with_alert(
+            session=session,
+            current_user=current_user,
+            content=content,
+            filename=file.filename,
+            scope="parse_text_receipt",
+        )
+        if not result or not result.recognized_text:
             return ReceiptSuggestion(
                 confidence=0,
                 message="Не удалось уверенно распознать чек. Открылся ручной ввод: проверьте дату, сумму, литры и цену.",
@@ -133,8 +207,22 @@ async def recognize_license_plate(
 ) -> OCRResultRead:
     await check_rate_limit(scope="ocr_license_plate", limit=8, window_seconds=60, request=request, user=current_user, session=session)
     content = await file.read()
-    validate_upload(content=content, filename=file.filename, content_type=file.content_type, max_bytes=MAX_OCR_FILE_BYTES, allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES)
-    result = await get_ocr_provider().recognize(content, file.filename)
+    await validate_ocr_upload(
+        session=session,
+        current_user=current_user,
+        content=content,
+        filename=file.filename,
+        content_type=file.content_type,
+    )
+    result = await recognize_with_alert(
+        session=session,
+        current_user=current_user,
+        content=content,
+        filename=file.filename,
+        scope="license_plate",
+    )
+    if result is None:
+        return OCRResultRead(recognized_text="", candidates=[], provider="error")
     return OCRResultRead(
         recognized_text=result.recognized_text,
         candidates=[OCRCandidateRead(**item.__dict__) for item in result.candidates if item.type == "license_plate"],
@@ -151,8 +239,22 @@ async def recognize_vin(
 ) -> OCRResultRead:
     await check_rate_limit(scope="ocr_vin", limit=8, window_seconds=60, request=request, user=current_user, session=session)
     content = await file.read()
-    validate_upload(content=content, filename=file.filename, content_type=file.content_type, max_bytes=MAX_OCR_FILE_BYTES, allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES)
-    result = await get_ocr_provider().recognize(content, file.filename)
+    await validate_ocr_upload(
+        session=session,
+        current_user=current_user,
+        content=content,
+        filename=file.filename,
+        content_type=file.content_type,
+    )
+    result = await recognize_with_alert(
+        session=session,
+        current_user=current_user,
+        content=content,
+        filename=file.filename,
+        scope="vin",
+    )
+    if result is None:
+        return OCRResultRead(recognized_text="", candidates=[], provider="error")
     return OCRResultRead(
         recognized_text=result.recognized_text,
         candidates=[OCRCandidateRead(**item.__dict__) for item in result.candidates if item.type == "vin"],
@@ -169,8 +271,22 @@ async def recognize_service_document(
 ) -> OCRResultRead:
     await check_rate_limit(scope="ocr_service_document", limit=8, window_seconds=60, request=request, user=current_user, session=session)
     content = await file.read()
-    validate_upload(content=content, filename=file.filename, content_type=file.content_type, max_bytes=MAX_OCR_FILE_BYTES, allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES)
-    result = await get_ocr_provider().recognize(content, file.filename)
+    await validate_ocr_upload(
+        session=session,
+        current_user=current_user,
+        content=content,
+        filename=file.filename,
+        content_type=file.content_type,
+    )
+    result = await recognize_with_alert(
+        session=session,
+        current_user=current_user,
+        content=content,
+        filename=file.filename,
+        scope="service_document",
+    )
+    if result is None:
+        return OCRResultRead(recognized_text="", candidates=[], provider="error")
     return OCRResultRead(
         recognized_text=result.recognized_text,
         candidates=[OCRCandidateRead(**item.__dict__) for item in result.candidates],

app/main.py

@@ -25,7 +25,8 @@ from app.api import (
     work_orders,
 )
 from app.core.config import settings
-from app.db.session import get_session
+from app.db.session import async_session_factory, get_session
+from app.services.admin_notifications import create_admin_notification
 from app.services.rate_limit import get_redis_client
 
 
@@ -49,8 +50,29 @@ async def production_headers_and_metrics(request: Request, call_next):
     start = monotonic()
     try:
         response = await call_next(request)
-    except Exception:
+    except Exception as exc:
         REQUEST_ERRORS += 1
+        try:
+            async with async_session_factory() as session:
+                await create_admin_notification(
+                    session,
+                    event_type="system_error",
+                    title="Unhandled API error",
+                    body=f"{request.method} {request.url.path}\nError: {type(exc).__name__}",
+                    entity_type="system",
+                    entity_id=request.url.path,
+                    severity="error",
+                    idempotency_key=f"system_error:{request.url.path}:{type(exc).__name__}:{int(start // 60)}",
+                    metadata={
+                        "path": request.url.path,
+                        "method": request.method,
+                        "request_id": request_id,
+                        "error_type": type(exc).__name__,
+                    },
+                )
+                await session.commit()
+        except Exception:
+            pass
         raise
     duration = monotonic() - start
     REQUEST_COUNT += 1

app/services/admin_notifications.py

@@ -132,6 +132,22 @@ async def send_admin_telegram_notification(notification: AdminNotification) -> N
         notification.telegram_status = "sent"
 
 
+async def retry_admin_telegram_notifications(session: AsyncSession, *, limit: int = 50) -> int:
+    result = await session.execute(
+        select(AdminNotification)
+        .where(AdminNotification.telegram_status.in_(["pending", "failed"]))
+        .order_by(AdminNotification.created_at.asc())
+        .limit(limit)
+    )
+    delivered = 0
+    for notification in result.scalars():
+        await send_admin_telegram_notification(notification)
+        if notification.telegram_status == "sent":
+            delivered += 1
+    await session.commit()
+    return delivered
+
+
 async def mark_admin_notification_read(
     session: AsyncSession, notification: AdminNotification
 ) -> AdminNotification:

app/services/rate_limit.py

@@ -41,7 +41,13 @@ async def check_rate_limit(
     if settings.redis_url:
         allowed = await check_redis_rate_limit(scope, identifiers, limit, window_seconds)
         if not allowed:
-            await log_rate_limit_event(session, scope=scope, identifier="redis")
+            await log_rate_limit_event(
+                session,
+                scope=scope,
+                identifier="redis",
+                user=user,
+                request=request,
+            )
             raise_rate_limit(scope, window_seconds)
         return
 
@@ -52,7 +58,13 @@ async def check_rate_limit(
         while bucket and now - bucket[0] > window_seconds:
             bucket.popleft()
         if len(bucket) >= limit:
-            await log_rate_limit_event(session, scope=scope, identifier=str(identifier))
+            await log_rate_limit_event(
+                session,
+                scope=scope,
+                identifier=str(identifier),
+                user=user,
+                request=request,
+            )
             raise_rate_limit(scope, window_seconds)
     for identifier in identifiers:
         _buckets[(scope, identifier)].append(now)
@@ -107,18 +119,82 @@ async def log_rate_limit_event(
     *,
     scope: str,
     identifier: str,
+    user: User | None = None,
+    request: Request | None = None,
 ) -> None:
-    if session is None:
-        return
-    from app.models.car import AuditLog
+    client_host = request.client.host if request and request.client else None
+    user_agent = request.headers.get("user-agent") if request else None
+    metadata = {
+        "scope": scope,
+        "identifier": identifier,
+        "telegram_id": user.telegram_id if user else None,
+        "user_id": user.id if user else None,
+        "ip": client_host,
+    }
 
-    session.add(
-        AuditLog(
-            actor_user_id=None,
-            actor_role="system",
-            action="rate_limit.exceeded",
-            target_type=scope,
-            target_id=identifier[:80],
-            metadata_json={"scope": scope, "identifier": identifier},
-        )
+    if session is None:
+        from app.db.session import async_session_factory
+
+        async with async_session_factory() as event_session:
+            await persist_rate_limit_event(
+                event_session,
+                scope=scope,
+                identifier=identifier,
+                user=user,
+                client_host=client_host,
+                user_agent=user_agent,
+                metadata=metadata,
+            )
+        return
+
+    await persist_rate_limit_event(
+        session,
+        scope=scope,
+        identifier=identifier,
+        user=user,
+        client_host=client_host,
+        user_agent=user_agent,
+        metadata=metadata,
     )
+
+
+async def persist_rate_limit_event(
+    event_session: AsyncSession,
+    *,
+    scope: str,
+    identifier: str,
+    user: User | None,
+    client_host: str | None,
+    user_agent: str | None,
+    metadata: dict,
+) -> None:
+    from app.models.car import AuditLog
+    from app.services.admin_notifications import create_admin_notification
+
+    try:
+        event_session.add(
+            AuditLog(
+                actor_user_id=user.id if user else None,
+                actor_role=user.platform_role if user else "system",
+                action="rate_limit.exceeded",
+                target_type=scope,
+                target_id=identifier[:80],
+                metadata_json=metadata,
+                ip=client_host,
+                user_agent=user_agent[:256] if user_agent else None,
+            )
+        )
+        await create_admin_notification(
+            event_session,
+            event_type="rate_limit_exceeded",
+            title="Rate limit exceeded",
+            body=f"Scope: {scope}\nIdentifier: {identifier}",
+            entity_type="user" if user else "system",
+            entity_id=user.id if user else scope,
+            severity="warning",
+            idempotency_key=f"rate_limit:{scope}:{identifier}:{int(time.time() // max(60, 1))}",
+            metadata=metadata,
+        )
+        await event_session.commit()
+    except Exception:
+        await event_session.rollback()