harden deploy reports and admin alerts

2026-05-18 18:17:53 +09:00
parent 2d5695fdce
commit 22b9b40d78
12 changed files with 549 additions and 31 deletions
--- a/ADMIN.md
+++ b/ADMIN.md
@@ -53,6 +53,7 @@ Notifications:
 - `GET /api/admin/notifications`
 - `POST /api/admin/notifications/{id}/read`
 - `POST /api/admin/notifications/read-all`
+- `POST /api/admin/notifications/retry`
 - `POST /api/admin/notifications/{id}/dismiss`

 Data Explorer:
@@ -153,3 +154,18 @@ Data Explorer работает только по whitelist источников
 - `/admin_alerts`

 API дополнительно проверяет роль пользователя, поэтому команда не дает доступа без admin-role в БД.
+
+## Deploy Reports
+
+Для временного rsync-деплоя есть `scripts/rsync_deploy.sh`. Скрипт:
+
+- запускает локальные `ruff` и `pytest`;
+- отправляет Telegram progress/failure/success отчеты;
+- делает remote code backup без `.env`;
+- синхронизирует код через `rsync`;
+- собирает Docker images;
+- применяет Alembic migrations;
+- поднимает `api` и `bot`;
+- проверяет `/health`, `/ready`, `/metrics`, `/admin.html`, `/sto.html`, `/work_order.html`.
+
+Утилита `scripts/send_telegram_report.py` берет получателей из `ADMIN_NOTIFICATION_CHAT_ID`, `ADMIN_TELEGRAM_IDS` и, если env пустой, из пользователей БД с ролями `admin`, `super_admin`, `moderator`, `support`.
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -36,8 +36,9 @@ from app.services.admin_notifications import (
    create_admin_notification,
    dismiss_admin_notification,
    mark_admin_notification_read,
+    retry_admin_telegram_notifications,
 )
-from app.services.notifications import notify_user
+from app.services.notifications import notify_user, process_notification_queue

 router = APIRouter(prefix="/admin", tags=["admin"])

@@ -456,6 +457,35 @@ async def read_all_admin_notifications(
    return {"updated": len(rows)}


+@router.post("/notifications/retry")
+async def retry_notifications(
+    limit: int = 50,
+    session: AsyncSession = Depends(get_session),
+    current_user: User = Depends(get_current_telegram_user),
+) -> dict[str, Any]:
+    require_admin_access(current_user, FULL_ADMIN_ROLES | {"support"})
+    limit = min(max(limit, 1), 200)
+    service_delivered = await process_notification_queue(session, limit=limit)
+    admin_delivered = await retry_admin_telegram_notifications(session, limit=limit)
+    await log_audit(
+        session,
+        actor=current_user,
+        action="admin.notifications.retry",
+        target_type="notifications",
+        metadata={
+            "limit": limit,
+            "service_delivered": service_delivered,
+            "admin_delivered": admin_delivered,
+        },
+    )
+    await session.commit()
+    return {
+        "service_delivered": service_delivered,
+        "admin_delivered": admin_delivered,
+        "limit": limit,
+    }
+
+
@router.post("/notifications/{notification_id}/dismiss")
 async def dismiss_notification(
    notification_id: int,
--- a/app/api/ocr.py
+++ b/app/api/ocr.py
@@ -1,14 +1,16 @@
 import re
+import time
 from datetime import date
 from decimal import Decimal

-from fastapi import APIRouter, Depends, File, Request, UploadFile
+from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
 from pydantic import BaseModel
 from sqlalchemy.ext.asyncio import AsyncSession

 from app.api.deps import get_current_telegram_user
 from app.db.session import get_session
 from app.models.user import User
+from app.services.admin_notifications import create_admin_notification
 from app.services.ocr_provider import get_ocr_provider
 from app.services.rate_limit import check_rate_limit
 from app.services.uploads import SAFE_IMAGE_TYPES, SAFE_TEXT_TYPES, validate_upload
@@ -40,6 +42,72 @@ class OCRResultRead(BaseModel):
    provider: str = "heuristic"


+async def validate_ocr_upload(
+    *,
+    session: AsyncSession,
+    current_user: User,
+    content: bytes,
+    filename: str | None,
+    content_type: str | None,
+) -> str:
+    try:
+        return validate_upload(
+            content=content,
+            filename=filename,
+            content_type=content_type,
+            max_bytes=MAX_OCR_FILE_BYTES,
+            allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES,
+        )
+    except HTTPException as exc:
+        await create_admin_notification(
+            session,
+            event_type="upload_blocked",
+            title="Upload blocked",
+            body=f"OCR upload blocked: {filename or '-'}\nReason: {exc.detail}",
+            entity_type="user",
+            entity_id=current_user.id,
+            severity="warning",
+            idempotency_key=(
+                f"upload_blocked:{current_user.id}:{filename or 'upload'}:{exc.status_code}:"
+                f"{int(time.time() // 60)}"
+            ),
+            metadata={
+                "filename": filename,
+                "content_type": content_type,
+                "status_code": exc.status_code,
+                "detail": exc.detail,
+            },
+        )
+        await session.commit()
+        raise
+
+
+async def recognize_with_alert(
+    *,
+    session: AsyncSession,
+    current_user: User,
+    content: bytes,
+    filename: str | None,
+    scope: str,
+):
+    try:
+        return await get_ocr_provider().recognize(content, filename)
+    except Exception as exc:  # noqa: BLE001 - OCR must fail gracefully and alert admins
+        await create_admin_notification(
+            session,
+            event_type="ocr_failed",
+            title="OCR provider failed",
+            body=f"Scope: {scope}\nFile: {filename or '-'}\nError: {type(exc).__name__}",
+            entity_type="user",
+            entity_id=current_user.id,
+            severity="error",
+            idempotency_key=f"ocr_failed:{scope}:{current_user.id}:{int(time.time() // 60)}",
+            metadata={"scope": scope, "filename": filename, "error_type": type(exc).__name__},
+        )
+        await session.commit()
+        return None
+
+
@router.post("/parse-text-receipt", response_model=ReceiptSuggestion)
 async def parse_text_receipt(
    request: Request,
@@ -49,17 +117,23 @@ async def parse_text_receipt(
 ) -> ReceiptSuggestion:
    await check_rate_limit(scope="ocr", limit=10, window_seconds=60, request=request, user=current_user, session=session)
    content = await file.read()
-    validate_upload(
-        content=content,
+    await validate_ocr_upload(
+        session=session,
+        current_user=current_user,
        filename=file.filename,
        content_type=file.content_type,
-        max_bytes=MAX_OCR_FILE_BYTES,
-        allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES,
+        content=content,
    )
    content_type = (file.content_type or "").lower()
    if content_type.startswith("image/") or content_type == "application/pdf":
-        result = await get_ocr_provider().recognize(content, file.filename)
-        if not result.recognized_text:
+        result = await recognize_with_alert(
+            session=session,
+            current_user=current_user,
+            content=content,
+            filename=file.filename,
+            scope="parse_text_receipt",
+        )
+        if not result or not result.recognized_text:
            return ReceiptSuggestion(
                confidence=0,
                message="Не удалось уверенно распознать чек. Открылся ручной ввод: проверьте дату, сумму, литры и цену.",
@@ -133,8 +207,22 @@ async def recognize_license_plate(
 ) -> OCRResultRead:
    await check_rate_limit(scope="ocr_license_plate", limit=8, window_seconds=60, request=request, user=current_user, session=session)
    content = await file.read()
-    validate_upload(content=content, filename=file.filename, content_type=file.content_type, max_bytes=MAX_OCR_FILE_BYTES, allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES)
-    result = await get_ocr_provider().recognize(content, file.filename)
+    await validate_ocr_upload(
+        session=session,
+        current_user=current_user,
+        content=content,
+        filename=file.filename,
+        content_type=file.content_type,
+    )
+    result = await recognize_with_alert(
+        session=session,
+        current_user=current_user,
+        content=content,
+        filename=file.filename,
+        scope="license_plate",
+    )
+    if result is None:
+        return OCRResultRead(recognized_text="", candidates=[], provider="error")
    return OCRResultRead(
        recognized_text=result.recognized_text,
        candidates=[OCRCandidateRead(**item.__dict__) for item in result.candidates if item.type == "license_plate"],
@@ -151,8 +239,22 @@ async def recognize_vin(
 ) -> OCRResultRead:
    await check_rate_limit(scope="ocr_vin", limit=8, window_seconds=60, request=request, user=current_user, session=session)
    content = await file.read()
-    validate_upload(content=content, filename=file.filename, content_type=file.content_type, max_bytes=MAX_OCR_FILE_BYTES, allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES)
-    result = await get_ocr_provider().recognize(content, file.filename)
+    await validate_ocr_upload(
+        session=session,
+        current_user=current_user,
+        content=content,
+        filename=file.filename,
+        content_type=file.content_type,
+    )
+    result = await recognize_with_alert(
+        session=session,
+        current_user=current_user,
+        content=content,
+        filename=file.filename,
+        scope="vin",
+    )
+    if result is None:
+        return OCRResultRead(recognized_text="", candidates=[], provider="error")
    return OCRResultRead(
        recognized_text=result.recognized_text,
        candidates=[OCRCandidateRead(**item.__dict__) for item in result.candidates if item.type == "vin"],
@@ -169,8 +271,22 @@ async def recognize_service_document(
 ) -> OCRResultRead:
    await check_rate_limit(scope="ocr_service_document", limit=8, window_seconds=60, request=request, user=current_user, session=session)
    content = await file.read()
-    validate_upload(content=content, filename=file.filename, content_type=file.content_type, max_bytes=MAX_OCR_FILE_BYTES, allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES)
-    result = await get_ocr_provider().recognize(content, file.filename)
+    await validate_ocr_upload(
+        session=session,
+        current_user=current_user,
+        content=content,
+        filename=file.filename,
+        content_type=file.content_type,
+    )
+    result = await recognize_with_alert(
+        session=session,
+        current_user=current_user,
+        content=content,
+        filename=file.filename,
+        scope="service_document",
+    )
+    if result is None:
+        return OCRResultRead(recognized_text="", candidates=[], provider="error")
    return OCRResultRead(
        recognized_text=result.recognized_text,
        candidates=[OCRCandidateRead(**item.__dict__) for item in result.candidates],
--- a/app/main.py
+++ b/app/main.py
@@ -25,7 +25,8 @@ from app.api import (
    work_orders,
 )
 from app.core.config import settings
-from app.db.session import get_session
+from app.db.session import async_session_factory, get_session
+from app.services.admin_notifications import create_admin_notification
 from app.services.rate_limit import get_redis_client


@@ -49,8 +50,29 @@ async def production_headers_and_metrics(request: Request, call_next):
    start = monotonic()
    try:
        response = await call_next(request)
-    except Exception:
+    except Exception as exc:
        REQUEST_ERRORS += 1
+        try:
+            async with async_session_factory() as session:
+                await create_admin_notification(
+                    session,
+                    event_type="system_error",
+                    title="Unhandled API error",
+                    body=f"{request.method} {request.url.path}\nError: {type(exc).__name__}",
+                    entity_type="system",
+                    entity_id=request.url.path,
+                    severity="error",
+                    idempotency_key=f"system_error:{request.url.path}:{type(exc).__name__}:{int(start // 60)}",
+                    metadata={
+                        "path": request.url.path,
+                        "method": request.method,
+                        "request_id": request_id,
+                        "error_type": type(exc).__name__,
+                    },
+                )
+                await session.commit()
+        except Exception:
+            pass
        raise
    duration = monotonic() - start
    REQUEST_COUNT += 1
--- a/app/services/admin_notifications.py
+++ b/app/services/admin_notifications.py
@@ -132,6 +132,22 @@ async def send_admin_telegram_notification(notification: AdminNotification) -> N
        notification.telegram_status = "sent"


+async def retry_admin_telegram_notifications(session: AsyncSession, *, limit: int = 50) -> int:
+    result = await session.execute(
+        select(AdminNotification)
+        .where(AdminNotification.telegram_status.in_(["pending", "failed"]))
+        .order_by(AdminNotification.created_at.asc())
+        .limit(limit)
+    )
+    delivered = 0
+    for notification in result.scalars():
+        await send_admin_telegram_notification(notification)
+        if notification.telegram_status == "sent":
+            delivered += 1
+    await session.commit()
+    return delivered
+
+
 async def mark_admin_notification_read(
    session: AsyncSession, notification: AdminNotification
 ) -> AdminNotification:
--- a/app/services/rate_limit.py
+++ b/app/services/rate_limit.py
@@ -41,7 +41,13 @@ async def check_rate_limit(
    if settings.redis_url:
        allowed = await check_redis_rate_limit(scope, identifiers, limit, window_seconds)
        if not allowed:
-            await log_rate_limit_event(session, scope=scope, identifier="redis")
+            await log_rate_limit_event(
+                session,
+                scope=scope,
+                identifier="redis",
+                user=user,
+                request=request,
+            )
            raise_rate_limit(scope, window_seconds)
        return

@@ -52,7 +58,13 @@ async def check_rate_limit(
        while bucket and now - bucket[0] > window_seconds:
            bucket.popleft()
        if len(bucket) >= limit:
-            await log_rate_limit_event(session, scope=scope, identifier=str(identifier))
+            await log_rate_limit_event(
+                session,
+                scope=scope,
+                identifier=str(identifier),
+                user=user,
+                request=request,
+            )
            raise_rate_limit(scope, window_seconds)
    for identifier in identifiers:
        _buckets[(scope, identifier)].append(now)
@@ -107,18 +119,82 @@ async def log_rate_limit_event(
    *,
    scope: str,
    identifier: str,
+    user: User | None = None,
+    request: Request | None = None,
 ) -> None:
-    if session is None:
-        return
-    from app.models.car import AuditLog
+    client_host = request.client.host if request and request.client else None
+    user_agent = request.headers.get("user-agent") if request else None
+    metadata = {
+        "scope": scope,
+        "identifier": identifier,
+        "telegram_id": user.telegram_id if user else None,
+        "user_id": user.id if user else None,
+        "ip": client_host,
+    }

-    session.add(
+    if session is None:
+        from app.db.session import async_session_factory
+
+        async with async_session_factory() as event_session:
+            await persist_rate_limit_event(
+                event_session,
+                scope=scope,
+                identifier=identifier,
+                user=user,
+                client_host=client_host,
+                user_agent=user_agent,
+                metadata=metadata,
+            )
+        return
+
+    await persist_rate_limit_event(
+        session,
+        scope=scope,
+        identifier=identifier,
+        user=user,
+        client_host=client_host,
+        user_agent=user_agent,
+        metadata=metadata,
+    )
+
+
+async def persist_rate_limit_event(
+    event_session: AsyncSession,
+    *,
+    scope: str,
+    identifier: str,
+    user: User | None,
+    client_host: str | None,
+    user_agent: str | None,
+    metadata: dict,
+) -> None:
+    from app.models.car import AuditLog
+    from app.services.admin_notifications import create_admin_notification
+
+    try:
+        event_session.add(
            AuditLog(
-            actor_user_id=None,
-            actor_role="system",
+                actor_user_id=user.id if user else None,
+                actor_role=user.platform_role if user else "system",
                action="rate_limit.exceeded",
                target_type=scope,
                target_id=identifier[:80],
-            metadata_json={"scope": scope, "identifier": identifier},
+                metadata_json=metadata,
+                ip=client_host,
+                user_agent=user_agent[:256] if user_agent else None,
            )
        )
+        await create_admin_notification(
+            event_session,
+            event_type="rate_limit_exceeded",
+            title="Rate limit exceeded",
+            body=f"Scope: {scope}\nIdentifier: {identifier}",
+            entity_type="user" if user else "system",
+            entity_id=user.id if user else scope,
+            severity="warning",
+            idempotency_key=f"rate_limit:{scope}:{identifier}:{int(time.time() // max(60, 1))}",
+            metadata=metadata,
+        )
+        await event_session.commit()
+    except Exception:
+        await event_session.rollback()
--- a/scripts/rsync_deploy.sh
+++ b/scripts/rsync_deploy.sh
@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+REMOTE="${REMOTE:-root@drivers.smartsoltech.kr}"
+REMOTE_DIR="${REMOTE_DIR:-/opt/drivers_bot}"
+BASE_URL="${BASE_URL:-http://127.0.0.1:8000}"
+COMPOSE="${COMPOSE:-docker compose}"
+RUN_LOCAL_CHECKS="${RUN_LOCAL_CHECKS:-true}"
+BACKUP_BEFORE_DEPLOY="${BACKUP_BEFORE_DEPLOY:-true}"
+BRANCH="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)"
+REVISION="$(git rev-parse --short HEAD 2>/dev/null || echo unknown)"
+
+EXCLUDES=(
+  "--exclude=.git/"
+  "--exclude=.env"
+  "--exclude=.env.*"
+  "--exclude=.venv/"
+  "--exclude=venv/"
+  "--exclude=__pycache__/"
+  "--exclude=.pytest_cache/"
+  "--exclude=.ruff_cache/"
+  "--exclude=.history/"
+  "--exclude=backups/"
+  "--exclude=*.sqlite"
+  "--exclude=*.sqlite3"
+  "--exclude=*.db"
+)
+
+send_remote_report() {
+  local text="$1"
+  ssh "$REMOTE" "cd '$REMOTE_DIR' && CARPASS_REPORT_TEXT=\$(cat) $COMPOSE exec -T -e CARPASS_REPORT_TEXT api python scripts/send_telegram_report.py" <<<"$text" || true
+}
+
+fail_report() {
+  local line="${1:-unknown}"
+  send_remote_report "❌ CarPass rsync deploy failed
+Branch: $BRANCH
+Revision: $REVISION
+Step line: $line
+Target: $REMOTE"
+}
+
+trap 'fail_report "$LINENO"' ERR
+
+if [[ "$RUN_LOCAL_CHECKS" == "true" ]]; then
+  echo "Running local checks..."
+  .venv/bin/ruff check app bot tests
+  .venv/bin/pytest -q
+fi
+
+send_remote_report "🚀 CarPass rsync deploy started
+Branch: $BRANCH
+Revision: $REVISION
+Target: $REMOTE
+Checks: local=${RUN_LOCAL_CHECKS}"
+
+echo "Checking remote..."
+ssh "$REMOTE" "test -d '$REMOTE_DIR' && test -f '$REMOTE_DIR/docker-compose.yml'"
+
+if [[ "$BACKUP_BEFORE_DEPLOY" == "true" ]]; then
+  echo "Creating remote code backup..."
+  ssh "$REMOTE" "cd '$(dirname "$REMOTE_DIR")' && mkdir -p '$REMOTE_DIR/backups' && tar --exclude='$(basename "$REMOTE_DIR")/backups' --exclude='$(basename "$REMOTE_DIR")/.git' --exclude='$(basename "$REMOTE_DIR")/.env' --exclude='$(basename "$REMOTE_DIR")/.venv' -czf '$REMOTE_DIR/backups/code_pre_rsync_$(date +%Y%m%d%H%M%S).tgz' '$(basename "$REMOTE_DIR")'"
+fi
+
+echo "Syncing code with rsync..."
+rsync -az --delete "${EXCLUDES[@]}" ./ "$REMOTE:$REMOTE_DIR/"
+
+echo "Building remote images..."
+ssh "$REMOTE" "cd '$REMOTE_DIR' && $COMPOSE build"
+send_remote_report "🧱 CarPass rsync deploy progress
+Branch: $BRANCH
+Step: docker build completed"
+
+echo "Applying migrations..."
+ssh "$REMOTE" "cd '$REMOTE_DIR' && $COMPOSE run --rm api alembic upgrade head"
+send_remote_report "🗄️ CarPass rsync deploy progress
+Branch: $BRANCH
+Step: migrations applied"
+
+echo "Starting services..."
+ssh "$REMOTE" "cd '$REMOTE_DIR' && $COMPOSE up -d"
+
+echo "Waiting for API readiness..."
+ssh "$REMOTE" "cd '$REMOTE_DIR' && for i in \$(seq 1 30); do status=\$(docker inspect -f '{{.State.Health.Status}}' drivers_bot-api-1 2>/dev/null || echo missing); echo \"api_health=\$status\"; [ \"\$status\" = healthy ] && exit 0; sleep 2; done; $COMPOSE logs --tail=120 api; exit 1"
+
+echo "Running remote smoke tests..."
+ssh "$REMOTE" "cd '$REMOTE_DIR' && BASE_URL='$BASE_URL' ./scripts/smoke_test.sh && curl -fsSI '$BASE_URL/admin.html' | head -5 && $COMPOSE ps"
+
+send_remote_report "✅ CarPass rsync deploy completed
+Branch: $BRANCH
+Revision: $REVISION
+Migration: $(ssh "$REMOTE" "cd '$REMOTE_DIR' && curl -fsS '$BASE_URL/ready'" | tr '\n' ' ')
+Checks: /health ok, /ready ok, /metrics ok, /admin.html 200
+Services: api healthy, bot restarted"
+
+echo "Deploy completed."
--- a/scripts/send_telegram_report.py
+++ b/scripts/send_telegram_report.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import asyncio
+import os
+from collections.abc import Iterable
+
+import httpx
+from sqlalchemy import select
+
+from app.core.config import settings
+from app.db.session import async_session_factory
+from app.models import car, expense, gamification, push  # noqa: F401
+from app.models.user import User
+
+REPORT_ROLES = {"admin", "super_admin", "moderator", "support"}
+
+
+def env_recipients() -> list[str]:
+    recipients: list[str] = []
+    if settings.admin_notification_chat_id:
+        recipients.append(settings.admin_notification_chat_id)
+    recipients.extend(str(item) for item in settings.admin_telegram_id_list)
+    return recipients
+
+
+async def db_recipients() -> list[str]:
+    async with async_session_factory() as session:
+        result = await session.execute(
+            select(User.telegram_id).where(User.platform_role.in_(REPORT_ROLES))
+        )
+        return [str(row[0]) for row in result.all() if row[0]]
+
+
+def unique(values: Iterable[str]) -> list[str]:
+    return list(dict.fromkeys(item.strip() for item in values if item and item.strip()))
+
+
+async def send_report(text: str, *, dry_run: bool = False) -> int:
+    recipients = unique([*env_recipients(), *(await db_recipients())])
+    if dry_run:
+        print(f"telegram_report_dry_run recipients={len(recipients)}")
+        return len(recipients)
+    if not settings.bot_token or not recipients:
+        print("telegram_report_skipped")
+        return 0
+
+    sent = 0
+    async with httpx.AsyncClient(timeout=10) as client:
+        for chat_id in recipients:
+            try:
+                response = await client.post(
+                    f"https://api.telegram.org/bot{settings.bot_token}/sendMessage",
+                    json={"chat_id": chat_id, "text": text, "disable_web_page_preview": True},
+                )
+                response.raise_for_status()
+                sent += 1
+            except Exception as exc:  # noqa: BLE001 - deploy report must never fail deploy
+                print(f"telegram_report_failed chat_id={chat_id} error={type(exc).__name__}")
+    print(f"telegram_report_sent_count {sent}")
+    return sent
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Send a CarPass operational Telegram report.")
+    parser.add_argument("--text", help="Report text. Defaults to CARPASS_REPORT_TEXT.")
+    parser.add_argument("--dry-run", action="store_true")
+    return parser.parse_args()
+
+
+async def main() -> None:
+    args = parse_args()
+    text = args.text or os.getenv("CARPASS_REPORT_TEXT") or ""
+    if not text.strip():
+        raise SystemExit("Report text is required")
+    await send_report(text, dry_run=args.dry_run)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
--- a/scripts/smoke_test.sh
+++ b/scripts/smoke_test.sh
@@ -13,4 +13,10 @@ echo

 echo "Checking metrics..."
 curl -fsS "$BASE_URL/metrics" | grep -q "carpass_requests_total"
+
+for path in / /sto.html /admin.html /work_order.html; do
+  echo "Checking static page $path..."
+  curl -fsSI "$BASE_URL$path" | grep -q "200 OK"
+done
+
 echo "Smoke test passed."
--- a/tests/test_admin_control_center.py
+++ b/tests/test_admin_control_center.py
@@ -247,3 +247,54 @@ async def test_pending_sto_queue_and_approve_audit(
    assert approved.status_code == 200
    assert approved.json()["verification_status"] == "approved"
    assert any(item["action"] == "service_center.verify" for item in audit.json())
+
+
+@pytest.mark.asyncio
+async def test_blocked_ocr_upload_creates_admin_notification(
+    client, auth_headers, admin_auth_headers, internal_headers
+) -> None:
+    await ensure_admin(client, internal_headers)
+
+    response = await client.post(
+        "/api/ocr/vin",
+        headers=auth_headers,
+        files={"file": ("invoice.exe", b"not an image", "image/jpeg")},
+    )
+    notifications = await client.get("/api/admin/notifications?limit=100", headers=admin_auth_headers)
+
+    assert response.status_code == 415
+    assert any(item["event_type"] == "upload_blocked" for item in notifications.json()["rows"])
+
+
+@pytest.mark.asyncio
+async def test_rate_limit_creates_admin_notification(
+    client, auth_headers, admin_auth_headers, internal_headers
+) -> None:
+    await ensure_admin(client, internal_headers)
+
+    last_response = None
+    for index in range(9):
+        last_response = await client.post(
+            "/api/ocr/vin",
+            headers=auth_headers,
+            files={"file": (f"vin-{index}.txt", b"VIN KMHCT41BAHU123456", "text/plain")},
+        )
+    notifications = await client.get("/api/admin/notifications?limit=100", headers=admin_auth_headers)
+
+    assert last_response is not None
+    assert last_response.status_code == 429
+    assert any(item["event_type"] == "rate_limit_exceeded" for item in notifications.json()["rows"])
+
+
+@pytest.mark.asyncio
+async def test_admin_can_retry_notification_queues(
+    client, admin_auth_headers, internal_headers
+) -> None:
+    await ensure_admin(client, internal_headers)
+
+    response = await client.post("/api/admin/notifications/retry", headers=admin_auth_headers)
+    audit = await client.get("/api/admin/audit-log?action=admin.notifications.retry", headers=admin_auth_headers)
+
+    assert response.status_code == 200
+    assert {"service_delivered", "admin_delivered", "limit"} <= response.json().keys()
+    assert any(item["action"] == "admin.notifications.retry" for item in audit.json())
--- a/web/admin.html
+++ b/web/admin.html
@@ -84,8 +84,11 @@
            <p class="eyebrow">События</p>
            <h2>Admin notifications</h2>
          </div>
+          <div class="row-actions">
+            <button type="button" id="retryNotificationsBtn">Retry</button>
            <button type="button" id="readAllBtn">Прочитать все</button>
          </div>
+        </div>
        <div id="notificationsList" class="stack-list"></div>
      </section>

--- a/web/static/admin.js
+++ b/web/static/admin.js
@@ -321,6 +321,11 @@ const AdminPage = (() => {
      await api("/admin/notifications/read-all", { method: "POST" });
      await loadNotifications();
    });
+    qs("#retryNotificationsBtn")?.addEventListener("click", async () => {
+      const result = await api("/admin/notifications/retry", { method: "POST" });
+      toast(`Retry: service ${result.service_delivered}, admin ${result.admin_delivered}`);
+      await loadNotifications();
+    });
    document.querySelector("[data-list-filter='users']")?.addEventListener("submit", async (event) => {
      event.preventDefault();
      await loadUsers(formData(event.currentTarget).search || "");