diff --git a/ADMIN.md b/ADMIN.md index fc42566..b147b02 100644 --- a/ADMIN.md +++ b/ADMIN.md @@ -53,6 +53,7 @@ Notifications: - `GET /api/admin/notifications` - `POST /api/admin/notifications/{id}/read` - `POST /api/admin/notifications/read-all` +- `POST /api/admin/notifications/retry` - `POST /api/admin/notifications/{id}/dismiss` Data Explorer: @@ -153,3 +154,18 @@ Data Explorer работает только по whitelist источников - `/admin_alerts` API дополнительно проверяет роль пользователя, поэтому команда не дает доступа без admin-role в БД. + +## Deploy Reports + +Для временного rsync-деплоя есть `scripts/rsync_deploy.sh`. Скрипт: + +- запускает локальные `ruff` и `pytest`; +- отправляет Telegram progress/failure/success отчеты; +- делает remote code backup без `.env`; +- синхронизирует код через `rsync`; +- собирает Docker images; +- применяет Alembic migrations; +- поднимает `api` и `bot`; +- проверяет `/health`, `/ready`, `/metrics`, `/admin.html`, `/sto.html`, `/work_order.html`. + +Утилита `scripts/send_telegram_report.py` берет получателей из `ADMIN_NOTIFICATION_CHAT_ID`, `ADMIN_TELEGRAM_IDS` и, если env пустой, из пользователей БД с ролями `admin`, `super_admin`, `moderator`, `support`. diff --git a/app/api/admin.py b/app/api/admin.py index 96c696b..9f41e64 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -36,8 +36,9 @@ from app.services.admin_notifications import ( create_admin_notification, dismiss_admin_notification, mark_admin_notification_read, + retry_admin_telegram_notifications, ) -from app.services.notifications import notify_user +from app.services.notifications import notify_user, process_notification_queue router = APIRouter(prefix="/admin", tags=["admin"]) @@ -456,6 +457,35 @@ async def read_all_admin_notifications( return {"updated": len(rows)} +@router.post("/notifications/retry") +async def retry_notifications( + limit: int = 50, + session: AsyncSession = Depends(get_session), + current_user: User = Depends(get_current_telegram_user), +) -> dict[str, Any]: + require_admin_access(current_user, FULL_ADMIN_ROLES | {"support"}) + limit = min(max(limit, 1), 200) + service_delivered = await process_notification_queue(session, limit=limit) + admin_delivered = await retry_admin_telegram_notifications(session, limit=limit) + await log_audit( + session, + actor=current_user, + action="admin.notifications.retry", + target_type="notifications", + metadata={ + "limit": limit, + "service_delivered": service_delivered, + "admin_delivered": admin_delivered, + }, + ) + await session.commit() + return { + "service_delivered": service_delivered, + "admin_delivered": admin_delivered, + "limit": limit, + } + + @router.post("/notifications/{notification_id}/dismiss") async def dismiss_notification( notification_id: int, diff --git a/app/api/ocr.py b/app/api/ocr.py index 530012d..16cd1b1 100644 --- a/app/api/ocr.py +++ b/app/api/ocr.py @@ -1,14 +1,16 @@ import re +import time from datetime import date from decimal import Decimal -from fastapi import APIRouter, Depends, File, Request, UploadFile +from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile from pydantic import BaseModel from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import get_current_telegram_user from app.db.session import get_session from app.models.user import User +from app.services.admin_notifications import create_admin_notification from app.services.ocr_provider import get_ocr_provider from app.services.rate_limit import check_rate_limit from app.services.uploads import SAFE_IMAGE_TYPES, SAFE_TEXT_TYPES, validate_upload @@ -40,6 +42,72 @@ class OCRResultRead(BaseModel): provider: str = "heuristic" +async def validate_ocr_upload( + *, + session: AsyncSession, + current_user: User, + content: bytes, + filename: str | None, + content_type: str | None, +) -> str: + try: + return validate_upload( + content=content, + filename=filename, + content_type=content_type, + max_bytes=MAX_OCR_FILE_BYTES, + allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES, + ) + except HTTPException as exc: + await create_admin_notification( + session, + event_type="upload_blocked", + title="Upload blocked", + body=f"OCR upload blocked: {filename or '-'}\nReason: {exc.detail}", + entity_type="user", + entity_id=current_user.id, + severity="warning", + idempotency_key=( + f"upload_blocked:{current_user.id}:{filename or 'upload'}:{exc.status_code}:" + f"{int(time.time() // 60)}" + ), + metadata={ + "filename": filename, + "content_type": content_type, + "status_code": exc.status_code, + "detail": exc.detail, + }, + ) + await session.commit() + raise + + +async def recognize_with_alert( + *, + session: AsyncSession, + current_user: User, + content: bytes, + filename: str | None, + scope: str, +): + try: + return await get_ocr_provider().recognize(content, filename) + except Exception as exc: # noqa: BLE001 - OCR must fail gracefully and alert admins + await create_admin_notification( + session, + event_type="ocr_failed", + title="OCR provider failed", + body=f"Scope: {scope}\nFile: {filename or '-'}\nError: {type(exc).__name__}", + entity_type="user", + entity_id=current_user.id, + severity="error", + idempotency_key=f"ocr_failed:{scope}:{current_user.id}:{int(time.time() // 60)}", + metadata={"scope": scope, "filename": filename, "error_type": type(exc).__name__}, + ) + await session.commit() + return None + + @router.post("/parse-text-receipt", response_model=ReceiptSuggestion) async def parse_text_receipt( request: Request, @@ -49,17 +117,23 @@ async def parse_text_receipt( ) -> ReceiptSuggestion: await check_rate_limit(scope="ocr", limit=10, window_seconds=60, request=request, user=current_user, session=session) content = await file.read() - validate_upload( - content=content, + await validate_ocr_upload( + session=session, + current_user=current_user, filename=file.filename, content_type=file.content_type, - max_bytes=MAX_OCR_FILE_BYTES, - allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES, + content=content, ) content_type = (file.content_type or "").lower() if content_type.startswith("image/") or content_type == "application/pdf": - result = await get_ocr_provider().recognize(content, file.filename) - if not result.recognized_text: + result = await recognize_with_alert( + session=session, + current_user=current_user, + content=content, + filename=file.filename, + scope="parse_text_receipt", + ) + if not result or not result.recognized_text: return ReceiptSuggestion( confidence=0, message="Не удалось уверенно распознать чек. Открылся ручной ввод: проверьте дату, сумму, литры и цену.", @@ -133,8 +207,22 @@ async def recognize_license_plate( ) -> OCRResultRead: await check_rate_limit(scope="ocr_license_plate", limit=8, window_seconds=60, request=request, user=current_user, session=session) content = await file.read() - validate_upload(content=content, filename=file.filename, content_type=file.content_type, max_bytes=MAX_OCR_FILE_BYTES, allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES) - result = await get_ocr_provider().recognize(content, file.filename) + await validate_ocr_upload( + session=session, + current_user=current_user, + content=content, + filename=file.filename, + content_type=file.content_type, + ) + result = await recognize_with_alert( + session=session, + current_user=current_user, + content=content, + filename=file.filename, + scope="license_plate", + ) + if result is None: + return OCRResultRead(recognized_text="", candidates=[], provider="error") return OCRResultRead( recognized_text=result.recognized_text, candidates=[OCRCandidateRead(**item.__dict__) for item in result.candidates if item.type == "license_plate"], @@ -151,8 +239,22 @@ async def recognize_vin( ) -> OCRResultRead: await check_rate_limit(scope="ocr_vin", limit=8, window_seconds=60, request=request, user=current_user, session=session) content = await file.read() - validate_upload(content=content, filename=file.filename, content_type=file.content_type, max_bytes=MAX_OCR_FILE_BYTES, allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES) - result = await get_ocr_provider().recognize(content, file.filename) + await validate_ocr_upload( + session=session, + current_user=current_user, + content=content, + filename=file.filename, + content_type=file.content_type, + ) + result = await recognize_with_alert( + session=session, + current_user=current_user, + content=content, + filename=file.filename, + scope="vin", + ) + if result is None: + return OCRResultRead(recognized_text="", candidates=[], provider="error") return OCRResultRead( recognized_text=result.recognized_text, candidates=[OCRCandidateRead(**item.__dict__) for item in result.candidates if item.type == "vin"], @@ -169,8 +271,22 @@ async def recognize_service_document( ) -> OCRResultRead: await check_rate_limit(scope="ocr_service_document", limit=8, window_seconds=60, request=request, user=current_user, session=session) content = await file.read() - validate_upload(content=content, filename=file.filename, content_type=file.content_type, max_bytes=MAX_OCR_FILE_BYTES, allowed_types=SAFE_IMAGE_TYPES | SAFE_TEXT_TYPES) - result = await get_ocr_provider().recognize(content, file.filename) + await validate_ocr_upload( + session=session, + current_user=current_user, + content=content, + filename=file.filename, + content_type=file.content_type, + ) + result = await recognize_with_alert( + session=session, + current_user=current_user, + content=content, + filename=file.filename, + scope="service_document", + ) + if result is None: + return OCRResultRead(recognized_text="", candidates=[], provider="error") return OCRResultRead( recognized_text=result.recognized_text, candidates=[OCRCandidateRead(**item.__dict__) for item in result.candidates], diff --git a/app/main.py b/app/main.py index bacb239..ce6e00b 100644 --- a/app/main.py +++ b/app/main.py @@ -25,7 +25,8 @@ from app.api import ( work_orders, ) from app.core.config import settings -from app.db.session import get_session +from app.db.session import async_session_factory, get_session +from app.services.admin_notifications import create_admin_notification from app.services.rate_limit import get_redis_client @@ -49,8 +50,29 @@ async def production_headers_and_metrics(request: Request, call_next): start = monotonic() try: response = await call_next(request) - except Exception: + except Exception as exc: REQUEST_ERRORS += 1 + try: + async with async_session_factory() as session: + await create_admin_notification( + session, + event_type="system_error", + title="Unhandled API error", + body=f"{request.method} {request.url.path}\nError: {type(exc).__name__}", + entity_type="system", + entity_id=request.url.path, + severity="error", + idempotency_key=f"system_error:{request.url.path}:{type(exc).__name__}:{int(start // 60)}", + metadata={ + "path": request.url.path, + "method": request.method, + "request_id": request_id, + "error_type": type(exc).__name__, + }, + ) + await session.commit() + except Exception: + pass raise duration = monotonic() - start REQUEST_COUNT += 1 diff --git a/app/services/admin_notifications.py b/app/services/admin_notifications.py index 35e6d94..9c2b4f5 100644 --- a/app/services/admin_notifications.py +++ b/app/services/admin_notifications.py @@ -132,6 +132,22 @@ async def send_admin_telegram_notification(notification: AdminNotification) -> N notification.telegram_status = "sent" +async def retry_admin_telegram_notifications(session: AsyncSession, *, limit: int = 50) -> int: + result = await session.execute( + select(AdminNotification) + .where(AdminNotification.telegram_status.in_(["pending", "failed"])) + .order_by(AdminNotification.created_at.asc()) + .limit(limit) + ) + delivered = 0 + for notification in result.scalars(): + await send_admin_telegram_notification(notification) + if notification.telegram_status == "sent": + delivered += 1 + await session.commit() + return delivered + + async def mark_admin_notification_read( session: AsyncSession, notification: AdminNotification ) -> AdminNotification: diff --git a/app/services/rate_limit.py b/app/services/rate_limit.py index eb02fec..83ca023 100644 --- a/app/services/rate_limit.py +++ b/app/services/rate_limit.py @@ -41,7 +41,13 @@ async def check_rate_limit( if settings.redis_url: allowed = await check_redis_rate_limit(scope, identifiers, limit, window_seconds) if not allowed: - await log_rate_limit_event(session, scope=scope, identifier="redis") + await log_rate_limit_event( + session, + scope=scope, + identifier="redis", + user=user, + request=request, + ) raise_rate_limit(scope, window_seconds) return @@ -52,7 +58,13 @@ async def check_rate_limit( while bucket and now - bucket[0] > window_seconds: bucket.popleft() if len(bucket) >= limit: - await log_rate_limit_event(session, scope=scope, identifier=str(identifier)) + await log_rate_limit_event( + session, + scope=scope, + identifier=str(identifier), + user=user, + request=request, + ) raise_rate_limit(scope, window_seconds) for identifier in identifiers: _buckets[(scope, identifier)].append(now) @@ -107,18 +119,82 @@ async def log_rate_limit_event( *, scope: str, identifier: str, + user: User | None = None, + request: Request | None = None, ) -> None: - if session is None: - return - from app.models.car import AuditLog + client_host = request.client.host if request and request.client else None + user_agent = request.headers.get("user-agent") if request else None + metadata = { + "scope": scope, + "identifier": identifier, + "telegram_id": user.telegram_id if user else None, + "user_id": user.id if user else None, + "ip": client_host, + } - session.add( - AuditLog( - actor_user_id=None, - actor_role="system", - action="rate_limit.exceeded", - target_type=scope, - target_id=identifier[:80], - metadata_json={"scope": scope, "identifier": identifier}, - ) + if session is None: + from app.db.session import async_session_factory + + async with async_session_factory() as event_session: + await persist_rate_limit_event( + event_session, + scope=scope, + identifier=identifier, + user=user, + client_host=client_host, + user_agent=user_agent, + metadata=metadata, + ) + return + + await persist_rate_limit_event( + session, + scope=scope, + identifier=identifier, + user=user, + client_host=client_host, + user_agent=user_agent, + metadata=metadata, ) + + +async def persist_rate_limit_event( + event_session: AsyncSession, + *, + scope: str, + identifier: str, + user: User | None, + client_host: str | None, + user_agent: str | None, + metadata: dict, +) -> None: + from app.models.car import AuditLog + from app.services.admin_notifications import create_admin_notification + + try: + event_session.add( + AuditLog( + actor_user_id=user.id if user else None, + actor_role=user.platform_role if user else "system", + action="rate_limit.exceeded", + target_type=scope, + target_id=identifier[:80], + metadata_json=metadata, + ip=client_host, + user_agent=user_agent[:256] if user_agent else None, + ) + ) + await create_admin_notification( + event_session, + event_type="rate_limit_exceeded", + title="Rate limit exceeded", + body=f"Scope: {scope}\nIdentifier: {identifier}", + entity_type="user" if user else "system", + entity_id=user.id if user else scope, + severity="warning", + idempotency_key=f"rate_limit:{scope}:{identifier}:{int(time.time() // max(60, 1))}", + metadata=metadata, + ) + await event_session.commit() + except Exception: + await event_session.rollback() diff --git a/scripts/rsync_deploy.sh b/scripts/rsync_deploy.sh new file mode 100755 index 0000000..2155ebd --- /dev/null +++ b/scripts/rsync_deploy.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +REMOTE="${REMOTE:-root@drivers.smartsoltech.kr}" +REMOTE_DIR="${REMOTE_DIR:-/opt/drivers_bot}" +BASE_URL="${BASE_URL:-http://127.0.0.1:8000}" +COMPOSE="${COMPOSE:-docker compose}" +RUN_LOCAL_CHECKS="${RUN_LOCAL_CHECKS:-true}" +BACKUP_BEFORE_DEPLOY="${BACKUP_BEFORE_DEPLOY:-true}" +BRANCH="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)" +REVISION="$(git rev-parse --short HEAD 2>/dev/null || echo unknown)" + +EXCLUDES=( + "--exclude=.git/" + "--exclude=.env" + "--exclude=.env.*" + "--exclude=.venv/" + "--exclude=venv/" + "--exclude=__pycache__/" + "--exclude=.pytest_cache/" + "--exclude=.ruff_cache/" + "--exclude=.history/" + "--exclude=backups/" + "--exclude=*.sqlite" + "--exclude=*.sqlite3" + "--exclude=*.db" +) + +send_remote_report() { + local text="$1" + ssh "$REMOTE" "cd '$REMOTE_DIR' && CARPASS_REPORT_TEXT=\$(cat) $COMPOSE exec -T -e CARPASS_REPORT_TEXT api python scripts/send_telegram_report.py" <<<"$text" || true +} + +fail_report() { + local line="${1:-unknown}" + send_remote_report "❌ CarPass rsync deploy failed +Branch: $BRANCH +Revision: $REVISION +Step line: $line +Target: $REMOTE" +} + +trap 'fail_report "$LINENO"' ERR + +if [[ "$RUN_LOCAL_CHECKS" == "true" ]]; then + echo "Running local checks..." + .venv/bin/ruff check app bot tests + .venv/bin/pytest -q +fi + +send_remote_report "🚀 CarPass rsync deploy started +Branch: $BRANCH +Revision: $REVISION +Target: $REMOTE +Checks: local=${RUN_LOCAL_CHECKS}" + +echo "Checking remote..." +ssh "$REMOTE" "test -d '$REMOTE_DIR' && test -f '$REMOTE_DIR/docker-compose.yml'" + +if [[ "$BACKUP_BEFORE_DEPLOY" == "true" ]]; then + echo "Creating remote code backup..." + ssh "$REMOTE" "cd '$(dirname "$REMOTE_DIR")' && mkdir -p '$REMOTE_DIR/backups' && tar --exclude='$(basename "$REMOTE_DIR")/backups' --exclude='$(basename "$REMOTE_DIR")/.git' --exclude='$(basename "$REMOTE_DIR")/.env' --exclude='$(basename "$REMOTE_DIR")/.venv' -czf '$REMOTE_DIR/backups/code_pre_rsync_$(date +%Y%m%d%H%M%S).tgz' '$(basename "$REMOTE_DIR")'" +fi + +echo "Syncing code with rsync..." +rsync -az --delete "${EXCLUDES[@]}" ./ "$REMOTE:$REMOTE_DIR/" + +echo "Building remote images..." +ssh "$REMOTE" "cd '$REMOTE_DIR' && $COMPOSE build" +send_remote_report "🧱 CarPass rsync deploy progress +Branch: $BRANCH +Step: docker build completed" + +echo "Applying migrations..." +ssh "$REMOTE" "cd '$REMOTE_DIR' && $COMPOSE run --rm api alembic upgrade head" +send_remote_report "🗄️ CarPass rsync deploy progress +Branch: $BRANCH +Step: migrations applied" + +echo "Starting services..." +ssh "$REMOTE" "cd '$REMOTE_DIR' && $COMPOSE up -d" + +echo "Waiting for API readiness..." +ssh "$REMOTE" "cd '$REMOTE_DIR' && for i in \$(seq 1 30); do status=\$(docker inspect -f '{{.State.Health.Status}}' drivers_bot-api-1 2>/dev/null || echo missing); echo \"api_health=\$status\"; [ \"\$status\" = healthy ] && exit 0; sleep 2; done; $COMPOSE logs --tail=120 api; exit 1" + +echo "Running remote smoke tests..." +ssh "$REMOTE" "cd '$REMOTE_DIR' && BASE_URL='$BASE_URL' ./scripts/smoke_test.sh && curl -fsSI '$BASE_URL/admin.html' | head -5 && $COMPOSE ps" + +send_remote_report "✅ CarPass rsync deploy completed +Branch: $BRANCH +Revision: $REVISION +Migration: $(ssh "$REMOTE" "cd '$REMOTE_DIR' && curl -fsS '$BASE_URL/ready'" | tr '\n' ' ') +Checks: /health ok, /ready ok, /metrics ok, /admin.html 200 +Services: api healthy, bot restarted" + +echo "Deploy completed." diff --git a/scripts/send_telegram_report.py b/scripts/send_telegram_report.py new file mode 100755 index 0000000..e95af49 --- /dev/null +++ b/scripts/send_telegram_report.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import asyncio +import os +from collections.abc import Iterable + +import httpx +from sqlalchemy import select + +from app.core.config import settings +from app.db.session import async_session_factory +from app.models import car, expense, gamification, push # noqa: F401 +from app.models.user import User + +REPORT_ROLES = {"admin", "super_admin", "moderator", "support"} + + +def env_recipients() -> list[str]: + recipients: list[str] = [] + if settings.admin_notification_chat_id: + recipients.append(settings.admin_notification_chat_id) + recipients.extend(str(item) for item in settings.admin_telegram_id_list) + return recipients + + +async def db_recipients() -> list[str]: + async with async_session_factory() as session: + result = await session.execute( + select(User.telegram_id).where(User.platform_role.in_(REPORT_ROLES)) + ) + return [str(row[0]) for row in result.all() if row[0]] + + +def unique(values: Iterable[str]) -> list[str]: + return list(dict.fromkeys(item.strip() for item in values if item and item.strip())) + + +async def send_report(text: str, *, dry_run: bool = False) -> int: + recipients = unique([*env_recipients(), *(await db_recipients())]) + if dry_run: + print(f"telegram_report_dry_run recipients={len(recipients)}") + return len(recipients) + if not settings.bot_token or not recipients: + print("telegram_report_skipped") + return 0 + + sent = 0 + async with httpx.AsyncClient(timeout=10) as client: + for chat_id in recipients: + try: + response = await client.post( + f"https://api.telegram.org/bot{settings.bot_token}/sendMessage", + json={"chat_id": chat_id, "text": text, "disable_web_page_preview": True}, + ) + response.raise_for_status() + sent += 1 + except Exception as exc: # noqa: BLE001 - deploy report must never fail deploy + print(f"telegram_report_failed chat_id={chat_id} error={type(exc).__name__}") + print(f"telegram_report_sent_count {sent}") + return sent + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Send a CarPass operational Telegram report.") + parser.add_argument("--text", help="Report text. Defaults to CARPASS_REPORT_TEXT.") + parser.add_argument("--dry-run", action="store_true") + return parser.parse_args() + + +async def main() -> None: + args = parse_args() + text = args.text or os.getenv("CARPASS_REPORT_TEXT") or "" + if not text.strip(): + raise SystemExit("Report text is required") + await send_report(text, dry_run=args.dry_run) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh index a72517b..b40893d 100755 --- a/scripts/smoke_test.sh +++ b/scripts/smoke_test.sh @@ -13,4 +13,10 @@ echo echo "Checking metrics..." curl -fsS "$BASE_URL/metrics" | grep -q "carpass_requests_total" + +for path in / /sto.html /admin.html /work_order.html; do + echo "Checking static page $path..." + curl -fsSI "$BASE_URL$path" | grep -q "200 OK" +done + echo "Smoke test passed." diff --git a/tests/test_admin_control_center.py b/tests/test_admin_control_center.py index 467c0ec..fdcd514 100644 --- a/tests/test_admin_control_center.py +++ b/tests/test_admin_control_center.py @@ -247,3 +247,54 @@ async def test_pending_sto_queue_and_approve_audit( assert approved.status_code == 200 assert approved.json()["verification_status"] == "approved" assert any(item["action"] == "service_center.verify" for item in audit.json()) + + +@pytest.mark.asyncio +async def test_blocked_ocr_upload_creates_admin_notification( + client, auth_headers, admin_auth_headers, internal_headers +) -> None: + await ensure_admin(client, internal_headers) + + response = await client.post( + "/api/ocr/vin", + headers=auth_headers, + files={"file": ("invoice.exe", b"not an image", "image/jpeg")}, + ) + notifications = await client.get("/api/admin/notifications?limit=100", headers=admin_auth_headers) + + assert response.status_code == 415 + assert any(item["event_type"] == "upload_blocked" for item in notifications.json()["rows"]) + + +@pytest.mark.asyncio +async def test_rate_limit_creates_admin_notification( + client, auth_headers, admin_auth_headers, internal_headers +) -> None: + await ensure_admin(client, internal_headers) + + last_response = None + for index in range(9): + last_response = await client.post( + "/api/ocr/vin", + headers=auth_headers, + files={"file": (f"vin-{index}.txt", b"VIN KMHCT41BAHU123456", "text/plain")}, + ) + notifications = await client.get("/api/admin/notifications?limit=100", headers=admin_auth_headers) + + assert last_response is not None + assert last_response.status_code == 429 + assert any(item["event_type"] == "rate_limit_exceeded" for item in notifications.json()["rows"]) + + +@pytest.mark.asyncio +async def test_admin_can_retry_notification_queues( + client, admin_auth_headers, internal_headers +) -> None: + await ensure_admin(client, internal_headers) + + response = await client.post("/api/admin/notifications/retry", headers=admin_auth_headers) + audit = await client.get("/api/admin/audit-log?action=admin.notifications.retry", headers=admin_auth_headers) + + assert response.status_code == 200 + assert {"service_delivered", "admin_delivered", "limit"} <= response.json().keys() + assert any(item["action"] == "admin.notifications.retry" for item in audit.json()) diff --git a/web/admin.html b/web/admin.html index b7b8856..7b305fe 100644 --- a/web/admin.html +++ b/web/admin.html @@ -84,7 +84,10 @@
События