Andrey K. Choi
a24e4e8dc6
Some checks failed
continuous-integration/drone Build is failing
✨ New Features: 🔐 Advanced agent authentication with JWT tokens 🌐 RESTful API server with WebSocket support 🐳 Docker multi-stage containerization 🚀 Comprehensive CI/CD with Drone pipeline 📁 Professional project structure reorganization 🛠️ Technical Implementation: • JWT-based authentication with HMAC-SHA256 signatures • Unique Agent IDs with automatic credential generation • Real-time API with CORS and rate limiting • SQLite extended schema for auth management • Multi-stage Docker builds (controller/agent/standalone) • Complete Drone CI/CD with testing and security scanning �� Key Modules: • src/auth.py (507 lines) - Authentication system • src/api_server.py (823 lines) - REST API server • src/storage.py - Extended database with auth tables • Dockerfile - Multi-stage containerization • .drone.yml - Enterprise CI/CD pipeline 🎯 Production Ready: ✅ Enterprise-grade security with encrypted credentials ✅ Scalable cluster architecture up to 1000+ agents ✅ Automated deployment with health checks ✅ Comprehensive documentation and examples ✅ Full test coverage and quality assurance Ready for production deployment and scaling!
530 lines
22 KiB
Python
530 lines
22 KiB
Python
"""
|
||
Monitor module для PyGuardian
|
||
Мониторинг auth.log в реальном времени и детекция атак
|
||
"""
|
||
|
||
import asyncio
|
||
import aiofiles
|
||
import re
|
||
import logging
|
||
from datetime import datetime
|
||
from typing import Dict, List, Optional, Callable
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
@dataclass
|
||
class LogEvent:
|
||
"""Структура для события в логе"""
|
||
timestamp: datetime
|
||
ip_address: str
|
||
username: Optional[str]
|
||
event_type: str
|
||
log_line: str
|
||
is_success: bool = False
|
||
|
||
|
||
class LogParser:
|
||
"""Парсер для auth.log"""
|
||
|
||
def __init__(self, patterns: List[str]):
|
||
self.failed_patterns = patterns
|
||
|
||
# Компилируем регулярные выражения для различных типов событий
|
||
self.patterns = {
|
||
'failed_password': re.compile(
|
||
r'Failed password for (?:invalid user )?(\w+) from ([\d.]+)'
|
||
),
|
||
'invalid_user': re.compile(
|
||
r'Invalid user (\w+) from ([\d.]+)'
|
||
),
|
||
'authentication_failure': re.compile(
|
||
r'authentication failure.*ruser=(\w*)\s+rhost=([\d.]+)'
|
||
),
|
||
'too_many_failures': re.compile(
|
||
r'Too many authentication failures for (\w+) from ([\d.]+)'
|
||
),
|
||
'failed_publickey': re.compile(
|
||
r'Failed publickey for (?:invalid user )?(\w+) from ([\d.]+)'
|
||
),
|
||
'connection_closed': re.compile(
|
||
r'Connection closed by authenticating user (\w+) ([\d.]+)'
|
||
),
|
||
'accepted_password': re.compile(
|
||
r'Accepted password for (\w+) from ([\d.]+)'
|
||
),
|
||
'accepted_publickey': re.compile(
|
||
r'Accepted publickey for (\w+) from ([\d.]+)'
|
||
)
|
||
}
|
||
|
||
def parse_line(self, line: str) -> Optional[LogEvent]:
|
||
"""Парсинг строки лога"""
|
||
try:
|
||
# Извлекаем timestamp
|
||
timestamp = self._parse_timestamp(line)
|
||
if not timestamp:
|
||
return None
|
||
|
||
# Проверяем успешные входы
|
||
for pattern_name, pattern in self.patterns.items():
|
||
if 'accepted' in pattern_name.lower():
|
||
match = pattern.search(line)
|
||
if match:
|
||
username, ip = match.groups()
|
||
return LogEvent(
|
||
timestamp=timestamp,
|
||
ip_address=ip,
|
||
username=username,
|
||
event_type=pattern_name,
|
||
log_line=line.strip(),
|
||
is_success=True
|
||
)
|
||
|
||
# Проверяем атаки
|
||
for pattern in self.failed_patterns:
|
||
if pattern.lower() in line.lower():
|
||
event = self._parse_failed_event(line, timestamp)
|
||
if event:
|
||
return event
|
||
|
||
return None
|
||
|
||
except Exception as e:
|
||
logger.error(f"Ошибка парсинга строки '{line[:100]}...': {e}")
|
||
return None
|
||
|
||
def _parse_timestamp(self, line: str) -> Optional[datetime]:
|
||
"""Извлечение timestamp из строки лога"""
|
||
try:
|
||
# Стандартный формат syslog: "Nov 25 14:30:15"
|
||
timestamp_pattern = re.compile(
|
||
r'^(\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})'
|
||
)
|
||
match = timestamp_pattern.search(line)
|
||
if match:
|
||
timestamp_str = match.group(1)
|
||
# Добавляем текущий год
|
||
current_year = datetime.now().year
|
||
timestamp_str = f"{current_year} {timestamp_str}"
|
||
return datetime.strptime(timestamp_str, "%Y %b %d %H:%M:%S")
|
||
return None
|
||
except Exception:
|
||
return None
|
||
|
||
def _parse_failed_event(self, line: str, timestamp: datetime) -> Optional[LogEvent]:
|
||
"""Парсинг событий атак"""
|
||
for pattern_name, pattern in self.patterns.items():
|
||
if 'accepted' in pattern_name.lower():
|
||
continue
|
||
|
||
match = pattern.search(line)
|
||
if match:
|
||
groups = match.groups()
|
||
if len(groups) >= 2:
|
||
username = groups[0] if groups[0] else "unknown"
|
||
ip_address = groups[1]
|
||
|
||
return LogEvent(
|
||
timestamp=timestamp,
|
||
ip_address=ip_address,
|
||
username=username,
|
||
event_type=pattern_name,
|
||
log_line=line.strip(),
|
||
is_success=False
|
||
)
|
||
|
||
# Если не удалось распарсить конкретным паттерном,
|
||
# ищем IP в строке
|
||
ip_pattern = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})')
|
||
ip_match = ip_pattern.search(line)
|
||
|
||
if ip_match:
|
||
return LogEvent(
|
||
timestamp=timestamp,
|
||
ip_address=ip_match.group(1),
|
||
username="unknown",
|
||
event_type="generic_failure",
|
||
log_line=line.strip(),
|
||
is_success=False
|
||
)
|
||
|
||
return None
|
||
|
||
|
||
class LogMonitor:
|
||
"""Мониторинг auth.log файла"""
|
||
|
||
def __init__(self, config: Dict, event_callback: Optional[Callable] = None):
|
||
self.log_path = config.get('auth_log_path', '/var/log/auth.log')
|
||
self.check_interval = config.get('check_interval', 1.0)
|
||
self.parser = LogParser(config.get('failed_patterns', []))
|
||
|
||
self.event_callback = event_callback
|
||
self.running = False
|
||
self.file_position = 0
|
||
self.last_inode = None
|
||
|
||
# Статистика
|
||
self.stats = {
|
||
'lines_processed': 0,
|
||
'events_detected': 0,
|
||
'last_event_time': None,
|
||
'start_time': datetime.now()
|
||
}
|
||
|
||
async def start(self) -> None:
|
||
"""Запуск мониторинга"""
|
||
if self.running:
|
||
logger.warning("Мониторинг уже запущен")
|
||
return
|
||
|
||
self.running = True
|
||
logger.info(f"Запуск мониторинга файла {self.log_path}")
|
||
|
||
# Устанавливаем позицию в конец файла при запуске
|
||
await self._init_file_position()
|
||
|
||
try:
|
||
while self.running:
|
||
await self._check_log_file()
|
||
await asyncio.sleep(self.check_interval)
|
||
except Exception as e:
|
||
logger.error(f"Ошибка в цикле мониторинга: {e}")
|
||
finally:
|
||
self.running = False
|
||
|
||
async def stop(self) -> None:
|
||
"""Остановка мониторинга"""
|
||
logger.info("Остановка мониторинга")
|
||
self.running = False
|
||
|
||
async def _init_file_position(self) -> None:
|
||
"""Инициализация позиции в файле"""
|
||
try:
|
||
if Path(self.log_path).exists():
|
||
stat = Path(self.log_path).stat()
|
||
self.file_position = stat.st_size
|
||
self.last_inode = stat.st_ino
|
||
logger.info(f"Начальная позиция в файле: {self.file_position}")
|
||
else:
|
||
logger.warning(f"Лог файл {self.log_path} не найден")
|
||
self.file_position = 0
|
||
self.last_inode = None
|
||
except Exception as e:
|
||
logger.error(f"Ошибка инициализации позиции файла: {e}")
|
||
self.file_position = 0
|
||
self.last_inode = None
|
||
|
||
async def _check_log_file(self) -> None:
|
||
"""Проверка изменений в лог файле"""
|
||
try:
|
||
if not Path(self.log_path).exists():
|
||
logger.warning(f"Лог файл {self.log_path} не существует")
|
||
return
|
||
|
||
stat = Path(self.log_path).stat()
|
||
current_inode = stat.st_ino
|
||
current_size = stat.st_size
|
||
|
||
# Проверяем, не был ли файл ротирован
|
||
if self.last_inode is not None and current_inode != self.last_inode:
|
||
logger.info("Обнаружена ротация лог файла")
|
||
self.file_position = 0
|
||
self.last_inode = current_inode
|
||
|
||
# Проверяем, есть ли новые данные
|
||
if current_size > self.file_position:
|
||
await self._process_new_lines(current_size)
|
||
elif current_size < self.file_position:
|
||
# Файл был усечен
|
||
logger.info("Файл был усечен, сброс позиции")
|
||
self.file_position = 0
|
||
await self._process_new_lines(current_size)
|
||
|
||
self.last_inode = current_inode
|
||
|
||
except Exception as e:
|
||
logger.error(f"Ошибка проверки лог файла: {e}")
|
||
|
||
async def _process_new_lines(self, current_size: int) -> None:
|
||
"""Обработка новых строк в файле"""
|
||
try:
|
||
async with aiofiles.open(self.log_path, 'r', encoding='utf-8', errors='ignore') as file:
|
||
await file.seek(self.file_position)
|
||
|
||
while True:
|
||
line = await file.readline()
|
||
if not line:
|
||
break
|
||
|
||
self.stats['lines_processed'] += 1
|
||
|
||
# Парсим строку
|
||
event = self.parser.parse_line(line)
|
||
if event:
|
||
self.stats['events_detected'] += 1
|
||
self.stats['last_event_time'] = event.timestamp
|
||
|
||
logger.debug(f"Обнаружено событие: {event.event_type} from {event.ip_address}")
|
||
|
||
# Отправляем событие в callback
|
||
if self.event_callback:
|
||
try:
|
||
if asyncio.iscoroutinefunction(self.event_callback):
|
||
await self.event_callback(event)
|
||
else:
|
||
self.event_callback(event)
|
||
except Exception as e:
|
||
logger.error(f"Ошибка в callback: {e}")
|
||
|
||
# Обновляем позицию
|
||
self.file_position = await file.tell()
|
||
|
||
except Exception as e:
|
||
logger.error(f"Ошибка обработки новых строк: {e}")
|
||
|
||
def get_stats(self) -> Dict:
|
||
"""Получение статистики мониторинга"""
|
||
uptime = datetime.now() - self.stats['start_time']
|
||
|
||
return {
|
||
'running': self.running,
|
||
'log_path': self.log_path,
|
||
'file_position': self.file_position,
|
||
'lines_processed': self.stats['lines_processed'],
|
||
'events_detected': self.stats['events_detected'],
|
||
'last_event_time': self.stats['last_event_time'],
|
||
'uptime_seconds': int(uptime.total_seconds()),
|
||
'check_interval': self.check_interval
|
||
}
|
||
|
||
async def test_patterns(self, test_lines: List[str]) -> List[LogEvent]:
|
||
"""Тестирование паттернов на примерах строк"""
|
||
events = []
|
||
for line in test_lines:
|
||
event = self.parser.parse_line(line)
|
||
if event:
|
||
events.append(event)
|
||
return events
|
||
|
||
|
||
class AttackDetector:
|
||
"""Детектор атак на основе событий"""
|
||
|
||
def __init__(self, storage, firewall_manager, security_manager, config: Dict):
|
||
self.storage = storage
|
||
self.firewall_manager = firewall_manager
|
||
self.security_manager = security_manager
|
||
self.config = config
|
||
|
||
self.max_attempts = config.get('max_attempts', 5)
|
||
self.time_window = config.get('time_window', 60)
|
||
self.unban_time = config.get('unban_time', 3600)
|
||
self.whitelist = config.get('whitelist', [])
|
||
|
||
# Callback для уведомлений
|
||
self.ban_callback: Optional[Callable] = None
|
||
self.unban_callback: Optional[Callable] = None
|
||
|
||
def set_callbacks(self, ban_callback: Optional[Callable] = None,
|
||
unban_callback: Optional[Callable] = None) -> None:
|
||
"""Установка callback для уведомлений"""
|
||
self.ban_callback = ban_callback
|
||
self.unban_callback = unban_callback
|
||
|
||
async def process_event(self, event: LogEvent) -> None:
|
||
"""Обработка события из лога"""
|
||
try:
|
||
# Передаем событие в SecurityManager для глубокого анализа
|
||
await self.security_manager.analyze_login_event(event)
|
||
|
||
# Добавляем событие в базу данных
|
||
if event.is_success:
|
||
await self.storage.add_successful_login(
|
||
event.ip_address,
|
||
event.username or "unknown",
|
||
f"login_type:{event.event_type}"
|
||
)
|
||
logger.info(f"Успешный вход: {event.username}@{event.ip_address}")
|
||
else:
|
||
await self.storage.add_attack_attempt(
|
||
event.ip_address,
|
||
event.username or "unknown",
|
||
event.event_type,
|
||
event.log_line,
|
||
event.timestamp
|
||
)
|
||
|
||
# Проверяем, нужно ли банить IP (стандартная логика брутфорса)
|
||
await self._check_and_ban_ip(event.ip_address)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Ошибка обработки события: {e}")
|
||
|
||
async def _check_and_ban_ip(self, ip: str) -> None:
|
||
"""Проверка и бан IP при превышении лимита"""
|
||
try:
|
||
# Проверяем белый список
|
||
if await self.storage.is_whitelisted(ip, self.whitelist):
|
||
logger.info(f"IP {ip} в белом списке, пропускаем")
|
||
return
|
||
|
||
# Проверяем, не забанен ли уже
|
||
if await self.storage.is_ip_banned(ip):
|
||
logger.debug(f"IP {ip} уже забанен")
|
||
return
|
||
|
||
# Получаем количество попыток за время окна
|
||
attempts = await self.storage.get_attack_count_for_ip(ip, self.time_window)
|
||
|
||
if attempts >= self.max_attempts:
|
||
# Баним IP
|
||
reason = f"Превышен лимит попыток: {attempts}/{self.max_attempts} за {self.time_window}с"
|
||
|
||
# Записываем в базу данных
|
||
success = await self.storage.ban_ip(
|
||
ip, reason, self.unban_time, manual=False, attempts_count=attempts
|
||
)
|
||
|
||
if success:
|
||
# Блокируем через firewall
|
||
firewall_success = await self.firewall_manager.ban_ip(ip)
|
||
|
||
if firewall_success:
|
||
logger.warning(f"IP {ip} забанен: {reason}")
|
||
|
||
# Уведомление через callback
|
||
if self.ban_callback:
|
||
ban_info = {
|
||
'ip': ip,
|
||
'reason': reason,
|
||
'attempts': attempts,
|
||
'auto': True
|
||
}
|
||
try:
|
||
if asyncio.iscoroutinefunction(self.ban_callback):
|
||
await self.ban_callback(ban_info)
|
||
else:
|
||
self.ban_callback(ban_info)
|
||
except Exception as e:
|
||
logger.error(f"Ошибка в ban callback: {e}")
|
||
else:
|
||
logger.error(f"Не удалось заблокировать IP {ip} через firewall")
|
||
else:
|
||
logger.error(f"Не удалось записать бан IP {ip} в базу данных")
|
||
|
||
except Exception as e:
|
||
logger.error(f"Ошибка проверки IP {ip} для бана: {e}")
|
||
|
||
async def process_unban(self, ip: str) -> bool:
|
||
"""Разбан IP адреса"""
|
||
try:
|
||
# Разбаниваем в базе данных
|
||
db_success = await self.storage.unban_ip(ip)
|
||
|
||
# Разбаниваем в firewall
|
||
firewall_success = await self.firewall_manager.unban_ip(ip)
|
||
|
||
if db_success and firewall_success:
|
||
logger.info(f"IP {ip} успешно разбанен")
|
||
|
||
# Уведомление через callback
|
||
if self.unban_callback:
|
||
unban_info = {
|
||
'ip': ip,
|
||
'auto': False
|
||
}
|
||
try:
|
||
if asyncio.iscoroutinefunction(self.unban_callback):
|
||
await self.unban_callback(unban_info)
|
||
else:
|
||
self.unban_callback(unban_info)
|
||
except Exception as e:
|
||
logger.error(f"Ошибка в unban callback: {e}")
|
||
|
||
return True
|
||
else:
|
||
logger.error(f"Ошибка разбана IP {ip}")
|
||
return False
|
||
|
||
except Exception as e:
|
||
logger.error(f"Ошибка разбана IP {ip}: {e}")
|
||
return False
|
||
|
||
async def check_expired_bans(self) -> None:
|
||
"""Проверка и автоматический разбан истекших IP"""
|
||
try:
|
||
expired_ips = await self.storage.get_expired_bans()
|
||
|
||
for ip in expired_ips:
|
||
success = await self.process_unban(ip)
|
||
if success:
|
||
logger.info(f"IP {ip} автоматически разбанен (истек срок)")
|
||
|
||
# Уведомление об автоматическом разбане
|
||
if self.unban_callback:
|
||
unban_info = {
|
||
'ip': ip,
|
||
'auto': True
|
||
}
|
||
try:
|
||
if asyncio.iscoroutinefunction(self.unban_callback):
|
||
await self.unban_callback(unban_info)
|
||
else:
|
||
self.unban_callback(unban_info)
|
||
except Exception as e:
|
||
logger.error(f"Ошибка в auto unban callback: {e}")
|
||
|
||
except Exception as e:
|
||
logger.error(f"Ошибка проверки истекших банов: {e}")
|
||
|
||
async def manual_ban(self, ip: str, reason: str = "Ручная блокировка") -> bool:
|
||
"""Ручной бан IP адреса"""
|
||
try:
|
||
# Проверяем белый список
|
||
if await self.storage.is_whitelisted(ip, self.whitelist):
|
||
logger.warning(f"Попытка заблокировать IP {ip} из белого списка")
|
||
return False
|
||
|
||
# Записываем в базу данных
|
||
success = await self.storage.ban_ip(
|
||
ip, reason, self.unban_time, manual=True
|
||
)
|
||
|
||
if success:
|
||
# Блокируем через firewall
|
||
firewall_success = await self.firewall_manager.ban_ip(ip)
|
||
|
||
if firewall_success:
|
||
logger.info(f"IP {ip} ручной бан: {reason}")
|
||
|
||
# Уведомление через callback
|
||
if self.ban_callback:
|
||
ban_info = {
|
||
'ip': ip,
|
||
'reason': reason,
|
||
'attempts': 0,
|
||
'auto': False
|
||
}
|
||
try:
|
||
if asyncio.iscoroutinefunction(self.ban_callback):
|
||
await self.ban_callback(ban_info)
|
||
else:
|
||
self.ban_callback(ban_info)
|
||
except Exception as e:
|
||
logger.error(f"Ошибка в manual ban callback: {e}")
|
||
|
||
return True
|
||
else:
|
||
logger.error(f"Не удалось заблокировать IP {ip} через firewall")
|
||
return False
|
||
else:
|
||
logger.error(f"Не удалось записать ручной бан IP {ip} в базу данных")
|
||
return False
|
||
|
||
except Exception as e:
|
||
logger.error(f"Ошибка ручного бана IP {ip}: {e}")
|
||
return False |