diff --git a/antifroud/admin.py b/antifroud/admin.py index 452ceeaf..681f4bdb 100644 --- a/antifroud/admin.py +++ b/antifroud/admin.py @@ -10,7 +10,7 @@ from hotels.models import Hotel, Room import pymysql import logging from django.urls import reverse - +from datetime import datetime logger = logging.getLogger(__name__) @@ -111,10 +111,19 @@ class ExternalDBSettingsAdmin(admin.ModelAdmin): @admin.register(UserActivityLog) class UserActivityLogAdmin(admin.ModelAdmin): - list_display = ("id", "timestamp", "date_time", "page_id", "url_parameters", "page_url" ,"created", "page_title", "type", "hits") + list_display = ("id", "ip", 'get_location',"formatted_timestamp", "date_time", "page_id", "url_parameters", "page_url" ,"created", "page_title", "type", "hits") search_fields = ("page_title", "url_parameters") list_filter = ("page_title", "created") readonly_fields = ("created", "timestamp") + + def get_formatted_timestamp(self, obj): + """ + Метод для админки для преобразования timestamp в читаемый формат. + """ + return obj.formatted_timestamp # Используем свойство модели + get_formatted_timestamp.short_description = "Таймштамп" + + def get_hotel_name(self): """ Возвращает название отеля на основе связанного page_id. @@ -142,7 +151,6 @@ class UserActivityLogAdmin(admin.ModelAdmin): get_hotel_name.short_description = "Отель" get_room_number.short_description = "Комната" - from .views import import_selected_hotels # Регистрируем admin класс для ImportedHotel @admin.register(ImportedHotel) @@ -197,13 +205,13 @@ class ImportedHotelAdmin(admin.ModelAdmin): @admin.register(SyncLog) class SyncLogAdmin(admin.ModelAdmin): change_list_template = "antifroud/admin/sync_log.html" - list_display =['id', 'hotel', 'recieved_records', 'processed_records'] - search_fields = ['id', 'hotel', 'received_records', 'processed_records'] - list_filter = ['id', 'hotel', 'processed_records'] + list_display =['id', 'hotel', 'created', 'recieved_records', 'processed_records'] + search_fields = ['id', 'hotel', 'created', 'recieved_records', 'processed_records'] + list_filter = ['id', 'hotel', 'created', 'recieved_records', 'processed_records'] class Meta: model = SyncLog - fields = ['hotel', 'received_records', 'processed_records'] + fields = ['hotel', 'recieved_records', 'processed_records'] @admin.register(ViolationLog) diff --git a/antifroud/check_fraud.py b/antifroud/check_fraud.py index d04b28d9..3e86e7e0 100644 --- a/antifroud/check_fraud.py +++ b/antifroud/check_fraud.py @@ -33,31 +33,33 @@ class ReservationChecker: logger.error(message) def fetch_user_logs(self): - """ Извлекает записи из UserActivityLog за период. """ - user_logs = UserActivityLog.objects.filter(created__range=(self.start_time, self.end_time)) - self.log_info(f"Найдено {user_logs.count()} логов активности для анализа.") - return user_logs + try: + self.log_info("Начинается извлечение логов активности пользователей.") + user_logs = UserActivityLog.objects.filter(created__range=(self.start_time, self.end_time)) + self.log_info(f"Найдено {user_logs.count()} логов активности для анализа.") + return user_logs + except Exception as e: + self.log_error(f"Ошибка при извлечении логов активности: {e}") + return UserActivityLog.objects.none() def fetch_reservations(self): - """ Извлекает бронирования за период. """ - reservations = Reservation.objects.filter( - Q(check_in__lte=self.end_time) & Q(check_out__gte=self.start_time) - ) - self.log_info(f"Найдено {reservations.count()} бронирований для анализа.") - return reservations + try: + self.log_info("Начинается извлечение бронирований.") + reservations = Reservation.objects.filter( + Q(check_in__lte=self.end_time) & Q(check_out__gte=self.start_time) + ) + self.log_info(f"Найдено {reservations.count()} бронирований для анализа.") + return reservations + except Exception as e: + self.log_error(f"Ошибка при извлечении бронирований: {e}") + return Reservation.objects.none() def find_violations(self): - """ - Сравнивает записи бронирований и логи активности для выявления нарушений: - - Сканирование QR без бронирования. - - Бронь со статусом "заселен" без сканирования QR. - - Раннее заселение. - """ + self.log_info("Начинается анализ несоответствий.") user_logs = self.fetch_user_logs() reservations = self.fetch_reservations() - # Сопоставляем записи - log_lookup = {} # Словарь: (hotel_id, room_number) -> список логов + log_lookup = {} for log in user_logs: params = parse_qs(log.url_parameters or "") hotel_id = params.get("utm_content", [None])[0] @@ -70,7 +72,6 @@ class ReservationChecker: key = (reservation.hotel.hotel_id, reservation.room_number) logs = log_lookup.get(key, []) - # Бронь со статусом "заселен" без сканирования QR if reservation.status == "заселен" and not logs: self.record_violation( hotel=reservation.hotel, @@ -80,7 +81,6 @@ class ReservationChecker: f"не имеет записи сканирования QR." ) - # Раннее заселение for log in logs: if log.created < reservation.check_in: self.record_violation( @@ -91,7 +91,6 @@ class ReservationChecker: f"{reservation.check_in} для номера {reservation.room_number}." ) - # Сканирование QR без бронирования for (hotel_id, room_number), logs in log_lookup.items(): matching_reservations = reservations.filter( hotel__hotel_id=hotel_id, @@ -108,9 +107,6 @@ class ReservationChecker: ) def record_violation(self, hotel, room_number, violation_type, details): - """ - Записывает нарушение в список для последующего сохранения. - """ if hotel: self.violations.append(ViolationLog( hotel=hotel, @@ -121,7 +117,6 @@ class ReservationChecker: self.log_warning(f"Зафиксировано нарушение: {details}") def save_violations(self): - """ Сохраняет найденные нарушения в базу данных. """ if self.violations: ViolationLog.objects.bulk_create(self.violations) self.log_info(f"Создано {len(self.violations)} записей в ViolationLog.") @@ -129,7 +124,6 @@ class ReservationChecker: self.log_info("Нарушений не обнаружено.") def run_check(self): - """ Основной метод для запуска проверки. """ self.log_info(f"Запуск проверки с {self.start_time} по {self.end_time}.") try: self.find_violations() @@ -140,8 +134,10 @@ class ReservationChecker: # Функция для запуска из планировщика def run_reservation_check(): - checker = ReservationChecker() - checker.run_check() - - - + logger.info("Планировщик вызывает run_reservation_check.") + try: + checker = ReservationChecker() + checker.run_check() + except Exception as e: + logger.error(f"Ошибка при запуске проверки: {e}") + logger.info("run_reservation_check завершена.") \ No newline at end of file diff --git a/antifroud/data_sync.py b/antifroud/data_sync.py index 7cf38b2c..df58c3c9 100644 --- a/antifroud/data_sync.py +++ b/antifroud/data_sync.py @@ -6,10 +6,11 @@ from urllib.parse import unquote, parse_qs from django.utils import timezone import html from hotels.models import Room, Hotel -from .models import UserActivityLog, ExternalDBSettings +from .models import UserActivityLog, ExternalDBSettings, SyncLog from touchh.utils.log import CustomLogger from concurrent.futures import ThreadPoolExecutor, TimeoutError from decouple import config +from django.db.models import F class DatabaseConnector: def __init__(self, db_settings_id): @@ -40,8 +41,8 @@ class DatabaseConnector: def connect(self): try: - self.logger.info(f"Connecting to DB with settings: {self.db_settings}") self.connection = pymysql.connect( + host=self.db_settings["host"], port=self.db_settings["port"], user=self.db_settings["user"], @@ -78,9 +79,10 @@ class DataProcessor: self.logger = logger def decode_html_entities(self, text): - if text and isinstance(text, str): - return html.unescape(unquote(text)) - return text + if not text: + self.logger.warning("Empty text received for decoding HTML entities.") + return html.unescape(unquote(text)) if text else text + def parse_datetime(self, dt_str): try: @@ -92,7 +94,13 @@ class DataProcessor: self.logger.error(f"Datetime parsing error: {e}") return None - def url_parameters_parser(self, url_parameters): + def parse_url_parameters(self, url_parameters): + """ + Парсит строку URL-параметров в словарь. + + :param url_parameters: Строка с URL-параметрами. + :return: Словарь с распарсенными параметрами. + """ try: if not url_parameters: return {} @@ -100,7 +108,7 @@ class DataProcessor: parsed_params = parse_qs(decoded_params) return {key: value[0] for key, value in parsed_params.items()} except Exception as e: - self.logger.error(f"URL parameters parsing error: {e}") + self.logger.error(f"Error parsing URL parameters: {e}") return {} @@ -135,24 +143,26 @@ class HotelRoomManager: self.logger.warning(f"Room creation skipped: missing room_number for hotel {hotel.name}.") return None - external_id = f"{hotel.hotel_id}_{room_number}".lower() + try: + # Проверяем существование комнаты + room = Room.objects.filter(hotel=hotel, number=room_number).first() + if room: + self.logger.info(f"Room '{room_number}' already exists in hotel '{hotel.name}'.") + return room - room, created = Room.objects.get_or_create( - hotel=hotel, - number=room_number, - defaults={ - "external_id": external_id, - "description": "Автоматически созданная комната", - } - ) - - if created: - self.logger.info(f"Room '{room.number}' (external_id: {external_id}) created in hotel '{hotel.name}'") - else: - self.logger.info(f"Room '{room.number}' already exists in hotel '{hotel.name}'") - - return room + # Создаем комнату, если она не найдена + room = Room.objects.create( + hotel=hotel, + number=room_number, + external_id=f"{hotel.hotel_id}_{room_number}".lower(), + description="Automatically added room", + ) + self.logger.info(f"Room '{room.number}' created in hotel '{hotel.name}'.") + return room + except Exception as e: + self.logger.error(f"Error creating room '{room_number}' in hotel '{hotel.name}': {e}") + return None class DataSyncManager: def __init__(self, db_settings_id): @@ -167,6 +177,12 @@ class DataSyncManager: return record.id if record else 0 def fetch_new_data(self, last_id): + """ + Извлекает новые данные из таблицы для синхронизации. + + :param last_id: Последний обработанный ID. + :return: Список строк, полученных из базы данных. + """ query = f""" SELECT * FROM `{self.db_settings.get('table_name')}` WHERE id > {last_id} @@ -176,48 +192,94 @@ class DataSyncManager: ORDER BY id ASC LIMIT 1000; """ - self.logger.info(f"Fetching new data with query: {query}") - return self.db_connector.execute_query(query) - + + try: + rows = self.db_connector.execute_query(query) + self.logger.info(f"Fetched {len(rows)} records from the database.") + return rows + except Exception as e: + self.logger.error(f"Error fetching data: {e}") + return [] + def update_sync_log(self, hotel, recieved_records, processed_records): + try: + log, created = SyncLog.objects.get_or_create(hotel=hotel) + if created: + log.recieved_records = recieved_records + log.processed_records = processed_records + else: + log.recieved_records += recieved_records + log.processed_records += processed_records + log.save() + self.logger.info(f"Sync log updated for hotel '{hotel.name}'.") + except Exception as e: + self.logger.error(f"Error updating sync log for hotel '{hotel.name}': {e}") + def process_and_save_data(self, rows): + """ + Обрабатывает и сохраняет данные из внешнего источника. + + :param rows: Список строк данных, полученных из базы данных. + """ + seen_entries = set() + for row in rows: + # Получение и декодирование URL-параметров + url_parameters = row.get("url_parameters") + if not url_parameters: + self.logger.warning(f"Skipping record with missing URL parameters: {row}") + continue + + parsed_params = self.data_processor.parse_url_parameters(url_parameters) + hotel_id = parsed_params.get("utm_content") # Извлекаем hotel_id из параметров + room_number = parsed_params.get("utm_term") # Извлекаем room_number из параметров + + if not hotel_id or not room_number: + self.logger.warning(f"Skipping record with missing data: hotel_id={hotel_id}, room_number={room_number}") + continue + + # Проверка на дубликаты + if (hotel_id, room_number) in seen_entries: + self.logger.warning(f"Duplicate record skipped: hotel_id={hotel_id}, room_number={room_number}") + continue + + seen_entries.add((hotel_id, room_number)) + try: - url_params = self.data_processor.decode_html_entities(row.get("url_parameters", "")) - params = self.data_processor.url_parameters_parser(url_params) + # Получение или создание отеля + hotel = self.hotel_manager.get_or_create_hotel(hotel_id, row.get("page_title")) + if not hotel: + self.logger.warning(f"Skipping record: Failed to create or retrieve hotel with ID {hotel_id}") + continue - hotel_id = params.get("utm_content") - room_number = params.get("utm_term") - page_title = row.get("page_title") - external_id = row.get("id") - hits = row.get("hits") or 0 - - hotel = self.hotel_manager.get_or_create_hotel(hotel_id, page_title) + # Получение или создание комнаты room = self.hotel_manager.get_or_create_room(hotel, room_number) - page_url = row.get("page_url") + if not room: + self.logger.warning(f"Skipping record: Failed to create or retrieve room {room_number} in hotel {hotel.name}") + continue - if hits != 0 and page_title is not None: - UserActivityLog.objects.update_or_create( - external_id=external_id, - defaults={ - "user_id": row.get("user_id") or 0, - "timestamp": row.get("timestamp"), - "date_time": row.get("date_time"), - "ip": row.get("ip") or "0.0.0.0", - "created": self.data_processor.parse_datetime(row.get("created")) or timezone.now(), - "url_parameters": url_params, - "page_id": room.id if room else None, - "page_title": html.unescape(page_title), - "hits": hits, - "page_url": html.unescape(page_url), - } - ) - else: - self.logger.warning("Invalid data for UserActivityLog.") - self.logger.info(f"Record ID {external_id} processed successfully.") + # Создание или обновление записи активности пользователя + UserActivityLog.objects.update_or_create( + external_id=row.get("id"), + defaults={ + "user_id": row.get("user_id") or 0, + "ip": row.get("ip") or "0.0.0.0", + "created": self.data_processor.parse_datetime(row.get("created")), + "timestamp": row.get("timestamp") or datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "date_time": self.data_processor.parse_datetime(row.get("date_time")), + "url_parameters": parsed_params, + "page_title": self.data_processor.decode_html_entities(row.get("page_title")) or "Untitled", + "page_url": row.get("page_url") or "", + "hits": row.get("hits") or 0, + } + ) + self.logger.info(f"Record ID {row.get('id')} processed successfully.") except Exception as e: self.logger.error(f"Error processing record ID {row.get('id')}: {e}") + self.logger.info(f"Data processing completed. Processed {len(seen_entries)} unique records.") + + def sync(self): self.db_connector.connect() try: @@ -230,7 +292,7 @@ class DataSyncManager: def scheduled_sync(): - logger = CustomLogger(name="DatabaseSyncScheduler", log_level="DEBUG").get_logger() + logger = CustomLogger(name="DatabaseSyncScheduler", log_level="ERROR").get_logger() logger.info("Starting scheduled sync.") active_db_settings = ExternalDBSettings.objects.filter(is_active=True) @@ -256,5 +318,5 @@ def scheduled_sync(): future.result(timeout=300) except TimeoutError: logger.error("Sync task timed out.") - + logger.info("Scheduled sync completed.") diff --git a/antifroud/migrations/0018_remove_synclog_reservation_alter_synclog_created_and_more.py b/antifroud/migrations/0018_remove_synclog_reservation_alter_synclog_created_and_more.py new file mode 100644 index 00000000..43660759 --- /dev/null +++ b/antifroud/migrations/0018_remove_synclog_reservation_alter_synclog_created_and_more.py @@ -0,0 +1,44 @@ +# Generated by Django 5.1.4 on 2024-12-18 04:46 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('antifroud', '0017_violationlog_hits'), + ('hotels', '0014_alter_room_number'), + ] + + operations = [ + migrations.RemoveField( + model_name='synclog', + name='reservation', + ), + migrations.AlterField( + model_name='synclog', + name='created', + field=models.DateTimeField(auto_now=True, verbose_name='Дата обновления'), + ), + migrations.AlterField( + model_name='synclog', + name='hotel', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='hotels.hotel', unique=True, verbose_name='Отель'), + ), + migrations.AlterField( + model_name='synclog', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='synclog', + name='processed_records', + field=models.IntegerField(default=0, verbose_name='Обработанные записи'), + ), + migrations.AlterField( + model_name='synclog', + name='recieved_records', + field=models.IntegerField(default=0, verbose_name='Полученные записи'), + ), + ] diff --git a/antifroud/migrations/0019_alter_synclog_hotel.py b/antifroud/migrations/0019_alter_synclog_hotel.py new file mode 100644 index 00000000..2d699b44 --- /dev/null +++ b/antifroud/migrations/0019_alter_synclog_hotel.py @@ -0,0 +1,20 @@ +# Generated by Django 5.1.4 on 2024-12-18 04:55 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('antifroud', '0018_remove_synclog_reservation_alter_synclog_created_and_more'), + ('hotels', '0014_alter_room_number'), + ] + + operations = [ + migrations.AlterField( + model_name='synclog', + name='hotel', + field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='hotels.hotel', verbose_name='Отель'), + ), + ] diff --git a/antifroud/migrations/0020_alter_useractivitylog_hits.py b/antifroud/migrations/0020_alter_useractivitylog_hits.py new file mode 100644 index 00000000..67ad400f --- /dev/null +++ b/antifroud/migrations/0020_alter_useractivitylog_hits.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.4 on 2024-12-18 10:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('antifroud', '0019_alter_synclog_hotel'), + ] + + operations = [ + migrations.AlterField( + model_name='useractivitylog', + name='hits', + field=models.IntegerField(blank=True, default='0', null=True, verbose_name='Количество обращений'), + ), + ] diff --git a/antifroud/models.py b/antifroud/models.py index 582d4682..5434b5c8 100644 --- a/antifroud/models.py +++ b/antifroud/models.py @@ -1,7 +1,7 @@ from django.db import models from hotels.models import Hotel from hotels.models import Reservation - +from datetime import datetime, timezone class UserActivityLog(models.Model): external_id = models.CharField(max_length=255, unique=True, verbose_name="Внешний ID", db_index=True) @@ -23,11 +23,22 @@ class UserActivityLog(models.Model): page_title = models.TextField(blank=True, null=True, verbose_name="Заголовок страницы") type = models.CharField(max_length=50, verbose_name="Тип", blank=True, null=True) last_counter = models.IntegerField(verbose_name="Последний счетчик", blank=True, null=True) - hits = models.IntegerField(verbose_name="Количество обращений", blank=True, null=True) + hits = models.IntegerField(verbose_name="Количество обращений",default="0", blank=True, null=True) honeypot = models.BooleanField(verbose_name="Метка honeypot", blank=True, null=True) reply = models.BooleanField(verbose_name="Ответ пользователя", blank=True, null=True) page_url = models.URLField(blank=True, null=True, verbose_name="URL страницы") + @property + def formatted_timestamp(self): + """ + Преобразует Unix-временную метку в читаемую дату и время. + """ + if self.timestamp is not None: + return datetime.fromtimestamp(self.timestamp, tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S') + return "Нет данных" + + # Изменение имени столбца + class Meta: indexes = [ models.Index(fields=["external_id"], name="idx_external_id"), @@ -40,12 +51,21 @@ class UserActivityLog(models.Model): verbose_name_plural = "Логи активности пользователей" def __str__(self): return f"UserActivityLog {self.id}: {self.page_title}" - + class Meta: verbose_name = "Регистрация посетителей" verbose_name_plural = "Регистрации посетителей" - + def get_location(self): + if not self.ip: + return "IP-адрес отсутствует" + + try: + geoip_reader = Reader(settings.GEOIP_PATH + '/GeoLite2-City.mmdb') + response = geoip_reader.city(self.ip) + return f"{response.city.name}, {response.country.name}" + except Exception as e: + return "Местоположение недоступно" class ExternalDBSettings(models.Model): name = models.CharField(max_length=255, unique=True, help_text="Имя подключения для идентификации.") host = models.CharField(max_length=255, help_text="Адрес сервера базы данных.") @@ -165,18 +185,20 @@ class ImportedHotel(models.Model): class SyncLog(models.Model): """ - Журнал синхронизации. + Журнал синхронизации в разрезе отелей. """ - id = models.BigIntegerField(primary_key=True, unique=True, verbose_name="ID") - hotel = models.ForeignKey(Hotel, on_delete=models.CASCADE, verbose_name="Отель") - reservation = models.ForeignKey(Reservation, on_delete=models.CASCADE, verbose_name="Бронирование") - created = models.DateTimeField(auto_now_add=True, verbose_name="Дата создания") - recieved_records = models.IntegerField(verbose_name="Полученные записи") - processed_records = models.IntegerField(verbose_name="Обработанные записи") + hotel = models.OneToOneField(Hotel, on_delete=models.CASCADE, verbose_name="Отель") + created = models.DateTimeField(auto_now=True, verbose_name="Дата обновления") # Последняя дата обновления записи + recieved_records = models.IntegerField(default=0, verbose_name="Полученные записи") + processed_records = models.IntegerField(default=0, verbose_name="Обработанные записи") class Meta: verbose_name = "Журнал синхронизации" verbose_name_plural = "Журналы синхронизации" + + def __str__(self): + return f"Отель: {self.hotel.name} | Получено: {self.recieved_records} | Обработано: {self.processed_records}" + class ViolationLog(models.Model): diff --git a/scheduler/models.py b/scheduler/models.py index bb49a1e7..241ce945 100644 --- a/scheduler/models.py +++ b/scheduler/models.py @@ -1,6 +1,9 @@ from django.db import models from django.utils.timezone import now +import logging +logger = logging.getLogger(__name__) +logger.info("Загрузка модели ScheduledTask") class ScheduledTask(models.Model): task_name = models.CharField(max_length=255) diff --git a/touchh/settings.py b/touchh/settings.py index 621f8931..f2792e8c 100644 --- a/touchh/settings.py +++ b/touchh/settings.py @@ -90,7 +90,7 @@ TEMPLATES = [ ] WSGI_APPLICATION = 'touchh.wsgi.application' - +GEOIP_PATH = os.path.join(BASE_DIR, 'geoip') # Database # https://docs.djangoproject.com/en/5.1/ref/settings/#databases