sMerge branch 'master' of git.smartsoltech.kr:trevor/touchh_bot

antifroud/admin.py

@@ -10,7 +10,7 @@ from hotels.models import Hotel, Room
 import pymysql
 import logging
 from django.urls import reverse
-
+from datetime import datetime
 
 logger = logging.getLogger(__name__)
 
@@ -111,10 +111,19 @@ class ExternalDBSettingsAdmin(admin.ModelAdmin):
 
 @admin.register(UserActivityLog)
 class UserActivityLogAdmin(admin.ModelAdmin):
-    list_display = ("id", "timestamp", "date_time", "page_id", "url_parameters", "page_url" ,"created", "page_title", "type", "hits")
+    list_display = ("id", 'get_location',"formatted_timestamp", "date_time", "page_id", "url_parameters", "page_url" ,"created", "page_title", "type", "hits")
     search_fields = ("page_title", "url_parameters")
     list_filter = ("page_title", "created")
     readonly_fields = ("created", "timestamp")
+    
+    def get_formatted_timestamp(self, obj):
+        """
+        Метод для админки для преобразования timestamp в читаемый формат.
+        """
+        return obj.formatted_timestamp  # Используем свойство модели
+    get_formatted_timestamp.short_description = "Таймштамп"
+    
+    
     def get_hotel_name(self):
         """
         Возвращает название отеля на основе связанного page_id.
@@ -142,7 +151,6 @@ class UserActivityLogAdmin(admin.ModelAdmin):
     get_hotel_name.short_description = "Отель"
     get_room_number.short_description = "Комната"
 
-
 from .views import import_selected_hotels
 # Регистрируем admin класс для ImportedHotel
 @admin.register(ImportedHotel)
@@ -197,13 +205,13 @@ class ImportedHotelAdmin(admin.ModelAdmin):
 @admin.register(SyncLog)
 class SyncLogAdmin(admin.ModelAdmin):
     change_list_template = "antifroud/admin/sync_log.html"
-    list_display =['id', 'hotel', 'recieved_records', 'processed_records']
-    search_fields = ['id', 'hotel', 'received_records', 'processed_records']
-    list_filter = ['id', 'hotel', 'processed_records']
+    list_display =['id', 'hotel', 'created', 'recieved_records', 'processed_records']
+    search_fields = ['id', 'hotel', 'created', 'recieved_records', 'processed_records']
+    list_filter = ['id', 'hotel', 'created', 'recieved_records', 'processed_records']
 
     class Meta:
         model = SyncLog
-        fields = ['hotel', 'received_records', 'processed_records']
+        fields = ['hotel', 'recieved_records', 'processed_records']
         
 
 @admin.register(ViolationLog)

antifroud/check_fraud.py

@@ -33,31 +33,33 @@ class ReservationChecker:
         logger.error(message)
 
     def fetch_user_logs(self):
-        """ Извлекает записи из UserActivityLog за период. """
+        try:
+            self.log_info("Начинается извлечение логов активности пользователей.")
             user_logs = UserActivityLog.objects.filter(created__range=(self.start_time, self.end_time))
             self.log_info(f"Найдено {user_logs.count()} логов активности для анализа.")
             return user_logs
+        except Exception as e:
+            self.log_error(f"Ошибка при извлечении логов активности: {e}")
+            return UserActivityLog.objects.none()
 
     def fetch_reservations(self):
-        """ Извлекает бронирования за период. """
+        try:
+            self.log_info("Начинается извлечение бронирований.")
             reservations = Reservation.objects.filter(
                 Q(check_in__lte=self.end_time) & Q(check_out__gte=self.start_time)
             )
             self.log_info(f"Найдено {reservations.count()} бронирований для анализа.")
             return reservations
+        except Exception as e:
+            self.log_error(f"Ошибка при извлечении бронирований: {e}")
+            return Reservation.objects.none()
 
     def find_violations(self):
-        """
-        Сравнивает записи бронирований и логи активности для выявления нарушений:
-        - Сканирование QR без бронирования.
-        - Бронь со статусом "заселен" без сканирования QR.
-        - Раннее заселение.
-        """
+        self.log_info("Начинается анализ несоответствий.")
         user_logs = self.fetch_user_logs()
         reservations = self.fetch_reservations()
 
-        # Сопоставляем записи
-        log_lookup = {}  # Словарь: (hotel_id, room_number) -> список логов
+        log_lookup = {}
         for log in user_logs:
             params = parse_qs(log.url_parameters or "")
             hotel_id = params.get("utm_content", [None])[0]
@@ -70,7 +72,6 @@ class ReservationChecker:
             key = (reservation.hotel.hotel_id, reservation.room_number)
             logs = log_lookup.get(key, [])
 
-            # Бронь со статусом "заселен" без сканирования QR
             if reservation.status == "заселен" and not logs:
                 self.record_violation(
                     hotel=reservation.hotel,
@@ -80,7 +81,6 @@ class ReservationChecker:
                             f"не имеет записи сканирования QR."
                 )
 
-            # Раннее заселение
             for log in logs:
                 if log.created < reservation.check_in:
                     self.record_violation(
@@ -91,7 +91,6 @@ class ReservationChecker:
                                 f"{reservation.check_in} для номера {reservation.room_number}."
                     )
 
-        # Сканирование QR без бронирования
         for (hotel_id, room_number), logs in log_lookup.items():
             matching_reservations = reservations.filter(
                 hotel__hotel_id=hotel_id,
@@ -108,9 +107,6 @@ class ReservationChecker:
                     )
 
     def record_violation(self, hotel, room_number, violation_type, details):
-        """
-        Записывает нарушение в список для последующего сохранения.
-        """
         if hotel:
             self.violations.append(ViolationLog(
                 hotel=hotel,
@@ -121,7 +117,6 @@ class ReservationChecker:
             self.log_warning(f"Зафиксировано нарушение: {details}")
 
     def save_violations(self):
-        """ Сохраняет найденные нарушения в базу данных. """
         if self.violations:
             ViolationLog.objects.bulk_create(self.violations)
             self.log_info(f"Создано {len(self.violations)} записей в ViolationLog.")
@@ -129,7 +124,6 @@ class ReservationChecker:
             self.log_info("Нарушений не обнаружено.")
 
     def run_check(self):
-        """ Основной метод для запуска проверки. """
         self.log_info(f"Запуск проверки с {self.start_time} по {self.end_time}.")
         try:
             self.find_violations()
@@ -140,8 +134,10 @@ class ReservationChecker:
 
 # Функция для запуска из планировщика
 def run_reservation_check():
+    logger.info("Планировщик вызывает run_reservation_check.")
+    try:
         checker = ReservationChecker()
         checker.run_check()
-
-
-
+    except Exception as e:
+        logger.error(f"Ошибка при запуске проверки: {e}")
+    logger.info("run_reservation_check завершена.")

antifroud/data_sync.py

@@ -6,10 +6,11 @@ from urllib.parse import unquote, parse_qs
 from django.utils import timezone
 import html
 from hotels.models import Room, Hotel
-from .models import UserActivityLog, ExternalDBSettings
+from .models import UserActivityLog, ExternalDBSettings, SyncLog
 from touchh.utils.log import CustomLogger
 from concurrent.futures import ThreadPoolExecutor, TimeoutError
 from decouple import config
+from django.db.models import F
 
 class DatabaseConnector:
     def __init__(self, db_settings_id):
@@ -40,8 +41,8 @@ class DatabaseConnector:
 
     def connect(self):
         try:
-            self.logger.info(f"Connecting to DB with settings: {self.db_settings}")
             self.connection = pymysql.connect(
+
                 host=self.db_settings["host"],
                 port=self.db_settings["port"],
                 user=self.db_settings["user"],
@@ -78,9 +79,10 @@ class DataProcessor:
         self.logger = logger
 
     def decode_html_entities(self, text):
-        if text and isinstance(text, str):
-            return html.unescape(unquote(text))
-        return text
+        if not text:
+            self.logger.warning("Empty text received for decoding HTML entities.")
+        return html.unescape(unquote(text)) if text else text
+
 
     def parse_datetime(self, dt_str):
         try:
@@ -92,7 +94,13 @@ class DataProcessor:
             self.logger.error(f"Datetime parsing error: {e}")
         return None
 
-    def url_parameters_parser(self, url_parameters):
+    def parse_url_parameters(self, url_parameters):
+        """
+        Парсит строку URL-параметров в словарь.
+
+        :param url_parameters: Строка с URL-параметрами.
+        :return: Словарь с распарсенными параметрами.
+        """
         try:
             if not url_parameters:
                 return {}
@@ -100,7 +108,7 @@ class DataProcessor:
             parsed_params = parse_qs(decoded_params)
             return {key: value[0] for key, value in parsed_params.items()}
         except Exception as e:
-            self.logger.error(f"URL parameters parsing error: {e}")
+            self.logger.error(f"Error parsing URL parameters: {e}")
             return {}
 
 
@@ -135,24 +143,26 @@ class HotelRoomManager:
             self.logger.warning(f"Room creation skipped: missing room_number for hotel {hotel.name}.")
             return None
 
-        external_id = f"{hotel.hotel_id}_{room_number}".lower()
-
-        room, created = Room.objects.get_or_create(
-            hotel=hotel,
-            number=room_number,
-            defaults={
-                "external_id": external_id,
-                "description": "Автоматически созданная комната",
-            }
-        )
-
-        if created:
-            self.logger.info(f"Room '{room.number}' (external_id: {external_id}) created in hotel '{hotel.name}'")
-        else:
-            self.logger.info(f"Room '{room.number}' already exists in hotel '{hotel.name}'")
-
+        try:
+            # Проверяем существование комнаты
+            room = Room.objects.filter(hotel=hotel, number=room_number).first()
+            if room:
+                self.logger.info(f"Room '{room_number}' already exists in hotel '{hotel.name}'.")
                 return room
 
+            # Создаем комнату, если она не найдена
+            room = Room.objects.create(
+                hotel=hotel,
+                number=room_number,
+                external_id=f"{hotel.hotel_id}_{room_number}".lower(),
+                description="Automatically added room",
+            )
+            self.logger.info(f"Room '{room.number}' created in hotel '{hotel.name}'.")
+            return room
+
+        except Exception as e:
+            self.logger.error(f"Error creating room '{room_number}' in hotel '{hotel.name}': {e}")
+            return None
 
 class DataSyncManager:
     def __init__(self, db_settings_id):
@@ -167,6 +177,12 @@ class DataSyncManager:
         return record.id if record else 0
 
     def fetch_new_data(self, last_id):
+        """
+        Извлекает новые данные из таблицы для синхронизации.
+
+        :param last_id: Последний обработанный ID.
+        :return: Список строк, полученных из базы данных.
+        """
         query = f"""
             SELECT * FROM `{self.db_settings.get('table_name')}`
             WHERE id > {last_id}
@@ -176,48 +192,94 @@ class DataSyncManager:
             ORDER BY id ASC
             LIMIT 1000;
         """
-        
         self.logger.info(f"Fetching new data with query: {query}")
-        return self.db_connector.execute_query(query)
+        
+        try:
+            rows = self.db_connector.execute_query(query)
+            self.logger.info(f"Fetched {len(rows)} records from the database.")
+            return rows
+        except Exception as e:
+            self.logger.error(f"Error fetching data: {e}")
+            return []
+    def update_sync_log(self, hotel, recieved_records, processed_records):
+        try:
+            log, created = SyncLog.objects.get_or_create(hotel=hotel)
+            if created:
+                log.recieved_records = recieved_records
+                log.processed_records = processed_records
+            else:
+                log.recieved_records += recieved_records
+                log.processed_records += processed_records
+            log.save()
+            self.logger.info(f"Sync log updated for hotel '{hotel.name}'.")
+        except Exception as e:
+            self.logger.error(f"Error updating sync log for hotel '{hotel.name}': {e}")
             
     def process_and_save_data(self, rows):
+        """
+        Обрабатывает и сохраняет данные из внешнего источника.
+
+        :param rows: Список строк данных, полученных из базы данных.
+        """
+        seen_entries = set()
+
         for row in rows:
+            # Получение и декодирование URL-параметров
+            url_parameters = row.get("url_parameters")
+            if not url_parameters:
+                self.logger.warning(f"Skipping record with missing URL parameters: {row}")
+                continue
+
+            parsed_params = self.data_processor.parse_url_parameters(url_parameters)
+            hotel_id = parsed_params.get("utm_content")  # Извлекаем hotel_id из параметров
+            room_number = parsed_params.get("utm_term")  # Извлекаем room_number из параметров
+
+            if not hotel_id or not room_number:
+                self.logger.warning(f"Skipping record with missing data: hotel_id={hotel_id}, room_number={room_number}")
+                continue
+
+            # Проверка на дубликаты
+            if (hotel_id, room_number) in seen_entries:
+                self.logger.warning(f"Duplicate record skipped: hotel_id={hotel_id}, room_number={room_number}")
+                continue
+
+            seen_entries.add((hotel_id, room_number))
+
             try:
-                url_params = self.data_processor.decode_html_entities(row.get("url_parameters", ""))
-                params = self.data_processor.url_parameters_parser(url_params)
+                # Получение или создание отеля
+                hotel = self.hotel_manager.get_or_create_hotel(hotel_id, row.get("page_title"))
+                if not hotel:
+                    self.logger.warning(f"Skipping record: Failed to create or retrieve hotel with ID {hotel_id}")
+                    continue
 
-                hotel_id = params.get("utm_content")
-                room_number = params.get("utm_term")
-                page_title = row.get("page_title")
-                external_id = row.get("id")
-                hits = row.get("hits") or 0
-
-                hotel = self.hotel_manager.get_or_create_hotel(hotel_id, page_title)
+                # Получение или создание комнаты
                 room = self.hotel_manager.get_or_create_room(hotel, room_number)
-                page_url = row.get("page_url")
+                if not room:
+                    self.logger.warning(f"Skipping record: Failed to create or retrieve room {room_number} in hotel {hotel.name}")
+                    continue
 
-                if hits != 0 and page_title is not None:
+                # Создание или обновление записи активности пользователя
                 UserActivityLog.objects.update_or_create(
-                        external_id=external_id,
+                    external_id=row.get("id"),
                     defaults={
                         "user_id": row.get("user_id") or 0,
-                            "timestamp": row.get("timestamp"),
-                            "date_time": row.get("date_time"),
                         "ip": row.get("ip") or "0.0.0.0",
-                            "created": self.data_processor.parse_datetime(row.get("created")) or timezone.now(),
-                            "url_parameters": url_params,
-                            "page_id": room.id if room else None,
-                            "page_title": html.unescape(page_title),
-                            "hits": hits,
-                            "page_url": html.unescape(page_url),
+                        "created": self.data_processor.parse_datetime(row.get("created")),
+                        "timestamp": row.get("timestamp") or datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                        "date_time": self.data_processor.parse_datetime(row.get("date_time")),
+                        "url_parameters": parsed_params,
+                        "page_title": self.data_processor.decode_html_entities(row.get("page_title")) or "Untitled",
+                        "page_url": row.get("page_url") or "",
+                        "hits": row.get("hits") or 0,
                     }
                 )
-                else:
-                    self.logger.warning("Invalid data for UserActivityLog.")
-                self.logger.info(f"Record ID {external_id} processed successfully.")
+                self.logger.info(f"Record ID {row.get('id')} processed successfully.")
             except Exception as e:
                 self.logger.error(f"Error processing record ID {row.get('id')}: {e}")
 
+        self.logger.info(f"Data processing completed. Processed {len(seen_entries)} unique records.")
+
+
     def sync(self):
         self.db_connector.connect()
         try:
@@ -230,7 +292,7 @@ class DataSyncManager:
 
 
 def scheduled_sync():
-    logger = CustomLogger(name="DatabaseSyncScheduler", log_level="DEBUG").get_logger()
+    logger = CustomLogger(name="DatabaseSyncScheduler", log_level="ERROR").get_logger()
     logger.info("Starting scheduled sync.")
 
     active_db_settings = ExternalDBSettings.objects.filter(is_active=True)

antifroud/migrations/0018_remove_synclog_reservation_alter_synclog_created_and_more.py

@@ -0,0 +1,44 @@
+# Generated by Django 5.1.4 on 2024-12-18 04:46
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('antifroud', '0017_violationlog_hits'),
+        ('hotels', '0014_alter_room_number'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='synclog',
+            name='reservation',
+        ),
+        migrations.AlterField(
+            model_name='synclog',
+            name='created',
+            field=models.DateTimeField(auto_now=True, verbose_name='Дата обновления'),
+        ),
+        migrations.AlterField(
+            model_name='synclog',
+            name='hotel',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='hotels.hotel', unique=True, verbose_name='Отель'),
+        ),
+        migrations.AlterField(
+            model_name='synclog',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='synclog',
+            name='processed_records',
+            field=models.IntegerField(default=0, verbose_name='Обработанные записи'),
+        ),
+        migrations.AlterField(
+            model_name='synclog',
+            name='recieved_records',
+            field=models.IntegerField(default=0, verbose_name='Полученные записи'),
+        ),
+    ]

antifroud/migrations/0019_alter_synclog_hotel.py

@@ -0,0 +1,20 @@
+# Generated by Django 5.1.4 on 2024-12-18 04:55
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('antifroud', '0018_remove_synclog_reservation_alter_synclog_created_and_more'),
+        ('hotels', '0014_alter_room_number'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='synclog',
+            name='hotel',
+            field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='hotels.hotel', verbose_name='Отель'),
+        ),
+    ]

antifroud/migrations/0020_alter_useractivitylog_hits.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.4 on 2024-12-18 10:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('antifroud', '0019_alter_synclog_hotel'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='useractivitylog',
+            name='hits',
+            field=models.IntegerField(blank=True, default='0', null=True, verbose_name='Количество обращений'),
+        ),
+    ]

antifroud/models.py

@@ -1,7 +1,11 @@
 from django.db import models
 from hotels.models import Hotel
 from hotels.models import Reservation
-
+from datetime import datetime, timezone
+from geoip2.errors import AddressNotFoundError
+from geoip2.database import Reader
+from django.conf import settings
+import logging
 
 class UserActivityLog(models.Model):
     external_id = models.CharField(max_length=255, unique=True, verbose_name="Внешний ID", db_index=True)
@@ -23,11 +27,22 @@ class UserActivityLog(models.Model):
     page_title = models.TextField(blank=True, null=True, verbose_name="Заголовок страницы")
     type = models.CharField(max_length=50, verbose_name="Тип", blank=True, null=True)
     last_counter = models.IntegerField(verbose_name="Последний счетчик", blank=True, null=True)
-    hits = models.IntegerField(verbose_name="Количество обращений", blank=True, null=True)
+    hits = models.IntegerField(verbose_name="Количество обращений",default="0", blank=True, null=True)
     honeypot = models.BooleanField(verbose_name="Метка honeypot", blank=True, null=True)
     reply = models.BooleanField(verbose_name="Ответ пользователя", blank=True, null=True)
     page_url = models.URLField(blank=True, null=True, verbose_name="URL страницы")
 
+    @property
+    def formatted_timestamp(self):
+        """
+        Преобразует Unix-временную метку в читаемую дату и время.
+        """
+        if self.timestamp is not None:
+            return datetime.fromtimestamp(self.timestamp, tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
+        return "Нет данных"
+
+    # Изменение имени столбца
+    
     class Meta:
         indexes = [
             models.Index(fields=["external_id"], name="idx_external_id"),
@@ -45,7 +60,29 @@ class UserActivityLog(models.Model):
         verbose_name = "Регистрация посетителей"
         verbose_name_plural = "Регистрации посетителей"
 
+    def get_location(self):
+        if not self.ip:
+            return "IP-адрес отсутствует"
         
+        try:
+            db_path = f"{settings.GEOIP_PATH}/GeoLite2-City.mmdb"
+            geoip_reader = Reader(db_path)
+            response = geoip_reader.city(self.ip)
+
+            # Извлекаем город и страну на русском языке
+            city = response.city.names.get('ru', "Город неизвестен")
+            country = response.country.names.get('ru', "Страна неизвестна")
+
+            return f"{city}, {country}"
+
+        except AddressNotFoundError:
+            return "IP-адрес не найден в базе"
+        except FileNotFoundError:
+            logger.error(f"Файл базы данных GeoIP не найден по пути: {db_path}")
+            return "Файл базы данных GeoIP не найден"
+        except Exception as e:
+            logger.error(f"Ошибка при определении местоположения: {e}")
+            return "Местоположение недоступно"
 class ExternalDBSettings(models.Model):
     name = models.CharField(max_length=255, unique=True, help_text="Имя подключения для идентификации.")
     host = models.CharField(max_length=255, help_text="Адрес сервера базы данных.")
@@ -165,19 +202,21 @@ class ImportedHotel(models.Model):
 
 class SyncLog(models.Model):
     """
-    Журнал синхронизации.
+    Журнал синхронизации в разрезе отелей.
     """
-    id = models.BigIntegerField(primary_key=True, unique=True, verbose_name="ID")
-    hotel = models.ForeignKey(Hotel, on_delete=models.CASCADE, verbose_name="Отель")
-    reservation = models.ForeignKey(Reservation, on_delete=models.CASCADE, verbose_name="Бронирование")
-    created = models.DateTimeField(auto_now_add=True, verbose_name="Дата создания")
-    recieved_records = models.IntegerField(verbose_name="Полученные записи")
-    processed_records = models.IntegerField(verbose_name="Обработанные записи")
+    hotel = models.OneToOneField(Hotel, on_delete=models.CASCADE, verbose_name="Отель")
+    created = models.DateTimeField(auto_now=True, verbose_name="Дата обновления")  # Последняя дата обновления записи
+    recieved_records = models.IntegerField(default=0, verbose_name="Полученные записи")
+    processed_records = models.IntegerField(default=0, verbose_name="Обработанные записи")
 
     class Meta:
         verbose_name = "Журнал синхронизации"
         verbose_name_plural = "Журналы синхронизации"
 
+    def __str__(self):
+        return f"Отель: {self.hotel.name} | Получено: {self.recieved_records} | Обработано: {self.processed_records}"
+
+        
         
 class ViolationLog(models.Model):
     hotel = models.ForeignKey(Hotel, on_delete=models.CASCADE, verbose_name="Отель")

scheduler/models.py

@@ -1,6 +1,9 @@
 from django.db import models
 from django.utils.timezone import now
+import logging
 
+logger = logging.getLogger(__name__)
+logger.info("Загрузка модели ScheduledTask")
 
 class ScheduledTask(models.Model):
     task_name = models.CharField(max_length=255)

touchh/settings.py

@@ -90,7 +90,7 @@ TEMPLATES = [
 ]
 
 WSGI_APPLICATION = 'touchh.wsgi.application'
-
+GEOIP_PATH = os.path.join(BASE_DIR, 'geoip')
 
 # Database
 # https://docs.djangoproject.com/en/5.1/ref/settings/#databases

var_clean.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# Получаем список всех снапов с ревизиями, фильтруем по статусу "disabled"
+snap list --all | awk '/disabled/{print $1,$3}' | while read SNAP_NAME REVISION; do
+    echo "Удаляю $SNAP_NAME ревизию $REVISION..."
+    sudo snap remove "$SNAP_NAME" --revision="$REVISION"
+done