SmartSolTech/Touchh
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

geoip intgration

Browse Source
This commit is contained in:
trevor
2024-12-18 20:02:13 +09:00
parent 1e64a432ab
commit de1059bca1
9 changed files with 280 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
antifroud/admin.py
View File

@@ -10,7 +10,7 @@ from hotels.models import Hotel, Room
import pymysql
import logging
from django.urls import reverse
from datetime import datetime
logger = logging.getLogger(__name__)
@@ -111,10 +111,19 @@ class ExternalDBSettingsAdmin(admin.ModelAdmin):
@admin.register(UserActivityLog)
class UserActivityLogAdmin(admin.ModelAdmin):
list_display = ("id", "timestamp", "date_time", "page_id", "url_parameters", "page_url" ,"created", "page_title", "type", "hits")
list_display = ("id", "ip", 'get_location',"formatted_timestamp", "date_time", "page_id", "url_parameters", "page_url" ,"created", "page_title", "type", "hits")
search_fields = ("page_title", "url_parameters")
list_filter = ("page_title", "created")
readonly_fields = ("created", "timestamp")
def get_formatted_timestamp(self, obj):
"""
Метод для админки для преобразования timestamp в читаемый формат.
"""
return obj.formatted_timestamp # Используем свойство модели
get_formatted_timestamp.short_description = "Таймштамп"
def get_hotel_name(self):
"""
Возвращает название отеля на основе связанного page_id.
@@ -142,7 +151,6 @@ class UserActivityLogAdmin(admin.ModelAdmin):
get_hotel_name.short_description = "Отель"
get_room_number.short_description = "Комната"
from .views import import_selected_hotels
# Регистрируем admin класс для ImportedHotel
@admin.register(ImportedHotel)
@@ -197,13 +205,13 @@ class ImportedHotelAdmin(admin.ModelAdmin):
@admin.register(SyncLog)
class SyncLogAdmin(admin.ModelAdmin):
change_list_template = "antifroud/admin/sync_log.html"
list_display =['id', 'hotel', 'recieved_records', 'processed_records']
search_fields = ['id', 'hotel', 'received_records', 'processed_records']
list_filter = ['id', 'hotel', 'processed_records']
list_display =['id', 'hotel', 'created', 'recieved_records', 'processed_records']
search_fields = ['id', 'hotel', 'created', 'recieved_records', 'processed_records']
list_filter = ['id', 'hotel', 'created', 'recieved_records', 'processed_records']
class Meta:
model = SyncLog
fields = ['hotel', 'received_records', 'processed_records']
fields = ['hotel', 'recieved_records', 'processed_records']
@admin.register(ViolationLog)

58
antifroud/check_fraud.py
View File

@@ -33,31 +33,33 @@ class ReservationChecker:
logger.error(message)
def fetch_user_logs(self):
""" Извлекает записи из UserActivityLog за период. """
user_logs = UserActivityLog.objects.filter(created__range=(self.start_time, self.end_time))
self.log_info(f"Найдено {user_logs.count()} логов активности для анализа.")
return user_logs
try:
self.log_info("Начинается извлечение логов активности пользователей.")
user_logs = UserActivityLog.objects.filter(created__range=(self.start_time, self.end_time))
self.log_info(f"Найдено {user_logs.count()} логов активности для анализа.")
return user_logs
except Exception as e:
self.log_error(f"Ошибка при извлечении логов активности: {e}")
return UserActivityLog.objects.none()
def fetch_reservations(self):
""" Извлекает бронирования за период. """
reservations = Reservation.objects.filter(
Q(check_in__lte=self.end_time) & Q(check_out__gte=self.start_time)
)
self.log_info(f"Найдено {reservations.count()} бронирований для анализа.")
return reservations
try:
self.log_info("Начинается извлечение бронирований.")
reservations = Reservation.objects.filter(
Q(check_in__lte=self.end_time) & Q(check_out__gte=self.start_time)
)
self.log_info(f"Найдено {reservations.count()} бронирований для анализа.")
return reservations
except Exception as e:
self.log_error(f"Ошибка при извлечении бронирований: {e}")
return Reservation.objects.none()
def find_violations(self):
"""
Сравнивает записи бронирований и логи активности для выявления нарушений:
- Сканирование QR без бронирования.
- Бронь со статусом "заселен" без сканирования QR.
- Раннее заселение.
"""
self.log_info("Начинается анализ несоответствий.")
user_logs = self.fetch_user_logs()
reservations = self.fetch_reservations()
# Сопоставляем записи
log_lookup = {} # Словарь: (hotel_id, room_number) -> список логов
log_lookup = {}
for log in user_logs:
params = parse_qs(log.url_parameters or "")
hotel_id = params.get("utm_content", [None])[0]
@@ -70,7 +72,6 @@ class ReservationChecker:
key = (reservation.hotel.hotel_id, reservation.room_number)
logs = log_lookup.get(key, [])
# Бронь со статусом "заселен" без сканирования QR
if reservation.status == "заселен" and not logs:
self.record_violation(
hotel=reservation.hotel,
@@ -80,7 +81,6 @@ class ReservationChecker:
f"не имеет записи сканирования QR."
)
# Раннее заселение
for log in logs:
if log.created < reservation.check_in:
self.record_violation(
@@ -91,7 +91,6 @@ class ReservationChecker:
f"{reservation.check_in} для номера {reservation.room_number}."
)
# Сканирование QR без бронирования
for (hotel_id, room_number), logs in log_lookup.items():
matching_reservations = reservations.filter(
hotel__hotel_id=hotel_id,
@@ -108,9 +107,6 @@ class ReservationChecker:
)
def record_violation(self, hotel, room_number, violation_type, details):
"""
Записывает нарушение в список для последующего сохранения.
"""
if hotel:
self.violations.append(ViolationLog(
hotel=hotel,
@@ -121,7 +117,6 @@ class ReservationChecker:
self.log_warning(f"Зафиксировано нарушение: {details}")
def save_violations(self):
""" Сохраняет найденные нарушения в базу данных. """
if self.violations:
ViolationLog.objects.bulk_create(self.violations)
self.log_info(f"Создано {len(self.violations)} записей в ViolationLog.")
@@ -129,7 +124,6 @@ class ReservationChecker:
self.log_info("Нарушений не обнаружено.")
def run_check(self):
""" Основной метод для запуска проверки. """
self.log_info(f"Запуск проверки с {self.start_time} по {self.end_time}.")
try:
self.find_violations()
@@ -140,8 +134,10 @@ class ReservationChecker:
# Функция для запуска из планировщика
def run_reservation_check():
checker = ReservationChecker()
checker.run_check()
logger.info("Планировщик вызывает run_reservation_check.")
try:
checker = ReservationChecker()
checker.run_check()
except Exception as e:
logger.error(f"Ошибка при запуске проверки: {e}")
logger.info("run_reservation_check завершена.")

176
antifroud/data_sync.py
View File

@@ -6,10 +6,11 @@ from urllib.parse import unquote, parse_qs
from django.utils import timezone
import html
from hotels.models import Room, Hotel
from .models import UserActivityLog, ExternalDBSettings
from .models import UserActivityLog, ExternalDBSettings, SyncLog
from touchh.utils.log import CustomLogger
from concurrent.futures import ThreadPoolExecutor, TimeoutError
from decouple import config
from django.db.models import F
class DatabaseConnector:
def __init__(self, db_settings_id):
@@ -40,8 +41,8 @@ class DatabaseConnector:
def connect(self):
try:
self.logger.info(f"Connecting to DB with settings: {self.db_settings}")
self.connection = pymysql.connect(
host=self.db_settings["host"],
port=self.db_settings["port"],
user=self.db_settings["user"],
@@ -78,9 +79,10 @@ class DataProcessor:
self.logger = logger
def decode_html_entities(self, text):
if text and isinstance(text, str):
return html.unescape(unquote(text))
return text
if not text:
self.logger.warning("Empty text received for decoding HTML entities.")
return html.unescape(unquote(text)) if text else text
def parse_datetime(self, dt_str):
try:
@@ -92,7 +94,13 @@ class DataProcessor:
self.logger.error(f"Datetime parsing error: {e}")
return None
def url_parameters_parser(self, url_parameters):
def parse_url_parameters(self, url_parameters):
"""
Парсит строку URL-параметров в словарь.
:param url_parameters: Строка с URL-параметрами.
:return: Словарь с распарсенными параметрами.
"""
try:
if not url_parameters:
return {}
@@ -100,7 +108,7 @@ class DataProcessor:
parsed_params = parse_qs(decoded_params)
return {key: value[0] for key, value in parsed_params.items()}
except Exception as e:
self.logger.error(f"URL parameters parsing error: {e}")
self.logger.error(f"Error parsing URL parameters: {e}")
return {}
@@ -135,24 +143,26 @@ class HotelRoomManager:
self.logger.warning(f"Room creation skipped: missing room_number for hotel {hotel.name}.")
return None
external_id = f"{hotel.hotel_id}_{room_number}".lower()
try:
# Проверяем существование комнаты
room = Room.objects.filter(hotel=hotel, number=room_number).first()
if room:
self.logger.info(f"Room '{room_number}' already exists in hotel '{hotel.name}'.")
return room
room, created = Room.objects.get_or_create(
hotel=hotel,
number=room_number,
defaults={
"external_id": external_id,
"description": "Автоматически созданная комната",
}
)
if created:
self.logger.info(f"Room '{room.number}' (external_id: {external_id}) created in hotel '{hotel.name}'")
else:
self.logger.info(f"Room '{room.number}' already exists in hotel '{hotel.name}'")
return room
# Создаем комнату, если она не найдена
room = Room.objects.create(
hotel=hotel,
number=room_number,
external_id=f"{hotel.hotel_id}_{room_number}".lower(),
description="Automatically added room",
)
self.logger.info(f"Room '{room.number}' created in hotel '{hotel.name}'.")
return room
except Exception as e:
self.logger.error(f"Error creating room '{room_number}' in hotel '{hotel.name}': {e}")
return None
class DataSyncManager:
def __init__(self, db_settings_id):
@@ -167,6 +177,12 @@ class DataSyncManager:
return record.id if record else 0
def fetch_new_data(self, last_id):
"""
Извлекает новые данные из таблицы для синхронизации.
:param last_id: Последний обработанный ID.
:return: Список строк, полученных из базы данных.
"""
query = f"""
SELECT * FROM `{self.db_settings.get('table_name')}`
WHERE id > {last_id}
@@ -176,48 +192,94 @@ class DataSyncManager:
ORDER BY id ASC
LIMIT 1000;
"""
self.logger.info(f"Fetching new data with query: {query}")
return self.db_connector.execute_query(query)
try:
rows = self.db_connector.execute_query(query)
self.logger.info(f"Fetched {len(rows)} records from the database.")
return rows
except Exception as e:
self.logger.error(f"Error fetching data: {e}")
return []
def update_sync_log(self, hotel, recieved_records, processed_records):
try:
log, created = SyncLog.objects.get_or_create(hotel=hotel)
if created:
log.recieved_records = recieved_records
log.processed_records = processed_records
else:
log.recieved_records += recieved_records
log.processed_records += processed_records
log.save()
self.logger.info(f"Sync log updated for hotel '{hotel.name}'.")
except Exception as e:
self.logger.error(f"Error updating sync log for hotel '{hotel.name}': {e}")
def process_and_save_data(self, rows):
"""
Обрабатывает и сохраняет данные из внешнего источника.
:param rows: Список строк данных, полученных из базы данных.
"""
seen_entries = set()
for row in rows:
# Получение и декодирование URL-параметров
url_parameters = row.get("url_parameters")
if not url_parameters:
self.logger.warning(f"Skipping record with missing URL parameters: {row}")
continue
parsed_params = self.data_processor.parse_url_parameters(url_parameters)
hotel_id = parsed_params.get("utm_content") # Извлекаем hotel_id из параметров
room_number = parsed_params.get("utm_term") # Извлекаем room_number из параметров
if not hotel_id or not room_number:
self.logger.warning(f"Skipping record with missing data: hotel_id={hotel_id}, room_number={room_number}")
continue
# Проверка на дубликаты
if (hotel_id, room_number) in seen_entries:
self.logger.warning(f"Duplicate record skipped: hotel_id={hotel_id}, room_number={room_number}")
continue
seen_entries.add((hotel_id, room_number))
try:
url_params = self.data_processor.decode_html_entities(row.get("url_parameters", ""))
params = self.data_processor.url_parameters_parser(url_params)
# Получение или создание отеля
hotel = self.hotel_manager.get_or_create_hotel(hotel_id, row.get("page_title"))
if not hotel:
self.logger.warning(f"Skipping record: Failed to create or retrieve hotel with ID {hotel_id}")
continue
hotel_id = params.get("utm_content")
room_number = params.get("utm_term")
page_title = row.get("page_title")
external_id = row.get("id")
hits = row.get("hits") or 0
hotel = self.hotel_manager.get_or_create_hotel(hotel_id, page_title)
# Получение или создание комнаты
room = self.hotel_manager.get_or_create_room(hotel, room_number)
page_url = row.get("page_url")
if not room:
self.logger.warning(f"Skipping record: Failed to create or retrieve room {room_number} in hotel {hotel.name}")
continue
if hits != 0 and page_title is not None:
UserActivityLog.objects.update_or_create(
external_id=external_id,
defaults={
"user_id": row.get("user_id") or 0,
"timestamp": row.get("timestamp"),
"date_time": row.get("date_time"),
"ip": row.get("ip") or "0.0.0.0",
"created": self.data_processor.parse_datetime(row.get("created")) or timezone.now(),
"url_parameters": url_params,
"page_id": room.id if room else None,
"page_title": html.unescape(page_title),
"hits": hits,
"page_url": html.unescape(page_url),
}
)
else:
self.logger.warning("Invalid data for UserActivityLog.")
self.logger.info(f"Record ID {external_id} processed successfully.")
# Создание или обновление записи активности пользователя
UserActivityLog.objects.update_or_create(
external_id=row.get("id"),
defaults={
"user_id": row.get("user_id") or 0,
"ip": row.get("ip") or "0.0.0.0",
"created": self.data_processor.parse_datetime(row.get("created")),
"timestamp": row.get("timestamp") or datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"date_time": self.data_processor.parse_datetime(row.get("date_time")),
"url_parameters": parsed_params,
"page_title": self.data_processor.decode_html_entities(row.get("page_title")) or "Untitled",
"page_url": row.get("page_url") or "",
"hits": row.get("hits") or 0,
}
)
self.logger.info(f"Record ID {row.get('id')} processed successfully.")
except Exception as e:
self.logger.error(f"Error processing record ID {row.get('id')}: {e}")
self.logger.info(f"Data processing completed. Processed {len(seen_entries)} unique records.")
def sync(self):
self.db_connector.connect()
try:
@@ -230,7 +292,7 @@ class DataSyncManager:
def scheduled_sync():
logger = CustomLogger(name="DatabaseSyncScheduler", log_level="DEBUG").get_logger()
logger = CustomLogger(name="DatabaseSyncScheduler", log_level="ERROR").get_logger()
logger.info("Starting scheduled sync.")
active_db_settings = ExternalDBSettings.objects.filter(is_active=True)
@@ -256,5 +318,5 @@ def scheduled_sync():
future.result(timeout=300)
except TimeoutError:
logger.error("Sync task timed out.")
logger.info("Scheduled sync completed.")

44
antifroud/migrations/0018_remove_synclog_reservation_alter_synclog_created_and_more.py Normal file
View File

@@ -0,0 +1,44 @@
# Generated by Django 5.1.4 on 2024-12-18 04:46
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('antifroud', '0017_violationlog_hits'),
('hotels', '0014_alter_room_number'),
]
operations = [
migrations.RemoveField(
model_name='synclog',
name='reservation',
),
migrations.AlterField(
model_name='synclog',
name='created',
field=models.DateTimeField(auto_now=True, verbose_name='Дата обновления'),
),
migrations.AlterField(
model_name='synclog',
name='hotel',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='hotels.hotel', unique=True, verbose_name='Отель'),
),
migrations.AlterField(
model_name='synclog',
name='id',
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
),
migrations.AlterField(
model_name='synclog',
name='processed_records',
field=models.IntegerField(default=0, verbose_name='Обработанные записи'),
),
migrations.AlterField(
model_name='synclog',
name='recieved_records',
field=models.IntegerField(default=0, verbose_name='Полученные записи'),
),
]

20
antifroud/migrations/0019_alter_synclog_hotel.py Normal file
View File

@@ -0,0 +1,20 @@
# Generated by Django 5.1.4 on 2024-12-18 04:55
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('antifroud', '0018_remove_synclog_reservation_alter_synclog_created_and_more'),
('hotels', '0014_alter_room_number'),
]
operations = [
migrations.AlterField(
model_name='synclog',
name='hotel',
field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='hotels.hotel', verbose_name='Отель'),
),
]

18
antifroud/migrations/0020_alter_useractivitylog_hits.py Normal file
View File

@@ -0,0 +1,18 @@
# Generated by Django 5.1.4 on 2024-12-18 10:45
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('antifroud', '0019_alter_synclog_hotel'),
]
operations = [
migrations.AlterField(
model_name='useractivitylog',
name='hits',
field=models.IntegerField(blank=True, default='0', null=True, verbose_name='Количество обращений'),
),
]

44
antifroud/models.py
View File

@@ -1,7 +1,7 @@
from django.db import models
from hotels.models import Hotel
from hotels.models import Reservation
from datetime import datetime, timezone
class UserActivityLog(models.Model):
external_id = models.CharField(max_length=255, unique=True, verbose_name="Внешний ID", db_index=True)
@@ -23,11 +23,22 @@ class UserActivityLog(models.Model):
page_title = models.TextField(blank=True, null=True, verbose_name="Заголовок страницы")
type = models.CharField(max_length=50, verbose_name="Тип", blank=True, null=True)
last_counter = models.IntegerField(verbose_name="Последний счетчик", blank=True, null=True)
hits = models.IntegerField(verbose_name="Количество обращений", blank=True, null=True)
hits = models.IntegerField(verbose_name="Количество обращений",default="0", blank=True, null=True)
honeypot = models.BooleanField(verbose_name="Метка honeypot", blank=True, null=True)
reply = models.BooleanField(verbose_name="Ответ пользователя", blank=True, null=True)
page_url = models.URLField(blank=True, null=True, verbose_name="URL страницы")
@property
def formatted_timestamp(self):
"""
Преобразует Unix-временную метку в читаемую дату и время.
"""
if self.timestamp is not None:
return datetime.fromtimestamp(self.timestamp, tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
return "Нет данных"
# Изменение имени столбца
class Meta:
indexes = [
models.Index(fields=["external_id"], name="idx_external_id"),
@@ -40,12 +51,21 @@ class UserActivityLog(models.Model):
verbose_name_plural = "Логи активности пользователей"
def __str__(self):
return f"UserActivityLog {self.id}: {self.page_title}"
class Meta:
verbose_name = "Регистрация посетителей"
verbose_name_plural = "Регистрации посетителей"
def get_location(self):
if not self.ip:
return "IP-адрес отсутствует"
try:
geoip_reader = Reader(settings.GEOIP_PATH + '/GeoLite2-City.mmdb')
response = geoip_reader.city(self.ip)
return f"{response.city.name}, {response.country.name}"
except Exception as e:
return "Местоположение недоступно"
class ExternalDBSettings(models.Model):
name = models.CharField(max_length=255, unique=True, help_text="Имя подключения для идентификации.")
host = models.CharField(max_length=255, help_text="Адрес сервера базы данных.")
@@ -165,18 +185,20 @@ class ImportedHotel(models.Model):
class SyncLog(models.Model):
"""
Журнал синхронизации.
Журнал синхронизации в разрезе отелей.
"""
id = models.BigIntegerField(primary_key=True, unique=True, verbose_name="ID")
hotel = models.ForeignKey(Hotel, on_delete=models.CASCADE, verbose_name="Отель")
reservation = models.ForeignKey(Reservation, on_delete=models.CASCADE, verbose_name="Бронирование")
created = models.DateTimeField(auto_now_add=True, verbose_name="Дата создания")
recieved_records = models.IntegerField(verbose_name="Полученные записи")
processed_records = models.IntegerField(verbose_name="Обработанные записи")
hotel = models.OneToOneField(Hotel, on_delete=models.CASCADE, verbose_name="Отель")
created = models.DateTimeField(auto_now=True, verbose_name="Дата обновления") # Последняя дата обновления записи
recieved_records = models.IntegerField(default=0, verbose_name="Полученные записи")
processed_records = models.IntegerField(default=0, verbose_name="Обработанные записи")
class Meta:
verbose_name = "Журнал синхронизации"
verbose_name_plural = "Журналы синхронизации"
def __str__(self):
return f"Отель: {self.hotel.name} | Получено: {self.recieved_records} | Обработано: {self.processed_records}"
class ViolationLog(models.Model):