SmartSolTech/Touchh
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

RealtyCalendar plugin

Browse Source
This commit is contained in:
trevor
2024-12-24 15:43:43 +09:00
parent 77bf0b8381
commit d5622f41a2
58 changed files with 374 additions and 1714 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
antifroud/data_sync.py
View File

@@ -202,63 +202,56 @@ class DataSyncManager:
self.logger.error(f"Error fetching data: {e}")
return []
def update_sync_log(self, hotel, recieved_records, processed_records):
"""
Обновляет или создает запись в таблице SyncLog.
"""
try:
log, created = SyncLog.objects.get_or_create(hotel=hotel)
log, created = SyncLog.objects.update_or_create(
hotel=hotel,
defaults={
"recieved_records": recieved_records,
"processed_records": processed_records,
"created": timezone.now(), # Убедитесь, что дата обновляется
}
)
if created:
log.recieved_records = recieved_records
log.processed_records = processed_records
self.logger.info(f"Sync log created for hotel '{hotel.name}'.")
else:
log.recieved_records += recieved_records
log.processed_records += processed_records
log.save()
self.logger.info(f"Sync log updated for hotel '{hotel.name}'.")
self.logger.info(f"Sync log updated for hotel '{hotel.name}'.")
except Exception as e:
self.logger.error(f"Error updating sync log for hotel '{hotel.name}': {e}")
self.logger.info(f"Attempting to update sync log for hotel: {hotel.name}")
self.update_sync_log(hotel, recieved_records, processed_records)
def process_and_save_data(self, rows):
"""
Обрабатывает и сохраняет данные из внешнего источника.
:param rows: Список строк данных, полученных из базы данных.
"""
seen_entries = set()
hotel_processed_counts = {} # Словарь для подсчёта записей по каждому отелю
for row in rows:
# Получение и декодирование URL-параметров
url_parameters = row.get("url_parameters")
if not url_parameters:
self.logger.warning(f"Skipping record with missing URL parameters: {row}")
continue
parsed_params = self.data_processor.parse_url_parameters(url_parameters)
hotel_id = parsed_params.get("utm_content") # Извлекаем hotel_id из параметров
room_number = parsed_params.get("utm_term") # Извлекаем room_number из параметров
if not hotel_id or not room_number:
self.logger.warning(f"Skipping record with missing data: hotel_id={hotel_id}, room_number={room_number}")
continue
# Проверка на дубликаты
if (hotel_id, room_number) in seen_entries:
self.logger.warning(f"Duplicate record skipped: hotel_id={hotel_id}, room_number={room_number}")
continue
seen_entries.add((hotel_id, room_number))
try:
# Получение или создание отеля
url_parameters = row.get("url_parameters")
if not url_parameters:
self.logger.warning(f"Skipping record with missing URL parameters: {row}")
continue
parsed_params = self.data_processor.parse_url_parameters(url_parameters)
hotel_id = parsed_params.get("utm_content")
room_number = parsed_params.get("utm_term")
if not hotel_id or not room_number:
self.logger.warning(f"Skipping record with missing data: hotel_id={hotel_id}, room_number={room_number}")
continue
hotel = self.hotel_manager.get_or_create_hotel(hotel_id, row.get("page_title"))
if not hotel:
self.logger.warning(f"Skipping record: Failed to create or retrieve hotel with ID {hotel_id}")
continue
# Получение или создание комнаты
room = self.hotel_manager.get_or_create_room(hotel, room_number)
if not room:
self.logger.warning(f"Skipping record: Failed to create or retrieve room {room_number} in hotel {hotel.name}")
continue
# Создание или обновление записи активности пользователя
UserActivityLog.objects.update_or_create(
external_id=row.get("id"),
defaults={
@@ -270,14 +263,23 @@ class DataSyncManager:
"url_parameters": parsed_params,
"page_title": self.data_processor.decode_html_entities(row.get("page_title")) or "Untitled",
"page_url": row.get("page_url") or "",
"page_id": row.get("page_id") or 0,
"hits": row.get("hits") or 0,
}
)
self.logger.info(f"Record ID {row.get('id')} processed successfully.")
if hotel.id not in hotel_processed_counts:
hotel_processed_counts[hotel.id] = {"recieved_records": 0, "processed_records": 0}
hotel_processed_counts[hotel.id]["processed_records"] += 1
except Exception as e:
self.logger.error(f"Error processing record ID {row.get('id')}: {e}")
self.logger.info(f"Data processing completed. Processed {len(seen_entries)} unique records.")
for hotel_id, counts in hotel_processed_counts.items():
hotel = Hotel.objects.get(id=hotel_id)
self.update_sync_log(hotel, recieved_records=len(rows), processed_records=counts["processed_records"])
def sync(self):
@@ -292,7 +294,8 @@ class DataSyncManager:
def scheduled_sync():
logger = CustomLogger(name="DatabaseSyncScheduler", log_level="ERROR").get_logger()
import os
logger = CustomLogger(name="DatabaseSyncScheduler", log_level=os.getenv("SCHEDULED_SYNC_LOG_LEVEL", default="ERROR")).get_logger()
logger.info("Starting scheduled sync.")
active_db_settings = ExternalDBSettings.objects.filter(is_active=True)