From fdf74c11ecbb53643066adf75bd25bc048fb6ced Mon Sep 17 00:00:00 2001 From: UserIsMH Date: Sat, 21 Jun 2025 19:47:49 +0200 Subject: [PATCH] Monitoring Anpassung --- v2_adminpanel/routes/admin_routes.py | 65 +- v2_adminpanel/routes/monitoring_routes.py | 640 ++++++++++-------- v2_adminpanel/templates/base.html | 12 +- .../monitoring/unified_monitoring.html | 609 +++++++++++++++++ 4 files changed, 962 insertions(+), 364 deletions(-) create mode 100644 v2_adminpanel/templates/monitoring/unified_monitoring.html diff --git a/v2_adminpanel/routes/admin_routes.py b/v2_adminpanel/routes/admin_routes.py index 3d23e55..7521c6b 100644 --- a/v2_adminpanel/routes/admin_routes.py +++ b/v2_adminpanel/routes/admin_routes.py @@ -881,69 +881,8 @@ def license_analytics(): @admin_bp.route("/lizenzserver/anomalies") @login_required def license_anomalies(): - """Anomaly detection and management""" - try: - conn = get_connection() - cur = conn.cursor() - - # Filter parameters - severity = request.args.get('severity', 'all') - resolved = request.args.get('resolved', 'false') - - # Build query - query = """ - SELECT ad.*, l.license_key, c.name as customer_name, c.email - FROM anomaly_detections ad - LEFT JOIN licenses l ON ad.license_id = l.id - LEFT JOIN customers c ON l.customer_id = c.id - WHERE 1=1 - """ - params = [] - - if severity != 'all': - query += " AND ad.severity = %s" - params.append(severity) - - if resolved == 'false': - query += " AND ad.resolved = false" - elif resolved == 'true': - query += " AND ad.resolved = true" - - query += " ORDER BY ad.detected_at DESC LIMIT 100" - - cur.execute(query, params) - anomalies = cur.fetchall() - - # Get anomaly statistics - cur.execute(""" - SELECT anomaly_type, severity, COUNT(*) as count - FROM anomaly_detections - WHERE resolved = false - GROUP BY anomaly_type, severity - ORDER BY count DESC - """) - anomaly_stats = cur.fetchall() - - return render_template('license_anomalies.html', - anomalies=anomalies, - anomaly_stats=anomaly_stats, - severity=severity, - resolved=resolved - ) - - except Exception as e: - flash(f'Fehler beim Laden der Anomalie-Daten: {str(e)}', 'error') - return render_template('license_anomalies.html', - anomalies=[], - anomaly_stats=[], - severity='all', - resolved='false' - ) - finally: - if 'cur' in locals(): - cur.close() - if 'conn' in locals(): - conn.close() + """Redirect to unified monitoring page""" + return redirect(url_for('monitoring.unified_monitoring')) @admin_bp.route("/lizenzserver/anomaly//resolve", methods=["POST"]) diff --git a/v2_adminpanel/routes/monitoring_routes.py b/v2_adminpanel/routes/monitoring_routes.py index bc3b115..36ee35d 100644 --- a/v2_adminpanel/routes/monitoring_routes.py +++ b/v2_adminpanel/routes/monitoring_routes.py @@ -1,4 +1,4 @@ -from flask import Blueprint, render_template, jsonify, request, session +from flask import Blueprint, render_template, jsonify, request, session, redirect, url_for from functools import wraps import psycopg2 from psycopg2.extras import RealDictCursor @@ -8,10 +8,10 @@ from datetime import datetime, timedelta import logging from utils.partition_helper import ensure_partition_exists, check_table_exists -monitoring_bp = Blueprint('monitoring', __name__) +# Configure logging logger = logging.getLogger(__name__) -# Database connection +# Create a function to get database connection def get_db_connection(): return psycopg2.connect( host=os.environ.get('POSTGRES_HOST', 'postgres'), @@ -20,168 +20,277 @@ def get_db_connection(): password=os.environ.get('POSTGRES_PASSWORD', 'postgres') ) -# Login required decorator def login_required(f): @wraps(f) def decorated_function(*args, **kwargs): - if 'logged_in' not in session: - return render_template('error.html', - error_message='Nicht autorisiert', - details='Sie müssen angemeldet sein, um diese Seite zu sehen.') + if 'user_id' not in session: + return redirect(url_for('auth.login')) return f(*args, **kwargs) return decorated_function -@monitoring_bp.route('/live-dashboard') +# Create Blueprint +monitoring_bp = Blueprint('monitoring', __name__) + +@monitoring_bp.route('/monitoring') @login_required -def live_dashboard(): - """Live Dashboard showing active customer sessions and analytics""" +def unified_monitoring(): + """Unified monitoring dashboard combining live activity and anomaly detection""" try: conn = get_db_connection() cur = conn.cursor(cursor_factory=RealDictCursor) - # Check if license_heartbeats table exists - if not check_table_exists(conn, 'license_heartbeats'): - logger.warning("license_heartbeats table does not exist") - # Return empty data - return render_template('monitoring/live_dashboard.html', - active_sessions=[], - stats={'active_licenses': 0, 'active_devices': 0, 'total_heartbeats': 0}, - validation_timeline=[], - live_stats=[0, 0, 0, 0], - validation_rates=[], - recent_anomalies=[], - top_licenses=[]) + # Initialize default values + system_status = 'normal' + status_color = 'success' + active_alerts = 0 + live_metrics = { + 'active_licenses': 0, + 'total_validations': 0, + 'unique_devices': 0, + 'unique_ips': 0, + 'avg_response_time': 0 + } + trend_data = [] + activity_stream = [] + geo_data = [] + top_licenses = [] + anomaly_distribution = [] + performance_data = [] - # Ensure current month partition exists - ensure_partition_exists(conn, 'license_heartbeats', datetime.now()) + # Check if tables exist before querying + has_heartbeats = check_table_exists(conn, 'license_heartbeats') + has_anomalies = check_table_exists(conn, 'anomaly_detections') - # Get active customer sessions (last 5 minutes) - cur.execute(""" - SELECT - l.id, - l.license_key, - c.name as company_name, - lh.hardware_id, - lh.ip_address, - lh.timestamp as last_activity, - lh.session_data, - COUNT(DISTINCT lh.hardware_id) OVER (PARTITION BY l.id) as active_devices - FROM license_heartbeats lh - JOIN licenses l ON l.id = lh.license_id - JOIN customers c ON c.id = l.customer_id - WHERE lh.timestamp > NOW() - INTERVAL '5 minutes' - AND l.is_active = true - ORDER BY lh.timestamp DESC - LIMIT 100 - """) - active_sessions = cur.fetchall() + if has_anomalies: + # Get active alerts count + cur.execute(""" + SELECT COUNT(*) as count + FROM anomaly_detections + WHERE resolved = false + AND detected_at > NOW() - INTERVAL '24 hours' + """) + active_alerts = cur.fetchone()['count'] or 0 + + # Determine system status based on alerts + if active_alerts == 0: + system_status = 'normal' + status_color = 'success' + elif active_alerts < 5: + system_status = 'warning' + status_color = 'warning' + else: + system_status = 'critical' + status_color = 'danger' - # Get session statistics - cur.execute(""" - SELECT - COUNT(DISTINCT license_id) as active_licenses, - COUNT(DISTINCT hardware_id) as active_devices, - COUNT(*) as total_heartbeats - FROM license_heartbeats - WHERE timestamp > NOW() - INTERVAL '5 minutes' - """) - stats = cur.fetchone() + if has_heartbeats: + # Ensure current month partition exists + ensure_partition_exists(conn, 'license_heartbeats', datetime.now()) + + # Executive summary metrics + cur.execute(""" + SELECT + COUNT(DISTINCT license_id) as active_licenses, + COUNT(*) as total_validations, + COUNT(DISTINCT hardware_id) as unique_devices, + COUNT(DISTINCT ip_address) as unique_ips, + AVG(CASE WHEN response_time IS NOT NULL THEN response_time ELSE 0 END) as avg_response_time + FROM license_heartbeats + WHERE timestamp > NOW() - INTERVAL '5 minutes' + """) + result = cur.fetchone() + if result: + live_metrics = result + + # Get 24h trend data for metrics + cur.execute(""" + SELECT + DATE_TRUNC('hour', timestamp) as hour, + COUNT(DISTINCT license_id) as licenses, + COUNT(*) as validations + FROM license_heartbeats + WHERE timestamp > NOW() - INTERVAL '24 hours' + GROUP BY hour + ORDER BY hour + """) + trend_data = cur.fetchall() + + # Activity stream - just validations if no anomalies table + if has_anomalies: + cur.execute(""" + WITH combined_events AS ( + -- Normal validations + SELECT + lh.timestamp, + 'validation' as event_type, + 'normal' as severity, + l.license_key, + c.name as customer_name, + lh.ip_address, + lh.hardware_id, + NULL as anomaly_type, + NULL as description + FROM license_heartbeats lh + JOIN licenses l ON l.id = lh.license_id + JOIN customers c ON c.id = l.customer_id + WHERE lh.timestamp > NOW() - INTERVAL '1 hour' + + UNION ALL + + -- Anomalies + SELECT + ad.detected_at as timestamp, + 'anomaly' as event_type, + ad.severity, + l.license_key, + c.name as customer_name, + ad.ip_address, + ad.hardware_id, + ad.anomaly_type, + ad.description + FROM anomaly_detections ad + LEFT JOIN licenses l ON l.id = ad.license_id + LEFT JOIN customers c ON c.id = l.customer_id + WHERE ad.detected_at > NOW() - INTERVAL '1 hour' + ) + SELECT * FROM combined_events + ORDER BY timestamp DESC + LIMIT 100 + """) + else: + # Just show validations + cur.execute(""" + SELECT + lh.timestamp, + 'validation' as event_type, + 'normal' as severity, + l.license_key, + c.name as customer_name, + lh.ip_address, + lh.hardware_id, + NULL as anomaly_type, + NULL as description + FROM license_heartbeats lh + JOIN licenses l ON l.id = lh.license_id + JOIN customers c ON c.id = l.customer_id + WHERE lh.timestamp > NOW() - INTERVAL '1 hour' + ORDER BY lh.timestamp DESC + LIMIT 100 + """) + activity_stream = cur.fetchall() + + # Geographic distribution + cur.execute(""" + SELECT + ip_address, + COUNT(*) as request_count, + COUNT(DISTINCT license_id) as license_count + FROM license_heartbeats + WHERE timestamp > NOW() - INTERVAL '1 hour' + GROUP BY ip_address + ORDER BY request_count DESC + LIMIT 20 + """) + geo_data = cur.fetchall() + + # Top active licenses + if has_anomalies: + cur.execute(""" + SELECT + l.id, + l.license_key, + c.name as customer_name, + COUNT(DISTINCT lh.hardware_id) as device_count, + COUNT(lh.*) as validation_count, + MAX(lh.timestamp) as last_seen, + COUNT(DISTINCT ad.id) as anomaly_count + FROM licenses l + JOIN customers c ON c.id = l.customer_id + LEFT JOIN license_heartbeats lh ON l.id = lh.license_id + AND lh.timestamp > NOW() - INTERVAL '1 hour' + LEFT JOIN anomaly_detections ad ON l.id = ad.license_id + AND ad.detected_at > NOW() - INTERVAL '24 hours' + WHERE lh.license_id IS NOT NULL + GROUP BY l.id, l.license_key, c.name + ORDER BY validation_count DESC + LIMIT 10 + """) + else: + cur.execute(""" + SELECT + l.id, + l.license_key, + c.name as customer_name, + COUNT(DISTINCT lh.hardware_id) as device_count, + COUNT(lh.*) as validation_count, + MAX(lh.timestamp) as last_seen, + 0 as anomaly_count + FROM licenses l + JOIN customers c ON c.id = l.customer_id + LEFT JOIN license_heartbeats lh ON l.id = lh.license_id + AND lh.timestamp > NOW() - INTERVAL '1 hour' + WHERE lh.license_id IS NOT NULL + GROUP BY l.id, l.license_key, c.name + ORDER BY validation_count DESC + LIMIT 10 + """) + top_licenses = cur.fetchall() + + # Performance metrics + cur.execute(""" + SELECT + DATE_TRUNC('minute', timestamp) as minute, + AVG(response_time) as avg_response_time, + MAX(response_time) as max_response_time, + COUNT(*) as request_count + FROM license_heartbeats + WHERE timestamp > NOW() - INTERVAL '30 minutes' + AND response_time IS NOT NULL + GROUP BY minute + ORDER BY minute DESC + """) + performance_data = cur.fetchall() - # Get validations per minute (for both charts) - cur.execute(""" - SELECT - DATE_TRUNC('minute', timestamp) as minute, - COUNT(*) as validations - FROM license_heartbeats - WHERE timestamp > NOW() - INTERVAL '60 minutes' - GROUP BY minute - ORDER BY minute DESC - LIMIT 60 - """) - validation_timeline = cur.fetchall() - - # Get live statistics for analytics cards - cur.execute(""" - SELECT - COUNT(DISTINCT license_id) as active_licenses, - COUNT(*) as total_validations, - COUNT(DISTINCT hardware_id) as unique_devices, - COUNT(DISTINCT ip_address) as unique_ips - FROM license_heartbeats - WHERE timestamp > NOW() - INTERVAL '5 minutes' - """) - live_stats_data = cur.fetchone() - live_stats = [ - live_stats_data['active_licenses'] or 0, - live_stats_data['total_validations'] or 0, - live_stats_data['unique_devices'] or 0, - live_stats_data['unique_ips'] or 0 - ] - - # Get validation rates for analytics chart (last 30 minutes) - cur.execute(""" - SELECT - DATE_TRUNC('minute', timestamp) as minute, - COUNT(*) as count - FROM license_heartbeats - WHERE timestamp > NOW() - INTERVAL '30 minutes' - GROUP BY minute - ORDER BY minute DESC - LIMIT 30 - """) - validation_rates = [(row['minute'].isoformat(), row['count']) for row in cur.fetchall()] - - # Get recent anomalies - cur.execute(""" - SELECT - ad.*, - l.license_key, - c.name as customer_name - FROM anomaly_detections ad - LEFT JOIN licenses l ON l.id = ad.license_id - LEFT JOIN customers c ON c.id = l.customer_id - WHERE ad.detected_at > NOW() - INTERVAL '24 hours' - ORDER BY ad.detected_at DESC - LIMIT 10 - """) - recent_anomalies = cur.fetchall() - - # Get top active licenses - cur.execute(""" - SELECT - l.license_key, - c.name as customer_name, - COUNT(DISTINCT lh.hardware_id) as device_count, - COUNT(*) as validation_count, - MAX(lh.timestamp) as last_seen - FROM license_heartbeats lh - JOIN licenses l ON l.id = lh.license_id - JOIN customers c ON c.id = l.customer_id - WHERE lh.timestamp > NOW() - INTERVAL '15 minutes' - GROUP BY l.license_key, c.name - ORDER BY validation_count DESC - LIMIT 10 - """) - top_licenses = cur.fetchall() + if has_anomalies: + # Anomaly distribution + cur.execute(""" + SELECT + anomaly_type, + COUNT(*) as count, + MAX(severity) as max_severity + FROM anomaly_detections + WHERE detected_at > NOW() - INTERVAL '24 hours' + GROUP BY anomaly_type + ORDER BY count DESC + """) + anomaly_distribution = cur.fetchall() cur.close() conn.close() - return render_template('monitoring/live_dashboard.html', - active_sessions=active_sessions, - stats=stats, - validation_timeline=validation_timeline, - live_stats=live_stats, - validation_rates=validation_rates, - recent_anomalies=recent_anomalies, - top_licenses=top_licenses) - + return render_template('monitoring/unified_monitoring.html', + system_status=system_status, + status_color=status_color, + active_alerts=active_alerts, + live_metrics=live_metrics, + trend_data=trend_data, + activity_stream=activity_stream, + geo_data=geo_data, + top_licenses=top_licenses, + anomaly_distribution=anomaly_distribution, + performance_data=performance_data) + except Exception as e: - logger.error(f"Error in live dashboard: {str(e)}") + logger.error(f"Error in unified monitoring: {str(e)}") return render_template('error.html', - error_message='Fehler beim Laden des Dashboards', + error_message='Fehler beim Laden des Monitorings', details=str(e)) +@monitoring_bp.route('/live-dashboard') +@login_required +def live_dashboard(): + """Redirect to unified monitoring dashboard""" + return redirect(url_for('monitoring.unified_monitoring')) + @monitoring_bp.route('/alerts') @login_required @@ -195,25 +304,26 @@ def alerts(): if response.status_code == 200: alerts = response.json() except: - # Fallback to database anomalies + # Fallback to database anomalies if table exists conn = get_db_connection() - cur = conn.cursor(cursor_factory=RealDictCursor) - - cur.execute(""" - SELECT - ad.*, - l.license_key, - c.name as company_name - FROM anomaly_detections ad - LEFT JOIN licenses l ON l.id = ad.license_id - LEFT JOIN customers c ON c.id = l.customer_id - WHERE ad.resolved = false - ORDER BY ad.detected_at DESC - LIMIT 50 - """) - alerts = cur.fetchall() - - cur.close() + if check_table_exists(conn, 'anomaly_detections'): + cur = conn.cursor(cursor_factory=RealDictCursor) + + cur.execute(""" + SELECT + ad.*, + l.license_key, + c.name as company_name + FROM anomaly_detections ad + LEFT JOIN licenses l ON l.id = ad.license_id + LEFT JOIN customers c ON c.id = l.customer_id + WHERE ad.resolved = false + ORDER BY ad.detected_at DESC + LIMIT 50 + """) + alerts = cur.fetchall() + + cur.close() conn.close() return render_template('monitoring/alerts.html', alerts=alerts) @@ -226,146 +336,94 @@ def analytics(): conn = get_db_connection() cur = conn.cursor(cursor_factory=RealDictCursor) - # Get live statistics for the top cards - cur.execute(""" - SELECT - COUNT(DISTINCT license_id) as active_licenses, - COUNT(*) as total_validations, - COUNT(DISTINCT hardware_id) as unique_devices, - COUNT(DISTINCT ip_address) as unique_ips - FROM license_heartbeats - WHERE timestamp > NOW() - INTERVAL '5 minutes' - """) - live_stats = cur.fetchone() - live_stats = [ - live_stats['active_licenses'] or 0, - live_stats['total_validations'] or 0, - live_stats['unique_devices'] or 0, - live_stats['unique_ips'] or 0 - ] + # Initialize default values + live_stats = [0, 0, 0, 0] + validation_rates = [] - # Get validation rates for chart - cur.execute(""" - SELECT - DATE_TRUNC('minute', timestamp) as minute, - COUNT(*) as count - FROM license_heartbeats - WHERE timestamp > NOW() - INTERVAL '30 minutes' - GROUP BY minute - ORDER BY minute DESC - LIMIT 30 - """) - validation_rates = [(row['minute'].isoformat(), row['count']) for row in cur.fetchall()] - - # Get recent anomalies - cur.execute(""" - SELECT - ad.*, - l.license_key, - c.name as customer_name - FROM anomaly_detections ad - LEFT JOIN licenses l ON l.id = ad.license_id - LEFT JOIN customers c ON c.id = l.customer_id - WHERE ad.detected_at > NOW() - INTERVAL '24 hours' - ORDER BY ad.detected_at DESC - LIMIT 10 - """) - recent_anomalies = cur.fetchall() - - # Get top active licenses - cur.execute(""" - SELECT - l.license_key, - c.name as customer_name, - COUNT(DISTINCT lh.hardware_id) as device_count, - COUNT(*) as validation_count, - MAX(lh.timestamp) as last_seen - FROM license_heartbeats lh - JOIN licenses l ON l.id = lh.license_id - JOIN customers c ON c.id = l.customer_id - WHERE lh.timestamp > NOW() - INTERVAL '15 minutes' - GROUP BY l.license_key, c.name - ORDER BY validation_count DESC - LIMIT 10 - """) - top_licenses = cur.fetchall() + if check_table_exists(conn, 'license_heartbeats'): + # Get live statistics for the top cards + cur.execute(""" + SELECT + COUNT(DISTINCT license_id) as active_licenses, + COUNT(*) as total_validations, + COUNT(DISTINCT hardware_id) as unique_devices, + COUNT(DISTINCT ip_address) as unique_ips + FROM license_heartbeats + WHERE timestamp > NOW() - INTERVAL '5 minutes' + """) + live_stats_data = cur.fetchone() + live_stats = [ + live_stats_data['active_licenses'] or 0, + live_stats_data['total_validations'] or 0, + live_stats_data['unique_devices'] or 0, + live_stats_data['unique_ips'] or 0 + ] + + # Get validation rates for the chart (last 30 minutes, aggregated by minute) + cur.execute(""" + SELECT + DATE_TRUNC('minute', timestamp) as minute, + COUNT(*) as count + FROM license_heartbeats + WHERE timestamp > NOW() - INTERVAL '30 minutes' + GROUP BY minute + ORDER BY minute DESC + LIMIT 30 + """) + validation_rates = [(row['minute'].isoformat(), row['count']) for row in cur.fetchall()] cur.close() conn.close() - return render_template('monitoring/analytics.html', + return render_template('monitoring/analytics.html', live_stats=live_stats, - validation_rates=validation_rates, - recent_anomalies=recent_anomalies, - top_licenses=top_licenses) + validation_rates=validation_rates) except Exception as e: logger.error(f"Error in analytics: {str(e)}") - return render_template('monitoring/analytics.html', - live_stats=[0, 0, 0, 0], - validation_rates=[], - recent_anomalies=[], - top_licenses=[]) + return render_template('error.html', + error_message='Fehler beim Laden der Analytics', + details=str(e)) -# API endpoints for live data -@monitoring_bp.route('/api/live-stats') -@login_required -def api_live_stats(): - """API endpoint for live statistics""" - try: - conn = get_db_connection() - cur = conn.cursor(cursor_factory=RealDictCursor) - - # Get current stats - cur.execute(""" - SELECT - COUNT(DISTINCT license_id) as active_licenses, - COUNT(DISTINCT hardware_id) as active_devices, - COUNT(*) as validations_last_minute - FROM license_heartbeats - WHERE timestamp > NOW() - INTERVAL '1 minute' - """) - stats = cur.fetchone() - - cur.close() - conn.close() - - return jsonify(stats) - - except Exception as e: - return jsonify({'error': str(e)}), 500 -@monitoring_bp.route('/api/active-sessions') +@monitoring_bp.route('/analytics/stream') @login_required -def api_active_sessions(): - """API endpoint for active customer sessions""" - try: - conn = get_db_connection() - cur = conn.cursor(cursor_factory=RealDictCursor) - - # Get active sessions with geo data - cur.execute(""" - SELECT - l.license_key, - c.name as company_name, - lh.hardware_id, - lh.ip_address, - lh.timestamp as last_activity, - EXTRACT(EPOCH FROM (NOW() - lh.timestamp)) as seconds_ago, - lh.session_data - FROM license_heartbeats lh - JOIN licenses l ON l.id = lh.license_id - JOIN customers c ON c.id = l.customer_id - WHERE lh.timestamp > NOW() - INTERVAL '5 minutes' - ORDER BY lh.timestamp DESC - LIMIT 50 - """) - sessions = cur.fetchall() - - cur.close() - conn.close() - - return jsonify(sessions) - - except Exception as e: - return jsonify({'error': str(e)}), 500 \ No newline at end of file +def analytics_stream(): + """Server-sent event stream for live analytics updates""" + def generate(): + while True: + try: + conn = get_db_connection() + cur = conn.cursor(cursor_factory=RealDictCursor) + + data = {'active_licenses': 0, 'total_validations': 0, + 'unique_devices': 0, 'unique_ips': 0} + + if check_table_exists(conn, 'license_heartbeats'): + cur.execute(""" + SELECT + COUNT(DISTINCT license_id) as active_licenses, + COUNT(*) as total_validations, + COUNT(DISTINCT hardware_id) as unique_devices, + COUNT(DISTINCT ip_address) as unique_ips + FROM license_heartbeats + WHERE timestamp > NOW() - INTERVAL '5 minutes' + """) + result = cur.fetchone() + if result: + data = dict(result) + + cur.close() + conn.close() + + yield f"data: {jsonify(data).get_data(as_text=True)}\n\n" + + except Exception as e: + logger.error(f"Error in analytics stream: {str(e)}") + yield f"data: {jsonify({'error': str(e)}).get_data(as_text=True)}\n\n" + + import time + time.sleep(5) # Update every 5 seconds + + from flask import Response + return Response(generate(), mimetype="text/event-stream") \ No newline at end of file diff --git a/v2_adminpanel/templates/base.html b/v2_adminpanel/templates/base.html index 29b303c..be63a1b 100644 --- a/v2_adminpanel/templates/base.html +++ b/v2_adminpanel/templates/base.html @@ -415,19 +415,11 @@ -