443 Zeilen
19 KiB
Python
443 Zeilen
19 KiB
Python
"""
|
|
Comprehensive error handling and fallback mechanisms for method rotation system.
|
|
Provides robust error recovery and graceful degradation strategies.
|
|
"""
|
|
|
|
import logging
|
|
from datetime import datetime, timedelta
|
|
from typing import Dict, Any, List, Optional, Callable
|
|
from enum import Enum
|
|
|
|
from domain.entities.method_rotation import MethodStrategy, RotationSession, RiskLevel
|
|
from application.use_cases.method_rotation_use_case import MethodRotationUseCase
|
|
|
|
|
|
class ErrorSeverity(Enum):
|
|
"""Error severity levels for rotation decisions"""
|
|
LOW = "low" # Minor issues, continue with current method
|
|
MEDIUM = "medium" # Moderate issues, consider rotation
|
|
HIGH = "high" # Serious issues, rotate immediately
|
|
CRITICAL = "critical" # Critical failure, enable emergency mode
|
|
|
|
|
|
class RotationErrorHandler:
|
|
"""
|
|
Handles errors and provides fallback mechanisms for the rotation system.
|
|
Implements intelligent error classification and recovery strategies.
|
|
"""
|
|
|
|
def __init__(self, method_rotation_use_case: MethodRotationUseCase):
|
|
self.method_rotation_use_case = method_rotation_use_case
|
|
self.logger = logging.getLogger(self.__class__.__name__)
|
|
|
|
# Error classification patterns
|
|
self.error_patterns = self._init_error_patterns()
|
|
|
|
# Fallback strategies
|
|
self.fallback_strategies = self._init_fallback_strategies()
|
|
|
|
# Emergency mode settings
|
|
self.emergency_thresholds = {
|
|
'failure_rate_threshold': 0.8,
|
|
'consecutive_failures_threshold': 5,
|
|
'time_window_minutes': 30
|
|
}
|
|
|
|
def handle_rotation_error(self, platform: str, session_id: str,
|
|
error_details: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""
|
|
Handle rotation system errors with intelligent recovery.
|
|
|
|
Args:
|
|
platform: Platform name
|
|
session_id: Current rotation session ID
|
|
error_details: Error information
|
|
|
|
Returns:
|
|
Recovery action result
|
|
"""
|
|
try:
|
|
# Classify error severity
|
|
severity = self._classify_error_severity(error_details)
|
|
|
|
# Log error with classification
|
|
self.logger.warning(f"Rotation error on {platform}: {error_details.get('message', 'Unknown')} (Severity: {severity.value})")
|
|
|
|
# Choose recovery strategy based on severity
|
|
if severity == ErrorSeverity.CRITICAL:
|
|
return self._handle_critical_error(platform, session_id, error_details)
|
|
elif severity == ErrorSeverity.HIGH:
|
|
return self._handle_high_severity_error(platform, session_id, error_details)
|
|
elif severity == ErrorSeverity.MEDIUM:
|
|
return self._handle_medium_severity_error(platform, session_id, error_details)
|
|
else:
|
|
return self._handle_low_severity_error(platform, session_id, error_details)
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error in rotation error handler: {e}")
|
|
return self._fallback_to_original_behavior(platform, error_details)
|
|
|
|
def handle_system_failure(self, platform: str, failure_details: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""
|
|
Handle complete rotation system failures with graceful degradation.
|
|
|
|
Args:
|
|
platform: Platform name
|
|
failure_details: System failure information
|
|
|
|
Returns:
|
|
Fallback strategy result
|
|
"""
|
|
self.logger.error(f"Rotation system failure on {platform}: {failure_details}")
|
|
|
|
try:
|
|
# Attempt to gracefully shut down rotation for this platform
|
|
self._disable_rotation_for_platform(platform, "system_failure")
|
|
|
|
# Enable emergency mode with safest methods only
|
|
self.method_rotation_use_case.enable_emergency_mode(
|
|
platform, f"System failure: {failure_details.get('message', 'Unknown')}"
|
|
)
|
|
|
|
return {
|
|
'success': True,
|
|
'action': 'emergency_mode_enabled',
|
|
'fallback_method': 'email',
|
|
'message': 'System failure handled, emergency mode activated'
|
|
}
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to handle system failure: {e}")
|
|
return self._fallback_to_original_behavior(platform, failure_details)
|
|
|
|
def check_and_handle_emergency_conditions(self, platform: str) -> bool:
|
|
"""
|
|
Check if emergency conditions are met and handle accordingly.
|
|
|
|
Args:
|
|
platform: Platform to check
|
|
|
|
Returns:
|
|
True if emergency mode was triggered
|
|
"""
|
|
try:
|
|
# Get platform statistics
|
|
stats = self.method_rotation_use_case.strategy_repo.get_platform_statistics(platform)
|
|
|
|
# Check failure rate threshold
|
|
recent_failures = stats.get('recent_failures_24h', 0)
|
|
recent_successes = stats.get('recent_successes_24h', 0)
|
|
total_recent = recent_failures + recent_successes
|
|
|
|
if total_recent > 0:
|
|
failure_rate = recent_failures / total_recent
|
|
|
|
if failure_rate >= self.emergency_thresholds['failure_rate_threshold']:
|
|
self.logger.warning(f"High failure rate detected on {platform}: {failure_rate:.2f}")
|
|
self.method_rotation_use_case.enable_emergency_mode(
|
|
platform, f"High failure rate: {failure_rate:.2f}"
|
|
)
|
|
return True
|
|
|
|
# Check consecutive failures
|
|
session_stats = self.method_rotation_use_case.session_repo.get_session_statistics(platform, days=1)
|
|
failed_sessions = session_stats.get('failed_sessions', 0)
|
|
|
|
if failed_sessions >= self.emergency_thresholds['consecutive_failures_threshold']:
|
|
self.logger.warning(f"High consecutive failures on {platform}: {failed_sessions}")
|
|
self.method_rotation_use_case.enable_emergency_mode(
|
|
platform, f"Consecutive failures: {failed_sessions}"
|
|
)
|
|
return True
|
|
|
|
return False
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to check emergency conditions: {e}")
|
|
return False
|
|
|
|
def recover_from_method_exhaustion(self, platform: str, session_id: str) -> Dict[str, Any]:
|
|
"""
|
|
Handle the case when all available methods have been exhausted.
|
|
|
|
Args:
|
|
platform: Platform name
|
|
session_id: Current session ID
|
|
|
|
Returns:
|
|
Recovery strategy result
|
|
"""
|
|
self.logger.warning(f"Method exhaustion on {platform}, implementing recovery strategy")
|
|
|
|
try:
|
|
# Enable emergency mode
|
|
self.method_rotation_use_case.enable_emergency_mode(
|
|
platform, "method_exhaustion"
|
|
)
|
|
|
|
# Reset method cooldowns for emergency use
|
|
self._reset_method_cooldowns(platform)
|
|
|
|
# Use safest method available
|
|
emergency_methods = self.method_rotation_use_case.strategy_repo.get_emergency_methods(platform)
|
|
|
|
if emergency_methods:
|
|
safest_method = emergency_methods[0]
|
|
|
|
return {
|
|
'success': True,
|
|
'action': 'emergency_recovery',
|
|
'method': safest_method.method_name,
|
|
'message': f'Recovered using emergency method: {safest_method.method_name}'
|
|
}
|
|
else:
|
|
# No emergency methods available, fall back to original behavior
|
|
return self._fallback_to_original_behavior(platform, {
|
|
'error': 'method_exhaustion',
|
|
'message': 'No emergency methods available'
|
|
})
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to recover from method exhaustion: {e}")
|
|
return self._fallback_to_original_behavior(platform, {'error': str(e)})
|
|
|
|
def _classify_error_severity(self, error_details: Dict[str, Any]) -> ErrorSeverity:
|
|
"""Classify error severity based on error patterns"""
|
|
error_message = error_details.get('message', '').lower()
|
|
error_type = error_details.get('error_type', '').lower()
|
|
|
|
# Critical errors
|
|
critical_patterns = [
|
|
'system failure', 'database error', 'connection refused',
|
|
'authentication failed', 'service unavailable'
|
|
]
|
|
|
|
if any(pattern in error_message or pattern in error_type for pattern in critical_patterns):
|
|
return ErrorSeverity.CRITICAL
|
|
|
|
# High severity errors
|
|
high_patterns = [
|
|
'account suspended', 'rate limit exceeded', 'quota exceeded',
|
|
'blocked', 'banned', 'captcha failed multiple times'
|
|
]
|
|
|
|
if any(pattern in error_message or pattern in error_type for pattern in high_patterns):
|
|
return ErrorSeverity.HIGH
|
|
|
|
# Medium severity errors
|
|
medium_patterns = [
|
|
'timeout', 'verification failed', 'invalid credentials',
|
|
'network error', 'temporary failure'
|
|
]
|
|
|
|
if any(pattern in error_message or pattern in error_type for pattern in medium_patterns):
|
|
return ErrorSeverity.MEDIUM
|
|
|
|
# Default to low severity
|
|
return ErrorSeverity.LOW
|
|
|
|
def _handle_critical_error(self, platform: str, session_id: str,
|
|
error_details: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Handle critical errors with immediate emergency mode activation"""
|
|
self.logger.error(f"Critical error on {platform}: {error_details}")
|
|
|
|
# Enable emergency mode immediately
|
|
self.method_rotation_use_case.enable_emergency_mode(
|
|
platform, f"Critical error: {error_details.get('message', 'Unknown')}"
|
|
)
|
|
|
|
# Archive current session
|
|
self.method_rotation_use_case.session_repo.archive_session(session_id, False)
|
|
|
|
return {
|
|
'success': True,
|
|
'action': 'emergency_mode',
|
|
'severity': 'critical',
|
|
'message': 'Critical error handled, emergency mode enabled'
|
|
}
|
|
|
|
def _handle_high_severity_error(self, platform: str, session_id: str,
|
|
error_details: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Handle high severity errors with method blocking and rotation"""
|
|
error_type = error_details.get('error_type', 'unknown')
|
|
|
|
# Block the current method temporarily
|
|
session = self.method_rotation_use_case.session_repo.find_by_id(session_id)
|
|
if session:
|
|
current_method = session.current_method
|
|
|
|
# Block method for extended period
|
|
self.method_rotation_use_case.state_repo.block_method(
|
|
platform, current_method, f"High severity error: {error_type}"
|
|
)
|
|
|
|
# Attempt rotation to different method
|
|
next_method = self.method_rotation_use_case.rotate_method(
|
|
session_id, f"high_severity_error_{error_type}"
|
|
)
|
|
|
|
if next_method:
|
|
return {
|
|
'success': True,
|
|
'action': 'method_rotation',
|
|
'blocked_method': current_method,
|
|
'new_method': next_method.method_name,
|
|
'message': f'Rotated from {current_method} to {next_method.method_name}'
|
|
}
|
|
|
|
# Check if emergency mode should be triggered
|
|
if self.check_and_handle_emergency_conditions(platform):
|
|
return {
|
|
'success': True,
|
|
'action': 'emergency_mode_triggered',
|
|
'message': 'Emergency conditions detected, emergency mode enabled'
|
|
}
|
|
|
|
return {
|
|
'success': False,
|
|
'action': 'rotation_failed',
|
|
'message': 'Could not rotate to alternative method'
|
|
}
|
|
|
|
def _handle_medium_severity_error(self, platform: str, session_id: str,
|
|
error_details: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Handle medium severity errors with conditional rotation"""
|
|
# Attempt rotation if failure count is high
|
|
session = self.method_rotation_use_case.session_repo.find_by_id(session_id)
|
|
|
|
if session and session.failure_count >= 2:
|
|
next_method = self.method_rotation_use_case.rotate_method(
|
|
session_id, f"medium_severity_error_{error_details.get('error_type', 'unknown')}"
|
|
)
|
|
|
|
if next_method:
|
|
return {
|
|
'success': True,
|
|
'action': 'conditional_rotation',
|
|
'new_method': next_method.method_name,
|
|
'message': f'Rotated to {next_method.method_name} after {session.failure_count} failures'
|
|
}
|
|
|
|
return {
|
|
'success': True,
|
|
'action': 'continue_current_method',
|
|
'message': 'Continuing with current method, failure count below threshold'
|
|
}
|
|
|
|
def _handle_low_severity_error(self, platform: str, session_id: str,
|
|
error_details: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Handle low severity errors with minimal intervention"""
|
|
return {
|
|
'success': True,
|
|
'action': 'continue_current_method',
|
|
'message': 'Low severity error, continuing with current method'
|
|
}
|
|
|
|
def _fallback_to_original_behavior(self, platform: str,
|
|
error_details: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Fallback to original behavior when rotation system fails completely"""
|
|
self.logger.warning(f"Falling back to original behavior for {platform}")
|
|
|
|
return {
|
|
'success': True,
|
|
'action': 'fallback_to_original',
|
|
'method': 'email', # Default method
|
|
'message': 'Rotation system disabled, using original behavior',
|
|
'fallback_reason': error_details.get('message', 'Unknown error')
|
|
}
|
|
|
|
def _disable_rotation_for_platform(self, platform: str, reason: str) -> None:
|
|
"""Temporarily disable rotation for a specific platform"""
|
|
try:
|
|
# Block all methods except the safest one
|
|
strategies = self.method_rotation_use_case.strategy_repo.find_active_by_platform(platform)
|
|
|
|
for strategy in strategies[1:]: # Keep the first (safest) method active
|
|
self.method_rotation_use_case.strategy_repo.disable_method(
|
|
platform, strategy.method_name, f"Platform disabled: {reason}"
|
|
)
|
|
|
|
self.logger.info(f"Rotation disabled for {platform}: {reason}")
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to disable rotation for {platform}: {e}")
|
|
|
|
def _reset_method_cooldowns(self, platform: str) -> None:
|
|
"""Reset all method cooldowns for emergency recovery"""
|
|
try:
|
|
strategies = self.method_rotation_use_case.strategy_repo.find_by_platform(platform)
|
|
|
|
for strategy in strategies:
|
|
strategy.last_failure = None
|
|
strategy.cooldown_period = 0
|
|
self.method_rotation_use_case.strategy_repo.save(strategy)
|
|
|
|
self.logger.info(f"Method cooldowns reset for {platform}")
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to reset cooldowns for {platform}: {e}")
|
|
|
|
def _init_error_patterns(self) -> Dict[str, List[str]]:
|
|
"""Initialize error classification patterns"""
|
|
return {
|
|
'rate_limit': [
|
|
'rate limit', 'too many requests', 'quota exceeded',
|
|
'zu viele anfragen', 'rate limiting', 'throttled'
|
|
],
|
|
'account_suspended': [
|
|
'suspended', 'banned', 'blocked', 'gesperrt',
|
|
'account disabled', 'violation', 'restricted'
|
|
],
|
|
'network_error': [
|
|
'network error', 'connection failed', 'timeout',
|
|
'netzwerkfehler', 'verbindung fehlgeschlagen', 'dns error'
|
|
],
|
|
'verification_failed': [
|
|
'verification failed', 'captcha', 'human verification',
|
|
'verifizierung fehlgeschlagen', 'bot detected'
|
|
],
|
|
'system_error': [
|
|
'internal server error', 'service unavailable', 'maintenance',
|
|
'server fehler', 'wartung', 'system down'
|
|
]
|
|
}
|
|
|
|
def _init_fallback_strategies(self) -> Dict[str, Callable]:
|
|
"""Initialize fallback strategy functions"""
|
|
return {
|
|
'rate_limit': self._handle_rate_limit_fallback,
|
|
'account_suspended': self._handle_suspension_fallback,
|
|
'network_error': self._handle_network_fallback,
|
|
'verification_failed': self._handle_verification_fallback,
|
|
'system_error': self._handle_system_error_fallback
|
|
}
|
|
|
|
def _handle_rate_limit_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Handle rate limiting with extended cooldowns"""
|
|
# Extend cooldown periods for all methods
|
|
strategies = self.method_rotation_use_case.strategy_repo.find_by_platform(platform)
|
|
|
|
for strategy in strategies:
|
|
strategy.cooldown_period = max(strategy.cooldown_period * 2, 1800) # At least 30 minutes
|
|
self.method_rotation_use_case.strategy_repo.save(strategy)
|
|
|
|
return {'action': 'extended_cooldown', 'cooldown_minutes': 30}
|
|
|
|
def _handle_suspension_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Handle account suspension with method blocking"""
|
|
# Enable emergency mode with only safest methods
|
|
self.method_rotation_use_case.enable_emergency_mode(platform, "account_suspension")
|
|
return {'action': 'emergency_mode', 'reason': 'account_suspension'}
|
|
|
|
def _handle_network_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Handle network errors with retry strategy"""
|
|
return {'action': 'retry_with_delay', 'delay_seconds': 60}
|
|
|
|
def _handle_verification_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Handle verification failures with method rotation"""
|
|
return {'action': 'rotate_method', 'reason': 'verification_failed'}
|
|
|
|
def _handle_system_error_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Handle system errors with graceful degradation"""
|
|
self._disable_rotation_for_platform(platform, "system_error")
|
|
return {'action': 'disable_rotation', 'reason': 'system_error'} |