""" Comprehensive error handling and fallback mechanisms for method rotation system. Provides robust error recovery and graceful degradation strategies. """ import logging from datetime import datetime, timedelta from typing import Dict, Any, List, Optional, Callable from enum import Enum from domain.entities.method_rotation import MethodStrategy, RotationSession, RiskLevel from application.use_cases.method_rotation_use_case import MethodRotationUseCase class ErrorSeverity(Enum): """Error severity levels for rotation decisions""" LOW = "low" # Minor issues, continue with current method MEDIUM = "medium" # Moderate issues, consider rotation HIGH = "high" # Serious issues, rotate immediately CRITICAL = "critical" # Critical failure, enable emergency mode class RotationErrorHandler: """ Handles errors and provides fallback mechanisms for the rotation system. Implements intelligent error classification and recovery strategies. """ def __init__(self, method_rotation_use_case: MethodRotationUseCase): self.method_rotation_use_case = method_rotation_use_case self.logger = logging.getLogger(self.__class__.__name__) # Error classification patterns self.error_patterns = self._init_error_patterns() # Fallback strategies self.fallback_strategies = self._init_fallback_strategies() # Emergency mode settings self.emergency_thresholds = { 'failure_rate_threshold': 0.8, 'consecutive_failures_threshold': 5, 'time_window_minutes': 30 } def handle_rotation_error(self, platform: str, session_id: str, error_details: Dict[str, Any]) -> Dict[str, Any]: """ Handle rotation system errors with intelligent recovery. Args: platform: Platform name session_id: Current rotation session ID error_details: Error information Returns: Recovery action result """ try: # Classify error severity severity = self._classify_error_severity(error_details) # Log error with classification self.logger.warning(f"Rotation error on {platform}: {error_details.get('message', 'Unknown')} (Severity: {severity.value})") # Choose recovery strategy based on severity if severity == ErrorSeverity.CRITICAL: return self._handle_critical_error(platform, session_id, error_details) elif severity == ErrorSeverity.HIGH: return self._handle_high_severity_error(platform, session_id, error_details) elif severity == ErrorSeverity.MEDIUM: return self._handle_medium_severity_error(platform, session_id, error_details) else: return self._handle_low_severity_error(platform, session_id, error_details) except Exception as e: self.logger.error(f"Error in rotation error handler: {e}") return self._fallback_to_original_behavior(platform, error_details) def handle_system_failure(self, platform: str, failure_details: Dict[str, Any]) -> Dict[str, Any]: """ Handle complete rotation system failures with graceful degradation. Args: platform: Platform name failure_details: System failure information Returns: Fallback strategy result """ self.logger.error(f"Rotation system failure on {platform}: {failure_details}") try: # Attempt to gracefully shut down rotation for this platform self._disable_rotation_for_platform(platform, "system_failure") # Enable emergency mode with safest methods only self.method_rotation_use_case.enable_emergency_mode( platform, f"System failure: {failure_details.get('message', 'Unknown')}" ) return { 'success': True, 'action': 'emergency_mode_enabled', 'fallback_method': 'email', 'message': 'System failure handled, emergency mode activated' } except Exception as e: self.logger.error(f"Failed to handle system failure: {e}") return self._fallback_to_original_behavior(platform, failure_details) def check_and_handle_emergency_conditions(self, platform: str) -> bool: """ Check if emergency conditions are met and handle accordingly. Args: platform: Platform to check Returns: True if emergency mode was triggered """ try: # Get platform statistics stats = self.method_rotation_use_case.strategy_repo.get_platform_statistics(platform) # Check failure rate threshold recent_failures = stats.get('recent_failures_24h', 0) recent_successes = stats.get('recent_successes_24h', 0) total_recent = recent_failures + recent_successes if total_recent > 0: failure_rate = recent_failures / total_recent if failure_rate >= self.emergency_thresholds['failure_rate_threshold']: self.logger.warning(f"High failure rate detected on {platform}: {failure_rate:.2f}") self.method_rotation_use_case.enable_emergency_mode( platform, f"High failure rate: {failure_rate:.2f}" ) return True # Check consecutive failures session_stats = self.method_rotation_use_case.session_repo.get_session_statistics(platform, days=1) failed_sessions = session_stats.get('failed_sessions', 0) if failed_sessions >= self.emergency_thresholds['consecutive_failures_threshold']: self.logger.warning(f"High consecutive failures on {platform}: {failed_sessions}") self.method_rotation_use_case.enable_emergency_mode( platform, f"Consecutive failures: {failed_sessions}" ) return True return False except Exception as e: self.logger.error(f"Failed to check emergency conditions: {e}") return False def recover_from_method_exhaustion(self, platform: str, session_id: str) -> Dict[str, Any]: """ Handle the case when all available methods have been exhausted. Args: platform: Platform name session_id: Current session ID Returns: Recovery strategy result """ self.logger.warning(f"Method exhaustion on {platform}, implementing recovery strategy") try: # Enable emergency mode self.method_rotation_use_case.enable_emergency_mode( platform, "method_exhaustion" ) # Reset method cooldowns for emergency use self._reset_method_cooldowns(platform) # Use safest method available emergency_methods = self.method_rotation_use_case.strategy_repo.get_emergency_methods(platform) if emergency_methods: safest_method = emergency_methods[0] return { 'success': True, 'action': 'emergency_recovery', 'method': safest_method.method_name, 'message': f'Recovered using emergency method: {safest_method.method_name}' } else: # No emergency methods available, fall back to original behavior return self._fallback_to_original_behavior(platform, { 'error': 'method_exhaustion', 'message': 'No emergency methods available' }) except Exception as e: self.logger.error(f"Failed to recover from method exhaustion: {e}") return self._fallback_to_original_behavior(platform, {'error': str(e)}) def _classify_error_severity(self, error_details: Dict[str, Any]) -> ErrorSeverity: """Classify error severity based on error patterns""" error_message = error_details.get('message', '').lower() error_type = error_details.get('error_type', '').lower() # Critical errors critical_patterns = [ 'system failure', 'database error', 'connection refused', 'authentication failed', 'service unavailable' ] if any(pattern in error_message or pattern in error_type for pattern in critical_patterns): return ErrorSeverity.CRITICAL # High severity errors high_patterns = [ 'account suspended', 'rate limit exceeded', 'quota exceeded', 'blocked', 'banned', 'captcha failed multiple times' ] if any(pattern in error_message or pattern in error_type for pattern in high_patterns): return ErrorSeverity.HIGH # Medium severity errors medium_patterns = [ 'timeout', 'verification failed', 'invalid credentials', 'network error', 'temporary failure' ] if any(pattern in error_message or pattern in error_type for pattern in medium_patterns): return ErrorSeverity.MEDIUM # Default to low severity return ErrorSeverity.LOW def _handle_critical_error(self, platform: str, session_id: str, error_details: Dict[str, Any]) -> Dict[str, Any]: """Handle critical errors with immediate emergency mode activation""" self.logger.error(f"Critical error on {platform}: {error_details}") # Enable emergency mode immediately self.method_rotation_use_case.enable_emergency_mode( platform, f"Critical error: {error_details.get('message', 'Unknown')}" ) # Archive current session self.method_rotation_use_case.session_repo.archive_session(session_id, False) return { 'success': True, 'action': 'emergency_mode', 'severity': 'critical', 'message': 'Critical error handled, emergency mode enabled' } def _handle_high_severity_error(self, platform: str, session_id: str, error_details: Dict[str, Any]) -> Dict[str, Any]: """Handle high severity errors with method blocking and rotation""" error_type = error_details.get('error_type', 'unknown') # Block the current method temporarily session = self.method_rotation_use_case.session_repo.find_by_id(session_id) if session: current_method = session.current_method # Block method for extended period self.method_rotation_use_case.state_repo.block_method( platform, current_method, f"High severity error: {error_type}" ) # Attempt rotation to different method next_method = self.method_rotation_use_case.rotate_method( session_id, f"high_severity_error_{error_type}" ) if next_method: return { 'success': True, 'action': 'method_rotation', 'blocked_method': current_method, 'new_method': next_method.method_name, 'message': f'Rotated from {current_method} to {next_method.method_name}' } # Check if emergency mode should be triggered if self.check_and_handle_emergency_conditions(platform): return { 'success': True, 'action': 'emergency_mode_triggered', 'message': 'Emergency conditions detected, emergency mode enabled' } return { 'success': False, 'action': 'rotation_failed', 'message': 'Could not rotate to alternative method' } def _handle_medium_severity_error(self, platform: str, session_id: str, error_details: Dict[str, Any]) -> Dict[str, Any]: """Handle medium severity errors with conditional rotation""" # Attempt rotation if failure count is high session = self.method_rotation_use_case.session_repo.find_by_id(session_id) if session and session.failure_count >= 2: next_method = self.method_rotation_use_case.rotate_method( session_id, f"medium_severity_error_{error_details.get('error_type', 'unknown')}" ) if next_method: return { 'success': True, 'action': 'conditional_rotation', 'new_method': next_method.method_name, 'message': f'Rotated to {next_method.method_name} after {session.failure_count} failures' } return { 'success': True, 'action': 'continue_current_method', 'message': 'Continuing with current method, failure count below threshold' } def _handle_low_severity_error(self, platform: str, session_id: str, error_details: Dict[str, Any]) -> Dict[str, Any]: """Handle low severity errors with minimal intervention""" return { 'success': True, 'action': 'continue_current_method', 'message': 'Low severity error, continuing with current method' } def _fallback_to_original_behavior(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]: """Fallback to original behavior when rotation system fails completely""" self.logger.warning(f"Falling back to original behavior for {platform}") return { 'success': True, 'action': 'fallback_to_original', 'method': 'email', # Default method 'message': 'Rotation system disabled, using original behavior', 'fallback_reason': error_details.get('message', 'Unknown error') } def _disable_rotation_for_platform(self, platform: str, reason: str) -> None: """Temporarily disable rotation for a specific platform""" try: # Block all methods except the safest one strategies = self.method_rotation_use_case.strategy_repo.find_active_by_platform(platform) for strategy in strategies[1:]: # Keep the first (safest) method active self.method_rotation_use_case.strategy_repo.disable_method( platform, strategy.method_name, f"Platform disabled: {reason}" ) self.logger.info(f"Rotation disabled for {platform}: {reason}") except Exception as e: self.logger.error(f"Failed to disable rotation for {platform}: {e}") def _reset_method_cooldowns(self, platform: str) -> None: """Reset all method cooldowns for emergency recovery""" try: strategies = self.method_rotation_use_case.strategy_repo.find_by_platform(platform) for strategy in strategies: strategy.last_failure = None strategy.cooldown_period = 0 self.method_rotation_use_case.strategy_repo.save(strategy) self.logger.info(f"Method cooldowns reset for {platform}") except Exception as e: self.logger.error(f"Failed to reset cooldowns for {platform}: {e}") def _init_error_patterns(self) -> Dict[str, List[str]]: """Initialize error classification patterns""" return { 'rate_limit': [ 'rate limit', 'too many requests', 'quota exceeded', 'zu viele anfragen', 'rate limiting', 'throttled' ], 'account_suspended': [ 'suspended', 'banned', 'blocked', 'gesperrt', 'account disabled', 'violation', 'restricted' ], 'network_error': [ 'network error', 'connection failed', 'timeout', 'netzwerkfehler', 'verbindung fehlgeschlagen', 'dns error' ], 'verification_failed': [ 'verification failed', 'captcha', 'human verification', 'verifizierung fehlgeschlagen', 'bot detected' ], 'system_error': [ 'internal server error', 'service unavailable', 'maintenance', 'server fehler', 'wartung', 'system down' ] } def _init_fallback_strategies(self) -> Dict[str, Callable]: """Initialize fallback strategy functions""" return { 'rate_limit': self._handle_rate_limit_fallback, 'account_suspended': self._handle_suspension_fallback, 'network_error': self._handle_network_fallback, 'verification_failed': self._handle_verification_fallback, 'system_error': self._handle_system_error_fallback } def _handle_rate_limit_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]: """Handle rate limiting with extended cooldowns""" # Extend cooldown periods for all methods strategies = self.method_rotation_use_case.strategy_repo.find_by_platform(platform) for strategy in strategies: strategy.cooldown_period = max(strategy.cooldown_period * 2, 1800) # At least 30 minutes self.method_rotation_use_case.strategy_repo.save(strategy) return {'action': 'extended_cooldown', 'cooldown_minutes': 30} def _handle_suspension_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]: """Handle account suspension with method blocking""" # Enable emergency mode with only safest methods self.method_rotation_use_case.enable_emergency_mode(platform, "account_suspension") return {'action': 'emergency_mode', 'reason': 'account_suspension'} def _handle_network_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]: """Handle network errors with retry strategy""" return {'action': 'retry_with_delay', 'delay_seconds': 60} def _handle_verification_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]: """Handle verification failures with method rotation""" return {'action': 'rotate_method', 'reason': 'verification_failed'} def _handle_system_error_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]: """Handle system errors with graceful degradation""" self._disable_rotation_for_platform(platform, "system_error") return {'action': 'disable_rotation', 'reason': 'system_error'}