Initial commit
Dieser Commit ist enthalten in:
Claude Project Manager
443
controllers/platform_controllers/rotation_error_handler.py
Normale Datei
443
controllers/platform_controllers/rotation_error_handler.py
Normale Datei
@ -0,0 +1,443 @@
|
||||
"""
|
||||
Comprehensive error handling and fallback mechanisms for method rotation system.
|
||||
Provides robust error recovery and graceful degradation strategies.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, Any, List, Optional, Callable
|
||||
from enum import Enum
|
||||
|
||||
from domain.entities.method_rotation import MethodStrategy, RotationSession, RiskLevel
|
||||
from application.use_cases.method_rotation_use_case import MethodRotationUseCase
|
||||
|
||||
|
||||
class ErrorSeverity(Enum):
|
||||
"""Error severity levels for rotation decisions"""
|
||||
LOW = "low" # Minor issues, continue with current method
|
||||
MEDIUM = "medium" # Moderate issues, consider rotation
|
||||
HIGH = "high" # Serious issues, rotate immediately
|
||||
CRITICAL = "critical" # Critical failure, enable emergency mode
|
||||
|
||||
|
||||
class RotationErrorHandler:
|
||||
"""
|
||||
Handles errors and provides fallback mechanisms for the rotation system.
|
||||
Implements intelligent error classification and recovery strategies.
|
||||
"""
|
||||
|
||||
def __init__(self, method_rotation_use_case: MethodRotationUseCase):
|
||||
self.method_rotation_use_case = method_rotation_use_case
|
||||
self.logger = logging.getLogger(self.__class__.__name__)
|
||||
|
||||
# Error classification patterns
|
||||
self.error_patterns = self._init_error_patterns()
|
||||
|
||||
# Fallback strategies
|
||||
self.fallback_strategies = self._init_fallback_strategies()
|
||||
|
||||
# Emergency mode settings
|
||||
self.emergency_thresholds = {
|
||||
'failure_rate_threshold': 0.8,
|
||||
'consecutive_failures_threshold': 5,
|
||||
'time_window_minutes': 30
|
||||
}
|
||||
|
||||
def handle_rotation_error(self, platform: str, session_id: str,
|
||||
error_details: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Handle rotation system errors with intelligent recovery.
|
||||
|
||||
Args:
|
||||
platform: Platform name
|
||||
session_id: Current rotation session ID
|
||||
error_details: Error information
|
||||
|
||||
Returns:
|
||||
Recovery action result
|
||||
"""
|
||||
try:
|
||||
# Classify error severity
|
||||
severity = self._classify_error_severity(error_details)
|
||||
|
||||
# Log error with classification
|
||||
self.logger.warning(f"Rotation error on {platform}: {error_details.get('message', 'Unknown')} (Severity: {severity.value})")
|
||||
|
||||
# Choose recovery strategy based on severity
|
||||
if severity == ErrorSeverity.CRITICAL:
|
||||
return self._handle_critical_error(platform, session_id, error_details)
|
||||
elif severity == ErrorSeverity.HIGH:
|
||||
return self._handle_high_severity_error(platform, session_id, error_details)
|
||||
elif severity == ErrorSeverity.MEDIUM:
|
||||
return self._handle_medium_severity_error(platform, session_id, error_details)
|
||||
else:
|
||||
return self._handle_low_severity_error(platform, session_id, error_details)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error in rotation error handler: {e}")
|
||||
return self._fallback_to_original_behavior(platform, error_details)
|
||||
|
||||
def handle_system_failure(self, platform: str, failure_details: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Handle complete rotation system failures with graceful degradation.
|
||||
|
||||
Args:
|
||||
platform: Platform name
|
||||
failure_details: System failure information
|
||||
|
||||
Returns:
|
||||
Fallback strategy result
|
||||
"""
|
||||
self.logger.error(f"Rotation system failure on {platform}: {failure_details}")
|
||||
|
||||
try:
|
||||
# Attempt to gracefully shut down rotation for this platform
|
||||
self._disable_rotation_for_platform(platform, "system_failure")
|
||||
|
||||
# Enable emergency mode with safest methods only
|
||||
self.method_rotation_use_case.enable_emergency_mode(
|
||||
platform, f"System failure: {failure_details.get('message', 'Unknown')}"
|
||||
)
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'action': 'emergency_mode_enabled',
|
||||
'fallback_method': 'email',
|
||||
'message': 'System failure handled, emergency mode activated'
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to handle system failure: {e}")
|
||||
return self._fallback_to_original_behavior(platform, failure_details)
|
||||
|
||||
def check_and_handle_emergency_conditions(self, platform: str) -> bool:
|
||||
"""
|
||||
Check if emergency conditions are met and handle accordingly.
|
||||
|
||||
Args:
|
||||
platform: Platform to check
|
||||
|
||||
Returns:
|
||||
True if emergency mode was triggered
|
||||
"""
|
||||
try:
|
||||
# Get platform statistics
|
||||
stats = self.method_rotation_use_case.strategy_repo.get_platform_statistics(platform)
|
||||
|
||||
# Check failure rate threshold
|
||||
recent_failures = stats.get('recent_failures_24h', 0)
|
||||
recent_successes = stats.get('recent_successes_24h', 0)
|
||||
total_recent = recent_failures + recent_successes
|
||||
|
||||
if total_recent > 0:
|
||||
failure_rate = recent_failures / total_recent
|
||||
|
||||
if failure_rate >= self.emergency_thresholds['failure_rate_threshold']:
|
||||
self.logger.warning(f"High failure rate detected on {platform}: {failure_rate:.2f}")
|
||||
self.method_rotation_use_case.enable_emergency_mode(
|
||||
platform, f"High failure rate: {failure_rate:.2f}"
|
||||
)
|
||||
return True
|
||||
|
||||
# Check consecutive failures
|
||||
session_stats = self.method_rotation_use_case.session_repo.get_session_statistics(platform, days=1)
|
||||
failed_sessions = session_stats.get('failed_sessions', 0)
|
||||
|
||||
if failed_sessions >= self.emergency_thresholds['consecutive_failures_threshold']:
|
||||
self.logger.warning(f"High consecutive failures on {platform}: {failed_sessions}")
|
||||
self.method_rotation_use_case.enable_emergency_mode(
|
||||
platform, f"Consecutive failures: {failed_sessions}"
|
||||
)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to check emergency conditions: {e}")
|
||||
return False
|
||||
|
||||
def recover_from_method_exhaustion(self, platform: str, session_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Handle the case when all available methods have been exhausted.
|
||||
|
||||
Args:
|
||||
platform: Platform name
|
||||
session_id: Current session ID
|
||||
|
||||
Returns:
|
||||
Recovery strategy result
|
||||
"""
|
||||
self.logger.warning(f"Method exhaustion on {platform}, implementing recovery strategy")
|
||||
|
||||
try:
|
||||
# Enable emergency mode
|
||||
self.method_rotation_use_case.enable_emergency_mode(
|
||||
platform, "method_exhaustion"
|
||||
)
|
||||
|
||||
# Reset method cooldowns for emergency use
|
||||
self._reset_method_cooldowns(platform)
|
||||
|
||||
# Use safest method available
|
||||
emergency_methods = self.method_rotation_use_case.strategy_repo.get_emergency_methods(platform)
|
||||
|
||||
if emergency_methods:
|
||||
safest_method = emergency_methods[0]
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'action': 'emergency_recovery',
|
||||
'method': safest_method.method_name,
|
||||
'message': f'Recovered using emergency method: {safest_method.method_name}'
|
||||
}
|
||||
else:
|
||||
# No emergency methods available, fall back to original behavior
|
||||
return self._fallback_to_original_behavior(platform, {
|
||||
'error': 'method_exhaustion',
|
||||
'message': 'No emergency methods available'
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to recover from method exhaustion: {e}")
|
||||
return self._fallback_to_original_behavior(platform, {'error': str(e)})
|
||||
|
||||
def _classify_error_severity(self, error_details: Dict[str, Any]) -> ErrorSeverity:
|
||||
"""Classify error severity based on error patterns"""
|
||||
error_message = error_details.get('message', '').lower()
|
||||
error_type = error_details.get('error_type', '').lower()
|
||||
|
||||
# Critical errors
|
||||
critical_patterns = [
|
||||
'system failure', 'database error', 'connection refused',
|
||||
'authentication failed', 'service unavailable'
|
||||
]
|
||||
|
||||
if any(pattern in error_message or pattern in error_type for pattern in critical_patterns):
|
||||
return ErrorSeverity.CRITICAL
|
||||
|
||||
# High severity errors
|
||||
high_patterns = [
|
||||
'account suspended', 'rate limit exceeded', 'quota exceeded',
|
||||
'blocked', 'banned', 'captcha failed multiple times'
|
||||
]
|
||||
|
||||
if any(pattern in error_message or pattern in error_type for pattern in high_patterns):
|
||||
return ErrorSeverity.HIGH
|
||||
|
||||
# Medium severity errors
|
||||
medium_patterns = [
|
||||
'timeout', 'verification failed', 'invalid credentials',
|
||||
'network error', 'temporary failure'
|
||||
]
|
||||
|
||||
if any(pattern in error_message or pattern in error_type for pattern in medium_patterns):
|
||||
return ErrorSeverity.MEDIUM
|
||||
|
||||
# Default to low severity
|
||||
return ErrorSeverity.LOW
|
||||
|
||||
def _handle_critical_error(self, platform: str, session_id: str,
|
||||
error_details: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Handle critical errors with immediate emergency mode activation"""
|
||||
self.logger.error(f"Critical error on {platform}: {error_details}")
|
||||
|
||||
# Enable emergency mode immediately
|
||||
self.method_rotation_use_case.enable_emergency_mode(
|
||||
platform, f"Critical error: {error_details.get('message', 'Unknown')}"
|
||||
)
|
||||
|
||||
# Archive current session
|
||||
self.method_rotation_use_case.session_repo.archive_session(session_id, False)
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'action': 'emergency_mode',
|
||||
'severity': 'critical',
|
||||
'message': 'Critical error handled, emergency mode enabled'
|
||||
}
|
||||
|
||||
def _handle_high_severity_error(self, platform: str, session_id: str,
|
||||
error_details: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Handle high severity errors with method blocking and rotation"""
|
||||
error_type = error_details.get('error_type', 'unknown')
|
||||
|
||||
# Block the current method temporarily
|
||||
session = self.method_rotation_use_case.session_repo.find_by_id(session_id)
|
||||
if session:
|
||||
current_method = session.current_method
|
||||
|
||||
# Block method for extended period
|
||||
self.method_rotation_use_case.state_repo.block_method(
|
||||
platform, current_method, f"High severity error: {error_type}"
|
||||
)
|
||||
|
||||
# Attempt rotation to different method
|
||||
next_method = self.method_rotation_use_case.rotate_method(
|
||||
session_id, f"high_severity_error_{error_type}"
|
||||
)
|
||||
|
||||
if next_method:
|
||||
return {
|
||||
'success': True,
|
||||
'action': 'method_rotation',
|
||||
'blocked_method': current_method,
|
||||
'new_method': next_method.method_name,
|
||||
'message': f'Rotated from {current_method} to {next_method.method_name}'
|
||||
}
|
||||
|
||||
# Check if emergency mode should be triggered
|
||||
if self.check_and_handle_emergency_conditions(platform):
|
||||
return {
|
||||
'success': True,
|
||||
'action': 'emergency_mode_triggered',
|
||||
'message': 'Emergency conditions detected, emergency mode enabled'
|
||||
}
|
||||
|
||||
return {
|
||||
'success': False,
|
||||
'action': 'rotation_failed',
|
||||
'message': 'Could not rotate to alternative method'
|
||||
}
|
||||
|
||||
def _handle_medium_severity_error(self, platform: str, session_id: str,
|
||||
error_details: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Handle medium severity errors with conditional rotation"""
|
||||
# Attempt rotation if failure count is high
|
||||
session = self.method_rotation_use_case.session_repo.find_by_id(session_id)
|
||||
|
||||
if session and session.failure_count >= 2:
|
||||
next_method = self.method_rotation_use_case.rotate_method(
|
||||
session_id, f"medium_severity_error_{error_details.get('error_type', 'unknown')}"
|
||||
)
|
||||
|
||||
if next_method:
|
||||
return {
|
||||
'success': True,
|
||||
'action': 'conditional_rotation',
|
||||
'new_method': next_method.method_name,
|
||||
'message': f'Rotated to {next_method.method_name} after {session.failure_count} failures'
|
||||
}
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'action': 'continue_current_method',
|
||||
'message': 'Continuing with current method, failure count below threshold'
|
||||
}
|
||||
|
||||
def _handle_low_severity_error(self, platform: str, session_id: str,
|
||||
error_details: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Handle low severity errors with minimal intervention"""
|
||||
return {
|
||||
'success': True,
|
||||
'action': 'continue_current_method',
|
||||
'message': 'Low severity error, continuing with current method'
|
||||
}
|
||||
|
||||
def _fallback_to_original_behavior(self, platform: str,
|
||||
error_details: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Fallback to original behavior when rotation system fails completely"""
|
||||
self.logger.warning(f"Falling back to original behavior for {platform}")
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'action': 'fallback_to_original',
|
||||
'method': 'email', # Default method
|
||||
'message': 'Rotation system disabled, using original behavior',
|
||||
'fallback_reason': error_details.get('message', 'Unknown error')
|
||||
}
|
||||
|
||||
def _disable_rotation_for_platform(self, platform: str, reason: str) -> None:
|
||||
"""Temporarily disable rotation for a specific platform"""
|
||||
try:
|
||||
# Block all methods except the safest one
|
||||
strategies = self.method_rotation_use_case.strategy_repo.find_active_by_platform(platform)
|
||||
|
||||
for strategy in strategies[1:]: # Keep the first (safest) method active
|
||||
self.method_rotation_use_case.strategy_repo.disable_method(
|
||||
platform, strategy.method_name, f"Platform disabled: {reason}"
|
||||
)
|
||||
|
||||
self.logger.info(f"Rotation disabled for {platform}: {reason}")
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to disable rotation for {platform}: {e}")
|
||||
|
||||
def _reset_method_cooldowns(self, platform: str) -> None:
|
||||
"""Reset all method cooldowns for emergency recovery"""
|
||||
try:
|
||||
strategies = self.method_rotation_use_case.strategy_repo.find_by_platform(platform)
|
||||
|
||||
for strategy in strategies:
|
||||
strategy.last_failure = None
|
||||
strategy.cooldown_period = 0
|
||||
self.method_rotation_use_case.strategy_repo.save(strategy)
|
||||
|
||||
self.logger.info(f"Method cooldowns reset for {platform}")
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to reset cooldowns for {platform}: {e}")
|
||||
|
||||
def _init_error_patterns(self) -> Dict[str, List[str]]:
|
||||
"""Initialize error classification patterns"""
|
||||
return {
|
||||
'rate_limit': [
|
||||
'rate limit', 'too many requests', 'quota exceeded',
|
||||
'zu viele anfragen', 'rate limiting', 'throttled'
|
||||
],
|
||||
'account_suspended': [
|
||||
'suspended', 'banned', 'blocked', 'gesperrt',
|
||||
'account disabled', 'violation', 'restricted'
|
||||
],
|
||||
'network_error': [
|
||||
'network error', 'connection failed', 'timeout',
|
||||
'netzwerkfehler', 'verbindung fehlgeschlagen', 'dns error'
|
||||
],
|
||||
'verification_failed': [
|
||||
'verification failed', 'captcha', 'human verification',
|
||||
'verifizierung fehlgeschlagen', 'bot detected'
|
||||
],
|
||||
'system_error': [
|
||||
'internal server error', 'service unavailable', 'maintenance',
|
||||
'server fehler', 'wartung', 'system down'
|
||||
]
|
||||
}
|
||||
|
||||
def _init_fallback_strategies(self) -> Dict[str, Callable]:
|
||||
"""Initialize fallback strategy functions"""
|
||||
return {
|
||||
'rate_limit': self._handle_rate_limit_fallback,
|
||||
'account_suspended': self._handle_suspension_fallback,
|
||||
'network_error': self._handle_network_fallback,
|
||||
'verification_failed': self._handle_verification_fallback,
|
||||
'system_error': self._handle_system_error_fallback
|
||||
}
|
||||
|
||||
def _handle_rate_limit_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Handle rate limiting with extended cooldowns"""
|
||||
# Extend cooldown periods for all methods
|
||||
strategies = self.method_rotation_use_case.strategy_repo.find_by_platform(platform)
|
||||
|
||||
for strategy in strategies:
|
||||
strategy.cooldown_period = max(strategy.cooldown_period * 2, 1800) # At least 30 minutes
|
||||
self.method_rotation_use_case.strategy_repo.save(strategy)
|
||||
|
||||
return {'action': 'extended_cooldown', 'cooldown_minutes': 30}
|
||||
|
||||
def _handle_suspension_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Handle account suspension with method blocking"""
|
||||
# Enable emergency mode with only safest methods
|
||||
self.method_rotation_use_case.enable_emergency_mode(platform, "account_suspension")
|
||||
return {'action': 'emergency_mode', 'reason': 'account_suspension'}
|
||||
|
||||
def _handle_network_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Handle network errors with retry strategy"""
|
||||
return {'action': 'retry_with_delay', 'delay_seconds': 60}
|
||||
|
||||
def _handle_verification_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Handle verification failures with method rotation"""
|
||||
return {'action': 'rotate_method', 'reason': 'verification_failed'}
|
||||
|
||||
def _handle_system_error_fallback(self, platform: str, error_details: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Handle system errors with graceful degradation"""
|
||||
self._disable_rotation_for_platform(platform, "system_error")
|
||||
return {'action': 'disable_rotation', 'reason': 'system_error'}
|
||||
In neuem Issue referenzieren
Einen Benutzer sperren