Initial commit
Dieser Commit ist enthalten in:
Claude Project Manager
413
utils/proxy_rotator.py
Normale Datei
413
utils/proxy_rotator.py
Normale Datei
@ -0,0 +1,413 @@
|
||||
# Path: p:/Chimaira/Code-Playwright/utils/proxy_rotator.py
|
||||
|
||||
"""
|
||||
Proxy-Rotations- und Verwaltungsfunktionalität.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import random
|
||||
import logging
|
||||
import requests
|
||||
import time
|
||||
from typing import Dict, List, Any, Optional, Tuple, Union
|
||||
|
||||
logger = logging.getLogger("proxy_rotator")
|
||||
|
||||
class ProxyRotator:
|
||||
"""Klasse zur Verwaltung und Rotation von Proxies."""
|
||||
|
||||
CONFIG_FILE = os.path.join("config", "proxies.json")
|
||||
|
||||
def __init__(self):
|
||||
"""Initialisiert den ProxyRotator und lädt die Konfiguration."""
|
||||
self.config = self.load_config()
|
||||
self.current_proxy = None
|
||||
self.last_rotation_time = 0
|
||||
|
||||
# Stelle sicher, dass das Konfigurationsverzeichnis existiert
|
||||
os.makedirs(os.path.dirname(self.CONFIG_FILE), exist_ok=True)
|
||||
|
||||
def load_config(self) -> Dict[str, Any]:
|
||||
"""Lädt die Proxy-Konfiguration aus der Konfigurationsdatei."""
|
||||
if not os.path.exists(self.CONFIG_FILE):
|
||||
return {
|
||||
"ipv4": [],
|
||||
"ipv6": [],
|
||||
"mobile": [],
|
||||
"mobile_api": {
|
||||
"marsproxies": "",
|
||||
"iproyal": ""
|
||||
},
|
||||
"rotation_interval": 300 # 5 Minuten
|
||||
}
|
||||
|
||||
try:
|
||||
with open(self.CONFIG_FILE, "r", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
|
||||
logger.info(f"Proxy-Konfiguration geladen: {len(config.get('ipv4', []))} IPv4, "
|
||||
f"{len(config.get('ipv6', []))} IPv6, {len(config.get('mobile', []))} Mobile")
|
||||
|
||||
return config
|
||||
except Exception as e:
|
||||
logger.error(f"Fehler beim Laden der Proxy-Konfiguration: {e}")
|
||||
return {
|
||||
"ipv4": [],
|
||||
"ipv6": [],
|
||||
"mobile": [],
|
||||
"mobile_api": {
|
||||
"marsproxies": "",
|
||||
"iproyal": ""
|
||||
},
|
||||
"rotation_interval": 300
|
||||
}
|
||||
|
||||
def save_config(self) -> bool:
|
||||
"""Speichert die Proxy-Konfiguration in die Konfigurationsdatei."""
|
||||
try:
|
||||
with open(self.CONFIG_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(self.config, f, indent=2)
|
||||
|
||||
logger.info("Proxy-Konfiguration gespeichert")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Fehler beim Speichern der Proxy-Konfiguration: {e}")
|
||||
return False
|
||||
|
||||
def get_config(self) -> Dict[str, Any]:
|
||||
"""Gibt die aktuelle Konfiguration zurück."""
|
||||
return self.config
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]) -> bool:
|
||||
"""Aktualisiert die Konfiguration mit den neuen Werten."""
|
||||
try:
|
||||
# Aktualisiere nur die bereitgestellten Schlüssel
|
||||
for key, value in new_config.items():
|
||||
self.config[key] = value
|
||||
|
||||
# Konfiguration speichern
|
||||
return self.save_config()
|
||||
except Exception as e:
|
||||
logger.error(f"Fehler beim Aktualisieren der Proxy-Konfiguration: {e}")
|
||||
return False
|
||||
|
||||
def get_proxies_by_type(self, proxy_type: str) -> List[str]:
|
||||
"""Gibt eine Liste von Proxies des angegebenen Typs zurück."""
|
||||
if proxy_type.lower() not in ["ipv4", "ipv6", "mobile"]:
|
||||
logger.warning(f"Ungültiger Proxy-Typ: {proxy_type}")
|
||||
return []
|
||||
|
||||
return self.config.get(proxy_type.lower(), [])
|
||||
|
||||
def get_random_proxy(self, proxy_type: str) -> Optional[str]:
|
||||
"""Gibt einen zufälligen Proxy des angegebenen Typs zurück."""
|
||||
proxies = self.get_proxies_by_type(proxy_type)
|
||||
|
||||
if not proxies:
|
||||
logger.warning(f"Keine Proxies vom Typ '{proxy_type}' verfügbar")
|
||||
return None
|
||||
|
||||
return random.choice(proxies)
|
||||
|
||||
def get_proxy(self, proxy_type=None):
|
||||
"""
|
||||
Gibt eine Proxy-Konfiguration für den angegebenen Typ zurück.
|
||||
|
||||
Args:
|
||||
proxy_type: Proxy-Typ ("ipv4", "ipv6", "mobile") oder None für zufälligen Typ
|
||||
|
||||
Returns:
|
||||
Dict mit Proxy-Konfiguration oder None, wenn kein Proxy verfügbar ist
|
||||
"""
|
||||
try:
|
||||
# Wenn kein Proxy-Typ angegeben ist, einen zufälligen verwenden
|
||||
if proxy_type is None:
|
||||
available_types = []
|
||||
if self.config.get("ipv4"):
|
||||
available_types.append("ipv4")
|
||||
if self.config.get("ipv6"):
|
||||
available_types.append("ipv6")
|
||||
if self.config.get("mobile"):
|
||||
available_types.append("mobile")
|
||||
|
||||
if not available_types:
|
||||
logger.warning("Keine Proxies verfügbar")
|
||||
return None
|
||||
|
||||
proxy_type = random.choice(available_types)
|
||||
|
||||
# Proxy vom angegebenen Typ holen
|
||||
proxy_list = self.get_proxies_by_type(proxy_type)
|
||||
|
||||
if not proxy_list:
|
||||
logger.warning(f"Keine Proxies vom Typ '{proxy_type}' verfügbar")
|
||||
return None
|
||||
|
||||
# Zufälligen Proxy aus der Liste auswählen
|
||||
proxy = random.choice(proxy_list)
|
||||
|
||||
# Proxy-URL parsen
|
||||
parts = proxy.split(":")
|
||||
|
||||
if len(parts) >= 4:
|
||||
# Format: host:port:username:password
|
||||
host, port, username, password = parts[:4]
|
||||
|
||||
return {
|
||||
"server": f"http://{host}:{port}",
|
||||
"username": username,
|
||||
"password": password
|
||||
}
|
||||
elif len(parts) >= 2:
|
||||
# Format: host:port
|
||||
host, port = parts[:2]
|
||||
|
||||
return {
|
||||
"server": f"http://{host}:{port}"
|
||||
}
|
||||
else:
|
||||
logger.warning(f"Ungültiges Proxy-Format: {self.mask_proxy_credentials(proxy)}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Fehler beim Abrufen des Proxys: {e}")
|
||||
return None
|
||||
|
||||
def get_next_proxy(self, proxy_type: str, force_rotation: bool = False) -> Optional[str]:
|
||||
"""
|
||||
Gibt den nächsten zu verwendenden Proxy zurück, unter Berücksichtigung des Rotationsintervalls.
|
||||
|
||||
Args:
|
||||
proxy_type: Typ des Proxys (ipv4, ipv6, mobile)
|
||||
force_rotation: Erzwingt eine Rotation, unabhängig vom Zeitintervall
|
||||
|
||||
Returns:
|
||||
Proxy-String oder None, wenn kein Proxy verfügbar ist
|
||||
"""
|
||||
current_time = time.time()
|
||||
interval = self.config.get("rotation_interval", 300) # Standardintervall: 5 Minuten
|
||||
|
||||
# Rotation durchführen, wenn das Intervall abgelaufen ist oder erzwungen wird
|
||||
if force_rotation or self.current_proxy is None or (current_time - self.last_rotation_time) > interval:
|
||||
self.current_proxy = self.get_random_proxy(proxy_type)
|
||||
self.last_rotation_time = current_time
|
||||
|
||||
if self.current_proxy:
|
||||
logger.info(f"Proxy rotiert zu: {self.mask_proxy_credentials(self.current_proxy)}")
|
||||
|
||||
return self.current_proxy
|
||||
|
||||
def test_proxy(self, proxy_type: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Testet einen Proxy des angegebenen Typs.
|
||||
|
||||
Args:
|
||||
proxy_type: Typ des zu testenden Proxys
|
||||
|
||||
Returns:
|
||||
Dictionary mit Testergebnissen
|
||||
"""
|
||||
proxy = self.get_random_proxy(proxy_type)
|
||||
|
||||
if not proxy:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Keine Proxies vom Typ '{proxy_type}' verfügbar"
|
||||
}
|
||||
|
||||
try:
|
||||
# Proxy-URL parsen
|
||||
parts = proxy.split(":")
|
||||
|
||||
if len(parts) >= 4:
|
||||
# Format: host:port:username:password
|
||||
host, port, username, password = parts[:4]
|
||||
proxy_url = f"http://{username}:{password}@{host}:{port}"
|
||||
elif len(parts) >= 2:
|
||||
# Format: host:port
|
||||
host, port = parts[:2]
|
||||
proxy_url = f"http://{host}:{port}"
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Ungültiges Proxy-Format: {self.mask_proxy_credentials(proxy)}"
|
||||
}
|
||||
|
||||
# Proxy-Konfiguration für requests
|
||||
proxies = {
|
||||
"http": proxy_url,
|
||||
"https": proxy_url
|
||||
}
|
||||
|
||||
# Startzeit für Antwortzeit-Messung
|
||||
start_time = time.time()
|
||||
|
||||
# Test-Anfrage über den Proxy
|
||||
response = requests.get("https://api.ipify.org?format=json", proxies=proxies, timeout=10)
|
||||
|
||||
# Antwortzeit berechnen
|
||||
response_time = time.time() - start_time
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
ip = data.get("ip", "Unbekannt")
|
||||
|
||||
# Länderinformationen abrufen (optional)
|
||||
country = self.get_country_for_ip(ip)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"ip": ip,
|
||||
"country": country,
|
||||
"response_time": response_time,
|
||||
"proxy_type": proxy_type
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Ungültige Antwort: HTTP {response.status_code}"
|
||||
}
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Zeitüberschreitung bei der Verbindung"
|
||||
}
|
||||
except requests.exceptions.ProxyError:
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Proxy-Fehler: Verbindung abgelehnt oder fehlgeschlagen"
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Fehler beim Testen des Proxys: {e}")
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
def get_country_for_ip(self, ip: str) -> Optional[str]:
|
||||
"""
|
||||
Ermittelt das Land für eine IP-Adresse.
|
||||
|
||||
Args:
|
||||
ip: IP-Adresse
|
||||
|
||||
Returns:
|
||||
Ländername oder None im Fehlerfall
|
||||
"""
|
||||
try:
|
||||
response = requests.get(f"https://ipapi.co/{ip}/json/", timeout=5)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return data.get("country_name")
|
||||
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def mask_proxy_credentials(self, proxy: str) -> str:
|
||||
"""
|
||||
Maskiert die Anmeldeinformationen in einem Proxy-String für die Protokollierung.
|
||||
|
||||
Args:
|
||||
proxy: Original-Proxy-String
|
||||
|
||||
Returns:
|
||||
Maskierter Proxy-String
|
||||
"""
|
||||
parts = proxy.split(":")
|
||||
|
||||
if len(parts) >= 4:
|
||||
# Format: host:port:username:password
|
||||
host, port = parts[0], parts[1]
|
||||
return f"{host}:{port}:***:***"
|
||||
|
||||
return proxy
|
||||
|
||||
def add_proxy(self, proxy: str, proxy_type: str) -> bool:
|
||||
"""
|
||||
Fügt einen neuen Proxy zur Konfiguration hinzu.
|
||||
|
||||
Args:
|
||||
proxy: Proxy-String im Format host:port:username:password
|
||||
proxy_type: Typ des Proxys (ipv4, ipv6, mobile)
|
||||
|
||||
Returns:
|
||||
True bei Erfolg, False bei Fehler
|
||||
"""
|
||||
if proxy_type.lower() not in ["ipv4", "ipv6", "mobile"]:
|
||||
logger.warning(f"Ungültiger Proxy-Typ: {proxy_type}")
|
||||
return False
|
||||
|
||||
proxy_list = self.config.get(proxy_type.lower(), [])
|
||||
|
||||
if proxy not in proxy_list:
|
||||
proxy_list.append(proxy)
|
||||
self.config[proxy_type.lower()] = proxy_list
|
||||
self.save_config()
|
||||
|
||||
logger.info(f"Proxy hinzugefügt: {self.mask_proxy_credentials(proxy)} (Typ: {proxy_type})")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def remove_proxy(self, proxy: str, proxy_type: str) -> bool:
|
||||
"""
|
||||
Entfernt einen Proxy aus der Konfiguration.
|
||||
|
||||
Args:
|
||||
proxy: Proxy-String
|
||||
proxy_type: Typ des Proxys (ipv4, ipv6, mobile)
|
||||
|
||||
Returns:
|
||||
True bei Erfolg, False bei Fehler
|
||||
"""
|
||||
if proxy_type.lower() not in ["ipv4", "ipv6", "mobile"]:
|
||||
logger.warning(f"Ungültiger Proxy-Typ: {proxy_type}")
|
||||
return False
|
||||
|
||||
proxy_list = self.config.get(proxy_type.lower(), [])
|
||||
|
||||
if proxy in proxy_list:
|
||||
proxy_list.remove(proxy)
|
||||
self.config[proxy_type.lower()] = proxy_list
|
||||
self.save_config()
|
||||
|
||||
logger.info(f"Proxy entfernt: {self.mask_proxy_credentials(proxy)} (Typ: {proxy_type})")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def format_proxy_for_playwright(self, proxy: str) -> Dict[str, str]:
|
||||
"""
|
||||
Formatiert einen Proxy-String für die Verwendung mit Playwright.
|
||||
|
||||
Args:
|
||||
proxy: Proxy-String im Format host:port:username:password
|
||||
|
||||
Returns:
|
||||
Dictionary mit Playwright-Proxy-Konfiguration
|
||||
"""
|
||||
parts = proxy.split(":")
|
||||
|
||||
if len(parts) >= 4:
|
||||
# Format: host:port:username:password
|
||||
host, port, username, password = parts[:4]
|
||||
|
||||
return {
|
||||
"server": f"{host}:{port}",
|
||||
"username": username,
|
||||
"password": password
|
||||
}
|
||||
elif len(parts) >= 2:
|
||||
# Format: host:port
|
||||
host, port = parts[:2]
|
||||
|
||||
return {
|
||||
"server": f"{host}:{port}"
|
||||
}
|
||||
else:
|
||||
logger.warning(f"Ungültiges Proxy-Format: {self.mask_proxy_credentials(proxy)}")
|
||||
return {}
|
||||
In neuem Issue referenzieren
Einen Benutzer sperren