Phase 18 (Verwaltung): fetch_strategy in CRUD + Edit-Modal

- migrations/2026-05-09e_fetch_strategy.py NEU: ALTER TABLE sources ADD COLUMN
  fetch_strategy. Pre-flagging fuer FT/WSJ/NZZ etc. (paywall) und Rheinische
  Post/Verfassungsschutz (googlebot).
- shared/services/source_health.py: gesynct vom Monitor (Phase-18-Code mit
  Retry-Logik + Strategien default/googlebot/paywall/skip).
- routers/sources.py: GlobalSourceCreate/Update um fetch_strategy
  (Pattern-Validation), SOURCE_UPDATE_COLUMNS + INSERT erweitert.
- dashboard.html: Edit-Modal hat jetzt Dropdown sourceFetchStrategy.
- sources.js: laedt + sendet fetch_strategy mit.

Cache-Buster 20260509c -> 20260509d.

migrations/2026-05-09e_fetch_strategy.py

@@ -0,0 +1,77 @@
+"""Migration 2026-05-09e: sources.fetch_strategy.
+
+Neues Feld zur Steuerung wie der Health-Check / RSS-Parser eine Quelle abrufen soll:
+
+  default:   normaler User-Agent (AegisSight-HealthCheck), bei 403/429 Retry mit Googlebot.
+  googlebot: direkt mit Googlebot-UA (fuer Sites die SEO-freundlich sind).
+  paywall:   bei 403 zweite Anfrage via removepaywalls.com (fuer Spiegel+/SZ+/FT etc.).
+  skip:      Health-Check ueberspringen (bekannte unerreichbare Quellen).
+
+Ausfuehrung:
+    DB_PATH=/home/claude-dev/osint-data/osint.db python3 migrations/2026-05-09e_fetch_strategy.py
+    DB_PATH=/home/claude-dev/AegisSight-Monitor-staging/data/osint.db python3 migrations/2026-05-09e_fetch_strategy.py
+"""
+import os
+import sqlite3
+import sys
+
+
+def main(db_path: str) -> int:
+    if not os.path.exists(db_path):
+        print(f"FEHLER: DB nicht gefunden: {db_path}", file=sys.stderr)
+        return 1
+
+    conn = sqlite3.connect(db_path, timeout=60)
+    conn.execute("PRAGMA busy_timeout = 60000")
+    conn.execute("PRAGMA journal_mode = WAL")
+
+    print(f"Migration auf {db_path}")
+
+    cols = [c[1] for c in conn.execute("PRAGMA table_info(sources)")]
+    if "fetch_strategy" in cols:
+        print("  = sources.fetch_strategy war bereits da")
+    else:
+        conn.execute(
+            "ALTER TABLE sources ADD COLUMN fetch_strategy TEXT DEFAULT 'default'"
+        )
+        print("  + sources.fetch_strategy hinzugefügt (Default 'default')")
+
+    # Bekannte Paywall-Domains pre-flagging
+    paywall_domains = (
+        "ft.com",
+        "wsj.com",
+        "nzz.ch",
+        "handelsblatt.com",
+        "wiwo.de",
+    )
+    for dom in paywall_domains:
+        conn.execute(
+            "UPDATE sources SET fetch_strategy = 'paywall' "
+            "WHERE LOWER(domain) = ? AND COALESCE(fetch_strategy, 'default') = 'default'",
+            (dom,),
+        )
+    print("  ~ paywall-Strategie für bekannte Domains gesetzt (FT, WSJ, NZZ, Handelsblatt, WiWo)")
+
+    # Bekannte Bot-Block-Domains: Googlebot probieren
+    bot_block_domains = (
+        "rheinische-post.de",
+        "rp-online.de",
+        "verfassungsschutz.de",
+    )
+    for dom in bot_block_domains:
+        conn.execute(
+            "UPDATE sources SET fetch_strategy = 'googlebot' "
+            "WHERE LOWER(domain) LIKE ? AND COALESCE(fetch_strategy, 'default') = 'default'",
+            (f"%{dom}",),
+        )
+    print("  ~ googlebot-Strategie für bekannte Bot-Block-Domains")
+
+    conn.commit()
+    conn.close()
+    print("Migration abgeschlossen.")
+    return 0
+
+
+if __name__ == "__main__":
+    db_path = os.environ.get("DB_PATH", "/home/claude-dev/osint-data/osint.db")
+    sys.exit(main(db_path))

src/routers/sources.py

@@ -24,7 +24,7 @@ logger = logging.getLogger("verwaltung.sources")
 
 router = APIRouter(prefix="/api/sources", tags=["sources"])
 
-SOURCE_UPDATE_COLUMNS = {"name", "url", "domain", "source_type", "category", "status", "notes", "language", "bias"}
+SOURCE_UPDATE_COLUMNS = {"name", "url", "domain", "source_type", "category", "status", "notes", "language", "bias", "fetch_strategy"}
 
 
 @router.get("/meta")
@@ -48,6 +48,7 @@ class GlobalSourceCreate(BaseModel):
     notes: Optional[str] = None
     language: Optional[str] = Field(default=None, max_length=100)
     bias: Optional[str] = Field(default=None, max_length=500)
+    fetch_strategy: Optional[str] = Field(default="default", pattern="^(default|googlebot|paywall|skip)$")
 
 
 class GlobalSourceUpdate(BaseModel):
@@ -143,10 +144,10 @@ async def create_global_source(
             )
 
     cursor = await db.execute(
-        """INSERT INTO sources (name, url, domain, source_type, category, status, notes, language, bias, added_by, tenant_id)
+        """INSERT INTO sources (name, url, domain, source_type, category, status, notes, language, bias, fetch_strategy, added_by, tenant_id)
-           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'system', NULL)""",
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'system', NULL)""",
         (data.name, data.url, data.domain, data.source_type, data.category, data.status, data.notes,
-         data.language, data.bias),
+         data.language, data.bias, data.fetch_strategy or "default"),
     )
     src_id = cursor.lastrowid
     await db.commit()

src/shared/services/source_health.py

@@ -15,6 +15,17 @@ except ImportError:
     HEALTH_CHECK_USER_AGENT = "Mozilla/5.0 (compatible; AegisSight-HealthCheck/1.0)"
     HEALTH_CHECK_TIMEOUT_S = 15.0
 
+# Phase 18: alternative User-Agents fuer Bot-Block-Bypass
+USER_AGENT_GOOGLEBOT = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+USER_AGENT_BROWSER = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/120.0 Safari/537.36"
+)
+REMOVEPAYWALLS_PREFIX = "https://www.removepaywalls.com/search?url="
+
+# HTTP-Codes, die einen Retry mit anderem UA rechtfertigen
+RETRY_ON_STATUS = {403, 406, 429}
+
 logger = logging.getLogger("osint.source_health")
 
 
@@ -24,7 +35,8 @@ async def run_health_checks(db: aiosqlite.Connection) -> dict:
 
     # Alle aktiven Quellen laden (global UND Tenant-spezifisch)
     cursor = await db.execute(
-        "SELECT id, name, url, domain, source_type, article_count, last_seen_at "
+        "SELECT id, name, url, domain, source_type, article_count, last_seen_at, "
+        "COALESCE(fetch_strategy, 'default') AS fetch_strategy "
         "FROM sources WHERE status = 'active' "
     )
     sources = [dict(row) for row in await cursor.fetchall()]
@@ -108,16 +120,54 @@ async def run_health_checks(db: aiosqlite.Connection) -> dict:
 async def _check_source_reachability(
     client: httpx.AsyncClient, source: dict,
 ) -> list[dict]:
-    """Prüft Erreichbarkeit und Feed-Validität einer Quelle."""
+    """Prüft Erreichbarkeit und Feed-Validität einer Quelle.
+
+    Phase 18: pro Quelle eine fetch_strategy ('default' | 'googlebot' | 'paywall' | 'skip').
+    Bei 'default' wird im Fehlerfall (403/406/429) ein Retry mit Googlebot-UA gemacht.
+    Bei 'paywall' wird auf removepaywalls.com umgeleitet.
+    Bei 'skip' wird kein Check ausgeführt.
+    """
     checks = []
     url = source["url"]
+    strategy = source.get("fetch_strategy") or "default"
 
-    # URL-Schema sicherstellen: t.me-Kanaele und andere Domains koennen ohne https:// vorkommen
+    # 'skip' -> kein Check (bekannte unerreichbare Quellen, z.B. Login-only)
+    if strategy == "skip":
+        checks.append({
+            "type": "reachability", "status": "ok",
+            "message": "Health-Check uebersprungen (fetch_strategy=skip)",
+        })
+        return checks
+
+    # URL-Schema sicherstellen
     if url and not url.startswith(("http://", "https://")):
         url = "https://" + url.lstrip("/")
 
+    # Initialen UA waehlen: googlebot direkt; paywall ueber removepaywalls; default normal
+    initial_ua = HEALTH_CHECK_USER_AGENT
+    initial_url = url
+    if strategy == "googlebot":
+        initial_ua = USER_AGENT_GOOGLEBOT
+    elif strategy == "paywall":
+        initial_url = REMOVEPAYWALLS_PREFIX + url
+        initial_ua = USER_AGENT_BROWSER
+
     try:
-        resp = await client.get(url)
+        resp = await client.get(initial_url, headers={"User-Agent": initial_ua})
+
+        # Bot-Block-Retry nur bei strategy='default'
+        if (
+            strategy == "default"
+            and resp.status_code in RETRY_ON_STATUS
+        ):
+            retry = await client.get(url, headers={"User-Agent": USER_AGENT_GOOGLEBOT})
+            if retry.status_code < 400:
+                resp = retry  # Retry hat geholfen
+                checks.append({
+                    "type": "reachability", "status": "warning",
+                    "message": f"Erreichbar nur mit Googlebot-UA (Standard-UA bekam HTTP {initial_url and 'unknown' or 'XXX'})",
+                })
+                # Hinweis-Eintrag, aber Hauptcheck folgt unten als 'ok' weil resp jetzt die Retry-Antwort ist
 
         if resp.status_code >= 400:
             checks.append({

src/static/dashboard.html

@@ -6,7 +6,7 @@
   <title>AegisSight Monitor-Verwaltung</title>
   <link rel="icon" type="image/svg+xml" href="/static/favicon.svg">
   <link rel="apple-touch-icon" href="/static/favicon.svg">
-  <link rel="stylesheet" href="/static/css/style.css?v=20260509c">
+  <link rel="stylesheet" href="/static/css/style.css?v=20260509d">
 
     <style>
       .source-badge { display:inline-block; padding:2px 8px; border-radius:4px; font-size:12px; font-weight:600; }
@@ -625,6 +625,15 @@
             <label for="sourceBias">Bias / Einordnung</label>
             <input type="text" id="sourceBias" placeholder="z.B. Nachrichtenagentur, faktenbasiert-neutral" maxlength="500">
           </div>
+          <div class="form-group">
+            <label for="sourceFetchStrategy">Fetch-Strategie (Health-Check)</label>
+            <select id="sourceFetchStrategy">
+              <option value="default">Standard (UA + Retry mit Googlebot bei 403)</option>
+              <option value="googlebot">Googlebot (direkt - fuer SEO-freundliche Sites)</option>
+              <option value="paywall">Paywall (via removepaywalls.com - z.B. FT, Spiegel+)</option>
+              <option value="skip">Skip (Health-Check ueberspringen)</option>
+            </select>
+          </div>
           <div class="form-group">
             <label for="sourceNotes">Notizen</label>
             <input type="text" id="sourceNotes" placeholder="Optional">
@@ -697,10 +706,10 @@
     </div>
   </div>
 
-  <script src="/static/js/app.js?v=20260509c"></script>
+  <script src="/static/js/app.js?v=20260509d"></script>
-  <script src="/static/js/sources.js?v=20260509c"></script>
+  <script src="/static/js/sources.js?v=20260509d"></script>
-  <script src="/static/js/source-health.js?v=20260509c"></script>
+  <script src="/static/js/source-health.js?v=20260509d"></script>
-  <script src="/static/js/audit.js?v=20260509c"></script>
+  <script src="/static/js/audit.js?v=20260509d"></script>
   <div id="toastContainer" class="toast-container" aria-live="polite" aria-atomic="true"></div>
 </body>
 </html>

src/static/index.html

@@ -7,7 +7,7 @@
   <title>AegisSight Monitor-Verwaltung - Anmeldung</title>
   <link rel="icon" type="image/svg+xml" href="/static/favicon.svg">
   <link rel="apple-touch-icon" href="/static/favicon.svg">
-  <link rel="stylesheet" href="/static/css/style.css?v=20260509c">
+  <link rel="stylesheet" href="/static/css/style.css?v=20260509d">
 </head>
 <body class="login-page">
   <div class="login-container">

src/static/js/sources.js

@@ -297,6 +297,7 @@ function editGlobalSource(id) {
   document.getElementById("sourceNotes").value = s.notes || "";
   document.getElementById("sourceLanguage").value = s.language || "";
   document.getElementById("sourceBias").value = s.bias || "";
+  document.getElementById("sourceFetchStrategy").value = s.fetch_strategy || "default";
   openModal("modalSource");
 }
 
@@ -324,6 +325,7 @@ function setupSourceForms() {
       notes: document.getElementById("sourceNotes").value || null,
       language: document.getElementById("sourceLanguage").value || null,
       bias: document.getElementById("sourceBias").value || null,
+      fetch_strategy: document.getElementById("sourceFetchStrategy").value || "default",
     };
 
     try {