Phase 1 Backend-Hygiene Quellen
- src/shared/ neu: source_rules, services/source_health, services/source_suggester, agents/claude_client als lokale Kopien aus dem Monitor-Repo (statt sys.path-Hack auf /home/claude-dev/AegisSight-Monitor/src - 5 sys.path.insert-Aufrufe entfernt) - src/routers/sources.py: Imports auf shared. umgestellt, Header neu sortiert (Docstring zuerst, sys/os raus), Mojibake (Triple-Encoded UTF-8) via ftfy gefixt - src/shared/services/source_suggester.py: Mojibake (Double-Encoded UTF-8) via ftfy gefixt - migrations/2026-05-09c_source_health_schema.py NEU: source_health_checks + source_suggestions Tabellen mit Indizes (idempotent), gezogen aus 3 Inline-DDL-Blöcken in routers/sources.py (/health/run, /health/run-stream, /health/search-fix) - src/config.py: CLAUDE_MODEL_MEDIUM und CLAUDE_MODEL_STANDARD ergänzt (vorher nur CLAUDE_MODEL_FAST - claude_client.py braucht alle drei) - requirements.txt: httpx + feedparser explizit (im venv schon vorhanden, jetzt dokumentiert)
Dieser Commit ist enthalten in:
@@ -1,23 +1,16 @@
|
||||
import os
|
||||
"""Grundquellen-Verwaltung und Kundenquellen-Übersicht."""
|
||||
import sys
|
||||
"""Grundquellen-Verwaltung und Kundenquellen-Übersicht."""
|
||||
import logging
|
||||
|
||||
# Monitor-Source-Rules verfügbar machen
|
||||
sys.path.insert(0, "/home/claude-dev/AegisSight-Monitor/src")
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Optional
|
||||
import aiosqlite
|
||||
|
||||
from auth import get_current_admin
|
||||
from database import db_dependency
|
||||
from audit import log_action, get_client_ip, row_to_dict
|
||||
import aiosqlite
|
||||
|
||||
sys.path.insert(0, os.path.join('/home/claude-dev/AegisSight-Monitor/src'))
|
||||
|
||||
from source_rules import (
|
||||
from shared.source_rules import (
|
||||
discover_source,
|
||||
discover_all_feeds,
|
||||
evaluate_feeds_with_claude,
|
||||
@@ -30,6 +23,8 @@ logger = logging.getLogger("verwaltung.sources")
|
||||
|
||||
router = APIRouter(prefix="/api/sources", tags=["sources"])
|
||||
|
||||
SOURCE_UPDATE_COLUMNS = {"name", "url", "domain", "source_type", "category", "status", "notes"}
|
||||
|
||||
|
||||
class GlobalSourceCreate(BaseModel):
|
||||
name: str = Field(min_length=1, max_length=200)
|
||||
@@ -334,7 +329,7 @@ async def add_discovered_sources(
|
||||
existing_urls.add(feed["url"])
|
||||
added += 1
|
||||
|
||||
# Web-Source für die Domain anlegen wenn noch nicht vorhanden
|
||||
# Web-Source für die Domain anlegen wenn noch nicht vorhanden
|
||||
if feeds and feeds[0].get("domain"):
|
||||
domain = feeds[0]["domain"]
|
||||
cursor = await db.execute(
|
||||
@@ -362,7 +357,7 @@ async def add_discovered_sources(
|
||||
|
||||
|
||||
|
||||
# --- Health-Check & Vorschläge ---
|
||||
# --- Health-Check & Vorschläge ---
|
||||
|
||||
@router.get("/health")
|
||||
async def get_health(
|
||||
@@ -370,7 +365,7 @@ async def get_health(
|
||||
db: aiosqlite.Connection = Depends(db_dependency),
|
||||
):
|
||||
"""Health-Check-Ergebnisse abrufen."""
|
||||
# Prüfen ob Tabelle existiert
|
||||
# Prüfen ob Tabelle existiert
|
||||
cursor = await db.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='source_health_checks'"
|
||||
)
|
||||
@@ -412,7 +407,7 @@ async def get_suggestions(
|
||||
admin: dict = Depends(get_current_admin),
|
||||
db: aiosqlite.Connection = Depends(db_dependency),
|
||||
):
|
||||
"""Alle Vorschläge abrufen (pending zuerst, dann letzte 20 bearbeitete)."""
|
||||
"""Alle Vorschläge abrufen (pending zuerst, dann letzte 20 bearbeitete)."""
|
||||
cursor = await db.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='source_suggestions'"
|
||||
)
|
||||
@@ -476,7 +471,7 @@ async def update_suggestion(
|
||||
"SELECT id FROM sources WHERE url = ? AND tenant_id IS NULL", (url,)
|
||||
)
|
||||
if await cursor.fetchone():
|
||||
result_action = "übersprungen (URL bereits vorhanden)"
|
||||
result_action = "übersprungen (URL bereits vorhanden)"
|
||||
new_status = "rejected"
|
||||
else:
|
||||
await db.execute(
|
||||
@@ -486,7 +481,7 @@ async def update_suggestion(
|
||||
)
|
||||
result_action = f"Quelle '{name}' angelegt"
|
||||
else:
|
||||
result_action = "übersprungen (keine URL)"
|
||||
result_action = "übersprungen (keine URL)"
|
||||
new_status = "rejected"
|
||||
|
||||
elif stype == "deactivate_source":
|
||||
@@ -499,7 +494,7 @@ async def update_suggestion(
|
||||
source_id = suggestion["source_id"]
|
||||
if source_id:
|
||||
await db.execute("DELETE FROM sources WHERE id = ?", (source_id,))
|
||||
result_action = "Quelle gelöscht"
|
||||
result_action = "Quelle gelöscht"
|
||||
|
||||
elif stype == "fix_url":
|
||||
source_id = suggestion["source_id"]
|
||||
@@ -509,7 +504,7 @@ async def update_suggestion(
|
||||
result_action = f"URL aktualisiert"
|
||||
|
||||
# Auto-Reject: Wenn fix_url oder add_source akzeptiert wird,
|
||||
# zugehörige deactivate_source-Vorschläge automatisch ablehnen
|
||||
# zugehörige deactivate_source-Vorschläge automatisch ablehnen
|
||||
if stype in ("fix_url", "add_source") and suggestion.get("source_id"):
|
||||
await db.execute(
|
||||
"UPDATE source_suggestions SET status = 'rejected', reviewed_at = CURRENT_TIMESTAMP "
|
||||
@@ -539,36 +534,9 @@ async def run_health_check_now(
|
||||
db: aiosqlite.Connection = Depends(db_dependency),
|
||||
):
|
||||
"""Health-Check manuell starten."""
|
||||
# Tabellen sicherstellen
|
||||
await db.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS source_health_checks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
||||
check_type TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
message TEXT,
|
||||
details TEXT,
|
||||
checked_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS source_suggestions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
suggestion_type TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
source_id INTEGER REFERENCES sources(id) ON DELETE SET NULL,
|
||||
suggested_data TEXT,
|
||||
priority TEXT DEFAULT 'medium',
|
||||
status TEXT DEFAULT 'pending',
|
||||
reviewed_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
""")
|
||||
await db.commit()
|
||||
|
||||
# source_health und source_suggester importieren
|
||||
sys.path.insert(0, "/home/claude-dev/AegisSight-Monitor/src")
|
||||
from services.source_health import run_health_checks
|
||||
from services.source_suggester import generate_suggestions
|
||||
from shared.services.source_health import run_health_checks
|
||||
from shared.services.source_suggester import generate_suggestions
|
||||
|
||||
result = await run_health_checks(db)
|
||||
suggestion_count = await generate_suggestions(db)
|
||||
@@ -593,26 +561,6 @@ async def run_health_check_stream(
|
||||
import asyncio
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# Tabellen sicherstellen
|
||||
await db.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS source_health_checks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
||||
check_type TEXT NOT NULL, status TEXT NOT NULL,
|
||||
message TEXT, details TEXT,
|
||||
checked_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS source_suggestions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
suggestion_type TEXT NOT NULL, title TEXT NOT NULL,
|
||||
description TEXT, source_id INTEGER REFERENCES sources(id) ON DELETE SET NULL,
|
||||
suggested_data TEXT, priority TEXT DEFAULT 'medium',
|
||||
status TEXT DEFAULT 'pending', reviewed_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
""")
|
||||
await db.commit()
|
||||
|
||||
# Quellen laden
|
||||
cursor = await db.execute(
|
||||
"SELECT id, name, url, domain, source_type, article_count, last_seen_at "
|
||||
@@ -733,8 +681,7 @@ async def run_health_check_stream(
|
||||
# Phase 2: Vorschlaege
|
||||
yield f"data: {_json.dumps({'phase': 'suggestions', 'checked': checked, 'total': total})}\n\n"
|
||||
|
||||
sys.path.insert(0, "/home/claude-dev/AegisSight-Monitor/src")
|
||||
from services.source_suggester import generate_suggestions
|
||||
from shared.services.source_suggester import generate_suggestions
|
||||
suggestion_count = await generate_suggestions(db)
|
||||
|
||||
# Fertig
|
||||
@@ -749,7 +696,7 @@ async def search_fix_for_source(
|
||||
admin: dict = Depends(get_current_admin),
|
||||
db: aiosqlite.Connection = Depends(db_dependency),
|
||||
):
|
||||
"""Sonnet mit WebSearch nach Lösung für eine kaputte Quelle suchen lassen."""
|
||||
"""Sonnet mit WebSearch nach Lösung für eine kaputte Quelle suchen lassen."""
|
||||
import json as _json
|
||||
|
||||
cursor = await db.execute(
|
||||
@@ -762,7 +709,7 @@ async def search_fix_for_source(
|
||||
|
||||
source = dict(source)
|
||||
|
||||
# Health-Check-Probleme für diese Quelle laden
|
||||
# Health-Check-Probleme für diese Quelle laden
|
||||
cursor = await db.execute(
|
||||
"SELECT check_type, status, message FROM source_health_checks WHERE source_id = ?",
|
||||
(source_id,),
|
||||
@@ -781,14 +728,14 @@ Kategorie: {source['category']}
|
||||
Probleme:
|
||||
{issues_text}
|
||||
|
||||
Aufgabe: Suche im Internet nach funktionierenden Alternativen für diese Quelle.
|
||||
- Finde konkrete RSS-Feed-URLs die tatsächlich funktionieren
|
||||
- Prüfe ob es alternative Zugangswege gibt (andere Subdomains, Feed-Aggregatoren, alternative URLs)
|
||||
- Gibt es eine Lösung oder ist die Quelle nur noch per WebSearch erreichbar?
|
||||
Aufgabe: Suche im Internet nach funktionierenden Alternativen für diese Quelle.
|
||||
- Finde konkrete RSS-Feed-URLs die tatsächlich funktionieren
|
||||
- Prüfe ob es alternative Zugangswege gibt (andere Subdomains, Feed-Aggregatoren, alternative URLs)
|
||||
- Gibt es eine Lösung oder ist die Quelle nur noch per WebSearch erreichbar?
|
||||
|
||||
Regeln:
|
||||
- Maximal 3 Lösungen vorschlagen (die besten)
|
||||
- Verwende echte deutsche Umlaute (ü, ä, ö, ß), keine Umschreibungen (ue, ae, oe, ss)
|
||||
- Maximal 3 Lösungen vorschlagen (die besten)
|
||||
- Verwende echte deutsche Umlaute (ü, ä, ö, ß), keine Umschreibungen (ue, ae, oe, ss)
|
||||
|
||||
Antworte NUR mit einem JSON-Objekt:
|
||||
{{
|
||||
@@ -798,16 +745,15 @@ Antworte NUR mit einem JSON-Objekt:
|
||||
"type": "replace_url|add_feed|deactivate",
|
||||
"name": "Anzeigename",
|
||||
"url": "https://...",
|
||||
"description": "Kurze Begründung"
|
||||
"description": "Kurze Begründung"
|
||||
}}
|
||||
],
|
||||
"summary": "Zusammenfassung in 1-2 Sätzen"
|
||||
"summary": "Zusammenfassung in 1-2 Sätzen"
|
||||
}}
|
||||
|
||||
Nur das JSON, kein anderer Text."""
|
||||
|
||||
sys.path.insert(0, "/home/claude-dev/AegisSight-Monitor/src")
|
||||
from agents.claude_client import call_claude
|
||||
from shared.agents.claude_client import call_claude
|
||||
|
||||
try:
|
||||
response, usage = await call_claude(prompt, tools="WebSearch,WebFetch")
|
||||
@@ -819,21 +765,7 @@ Nur das JSON, kein anderer Text."""
|
||||
else:
|
||||
result = {"fixable": False, "solutions": [], "summary": response[:500]}
|
||||
|
||||
# Lösungen als Vorschläge speichern
|
||||
await db.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS source_suggestions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
suggestion_type TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
source_id INTEGER REFERENCES sources(id) ON DELETE SET NULL,
|
||||
suggested_data TEXT,
|
||||
priority TEXT DEFAULT 'medium',
|
||||
status TEXT DEFAULT 'pending',
|
||||
reviewed_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
""")
|
||||
# Lösungen als Vorschläge speichern
|
||||
|
||||
for sol in result.get("solutions", []):
|
||||
sol_type = sol.get("type", "add_feed")
|
||||
|
||||
In neuem Issue referenzieren
Einen Benutzer sperren