fix(sources): Meta-Reparatur (telegram/x/x_account/pdf_document) + toten Code entfernt

- source_meta.py kennt jetzt den echten Bestand: Kategorien telegram (107 Quellen)
  und x (40), Typen x_account und pdf_document. Die 10 nie belegten
  Lagen-Kategorien und der ungenutzte Typ excluded sind raus.
- Pydantic-Pattern source_type um x_account ergaenzt, excluded raus.
  Behebt 422 beim Speichern von 40 X-Account-Quellen.
- Modal-Selects (Typ, Kategorie, PDF-Kategorie) werden aus /api/sources/meta
  befuellt statt hartkodiert. Submit sendet category/source_type nur noch,
  wenn nicht leer. Behebt stilles Loeschen der Kategorie bei Telegram-Quellen.
- source_suggester: Import auf shared.agents.claude_client angepasst
  (war im Portal seit jeher kaputt, Konvention wie source_classifier).
- Toter Code raus: POST /health/run-stream (147 Z. SSE-Duplikat),
  runHealthCheck() im Frontend, excluded_counts-CTE + Sperren-Spalte
  (immer 0), Trend-Delta (braucht 2 Runs, es gibt 1), Alignment-Chips
  (source_alignments ist leer), doppeltes formatDateTime, redundanter
  setupHealthTab-Listener, toter healthContent-Zugriff.
- Tests: Kategorien-/Typen-Sets neu gepinnt, Smoke-Karteileichen entfernt.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Dieser Commit ist enthalten in:
claude-dev
2026-07-25 17:58:08 +00:00
Ursprung 68fc1794a9
Commit 1a08facb64
10 geänderte Dateien mit 93 neuen und 458 gelöschten Zeilen

Datei anzeigen

@@ -7,7 +7,6 @@ import re
import uuid
from fastapi import APIRouter, BackgroundTasks, Depends, File, Form, HTTPException, Request, UploadFile, status
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from typing import Optional
import aiosqlite
@@ -16,7 +15,7 @@ from auth import get_current_admin
from database import db_dependency, get_db
from audit import log_action, get_client_ip
from source_meta import get_meta
from config import HEALTH_CHECK_USER_AGENT, HEALTH_CHECK_TIMEOUT_S, DB_PATH
from config import DB_PATH
from shared.source_rules import (
discover_source,
discover_all_feeds,
@@ -118,7 +117,7 @@ class GlobalSourceCreate(BaseModel):
name: str = Field(min_length=1, max_length=200)
url: Optional[str] = None
domain: Optional[str] = None
source_type: str = Field(default="rss_feed", pattern="^(rss_feed|web_source|excluded|telegram_channel|podcast_feed|pdf_document)$")
source_type: str = Field(default="rss_feed", pattern="^(rss_feed|web_source|telegram_channel|podcast_feed|pdf_document|x_account)$")
category: str = Field(default="sonstige")
status: str = Field(default="active", pattern="^(active|inactive)$")
notes: Optional[str] = None
@@ -131,7 +130,7 @@ class GlobalSourceUpdate(BaseModel):
name: Optional[str] = Field(default=None, max_length=200)
url: Optional[str] = None
domain: Optional[str] = None
source_type: Optional[str] = Field(default=None, pattern="^(rss_feed|web_source|excluded|telegram_channel|podcast_feed|pdf_document)$")
source_type: Optional[str] = Field(default=None, pattern="^(rss_feed|web_source|telegram_channel|podcast_feed|pdf_document|x_account)$")
category: Optional[str] = None
status: Optional[str] = Field(default=None, pattern="^(active|inactive)$")
notes: Optional[str] = None
@@ -168,14 +167,6 @@ async def list_global_sources(
AND source IS NOT NULL
GROUP BY LOWER(source)
),
excluded_counts AS (
SELECT LOWER(ued.domain) AS dom,
COUNT(DISTINCT u.organization_id) AS cnt
FROM user_excluded_domains ued
JOIN users u ON u.id = ued.user_id
WHERE ued.domain IS NOT NULL
GROUP BY LOWER(ued.domain)
),
health_agg AS (
SELECT source_id,
MAX(CASE WHEN status = 'error' THEN 3
@@ -193,11 +184,9 @@ async def list_global_sources(
ELSE NULL
END AS health_status,
COALESCE(ast.a7d, 0) AS articles_7d,
COALESCE(ast.a30d, 0) AS articles_30d,
COALESCE(ec.cnt, 0) AS tenant_excluded_count
COALESCE(ast.a30d, 0) AS articles_30d
FROM sources s
LEFT JOIN article_stats ast ON ast.s_lower = LOWER(s.name)
LEFT JOIN excluded_counts ec ON ec.dom = LOWER(s.domain)
LEFT JOIN health_agg ha ON ha.source_id = s.id
WHERE s.tenant_id IS NULL
ORDER BY s.category, s.source_type, s.name
@@ -936,154 +925,6 @@ async def update_suggestion(
return {"status": new_status, "action": result_action}
@router.post("/health/run-stream")
async def run_health_check_stream(
admin: dict = Depends(get_current_admin),
db: aiosqlite.Connection = Depends(db_dependency),
):
"""Health-Check mit Fortschrittsanzeige (SSE-Stream)."""
import json as _json
import asyncio
# Quellen laden
cursor = await db.execute(
"SELECT id, name, url, domain, source_type, article_count, last_seen_at "
"FROM sources WHERE status = 'active'" # tenant + global
)
sources = [dict(row) for row in await cursor.fetchall()]
sources_with_url = [s for s in sources if s["url"]]
total = len(sources_with_url)
async def generate():
import httpx
import feedparser
# Phase 1: Erreichbarkeit
yield f"data: {_json.dumps({'phase': 'check', 'checked': 0, 'total': total, 'current': ''})}\n\n"
# Bisherigen Stand archivieren, dann frisch
run_id = uuid.uuid4().hex[:12]
await db.execute(
"INSERT INTO source_health_history "
"(run_id, source_id, check_type, status, message, details, checked_at) "
"SELECT ?, source_id, check_type, status, message, details, checked_at "
"FROM source_health_checks",
(run_id,),
)
await db.execute("DELETE FROM source_health_checks")
await db.commit()
issues_found = 0
checked = 0
async with httpx.AsyncClient(
timeout=HEALTH_CHECK_TIMEOUT_S, follow_redirects=True,
headers={"User-Agent": HEALTH_CHECK_USER_AGENT},
) as client:
for source in sources_with_url:
try:
checks = []
try:
resp = await client.get(source["url"])
if resp.status_code >= 400:
checks.append({"type": "reachability", "status": "error",
"message": f"HTTP {resp.status_code} - nicht erreichbar"})
else:
checks.append({"type": "reachability", "status": "ok", "message": "Erreichbar"})
if source["source_type"] in ("rss_feed", "podcast_feed"):
text = resp.text[:20000]
if "<rss" not in text and "<feed" not in text and "<channel" not in text:
checks.append({"type": "feed_validity", "status": "error",
"message": "Kein RSS/Atom-Feed"})
else:
feed = await asyncio.to_thread(feedparser.parse, text)
if feed.get("bozo") and not feed.entries:
checks.append({"type": "feed_validity", "status": "error",
"message": "Feed fehlerhaft"})
elif not feed.entries:
checks.append({"type": "feed_validity", "status": "warning",
"message": "Feed leer"})
else:
checks.append({"type": "feed_validity", "status": "ok",
"message": f"Feed OK ({len(feed.entries)} Eintr.)"})
except httpx.TimeoutException:
checks.append({"type": "reachability", "status": "error", "message": "Timeout (15s)"})
except httpx.ConnectError:
checks.append({"type": "reachability", "status": "error", "message": "Verbindung fehlgeschlagen"})
except Exception as e:
checks.append({"type": "reachability", "status": "error", "message": f"{type(e).__name__}"})
for c in checks:
await db.execute(
"INSERT INTO source_health_checks (source_id, check_type, status, message) VALUES (?, ?, ?, ?)",
(source["id"], c["type"], c["status"], c["message"]))
if c["status"] != "ok":
issues_found += 1
except Exception:
pass
checked += 1
status_icon = "ok"
if any(c["status"] == "error" for c in checks):
status_icon = "error"
elif any(c["status"] == "warning" for c in checks):
status_icon = "warning"
yield f"data: {_json.dumps({'phase': 'check', 'checked': checked, 'total': total, 'current': source['name'], 'status': status_icon})}\n\n"
# Stale + Duplikate (schnell, kein Fortschritt noetig)
for source in sources:
if source["source_type"] in ("excluded", "web_source"):
continue
article_count = source.get("article_count") or 0
if article_count == 0:
await db.execute(
"INSERT INTO source_health_checks (source_id, check_type, status, message) VALUES (?, 'stale', 'warning', 'Noch nie Artikel geliefert')",
(source["id"],))
issues_found += 1
elif source.get("last_seen_at"):
try:
from datetime import datetime
last_dt = datetime.fromisoformat(source["last_seen_at"])
age = (datetime.now() - last_dt).days
if age > 30:
await db.execute(
"INSERT INTO source_health_checks (source_id, check_type, status, message) VALUES (?, 'stale', 'warning', ?)",
(source["id"], f"Letzter Artikel vor {age} Tagen"))
issues_found += 1
except (ValueError, TypeError):
pass
# Duplikate
url_map = {}
for s in sources:
if not s["url"]:
continue
url_norm = s["url"].lower().rstrip("/")
if url_norm in url_map:
existing = url_map[url_norm]
await db.execute(
"INSERT INTO source_health_checks (source_id, check_type, status, message) VALUES (?, 'duplicate', 'warning', ?)",
(s["id"], f"Doppelte URL wie '{existing['name']}' (ID {existing['id']})"))
issues_found += 1
else:
url_map[url_norm] = s
await db.commit()
# Phase 2: Vorschlaege
yield f"data: {_json.dumps({'phase': 'suggestions', 'checked': checked, 'total': total})}\n\n"
from shared.services.source_suggester import generate_suggestions
suggestion_count = await generate_suggestions(db)
# Fertig
yield f"data: {_json.dumps({'phase': 'done', 'checked': checked, 'total': total, 'issues': issues_found, 'suggestions': suggestion_count})}\n\n"
return StreamingResponse(generate(), media_type="text/event-stream")
@router.post("/health/search-fix/{source_id}")
async def search_fix_for_source(
source_id: int,

Datei anzeigen

@@ -5,7 +5,7 @@ import re
import aiosqlite
from agents.claude_client import call_claude
from shared.agents.claude_client import call_claude
from config import CLAUDE_MODEL_FAST
logger = logging.getLogger("osint.source_suggester")

Datei anzeigen

@@ -27,17 +27,9 @@ SOURCE_CATEGORIES: list[CategoryEntry] = [
{"key": "international", "label": "International"},
{"key": "regional", "label": "Regional"},
{"key": "boulevard", "label": "Boulevard"},
{"key": "stimmungsbild", "label": "Forum / Stimmungsbild"},
{"key": "telegram", "label": "Telegram"},
{"key": "x", "label": "X (Twitter)"},
{"key": "sonstige", "label": "Sonstige"},
{"key": "cybercrime", "label": "Cybercrime / Hacktivismus"},
{"key": "cybercrime-leaks", "label": "Cybercrime / Leaks"},
{"key": "ukraine-russland-krieg", "label": "Ukraine-Russland-Krieg"},
{"key": "irankonflikt", "label": "Irankonflikt"},
{"key": "osint-international", "label": "OSINT International"},
{"key": "extremismus-deutschland", "label": "Extremismus Deutschland"},
{"key": "russische-staatspropaganda", "label": "Russische Staatspropaganda"},
{"key": "russische-opposition", "label": "Russische Opposition / Exilmedien"},
{"key": "syrien-nahost", "label": "Syrien / Nahost"},
]
@@ -46,7 +38,8 @@ SOURCE_TYPES: list[TypeEntry] = [
{"key": "web_source", "label": "Webquelle"},
{"key": "telegram_channel", "label": "Telegram-Kanal"},
{"key": "podcast_feed", "label": "Podcast-Feed"},
{"key": "excluded", "label": "Ausgeschlossen"},
{"key": "x_account", "label": "X-Konto"},
{"key": "pdf_document", "label": "PDF-Dokument"},
]

Datei anzeigen

@@ -995,20 +995,6 @@ input[type="date"].filter-select { padding: 6px 10px; }
.activity-cell.activity-zero {
color: #475569;
}
.exclude-badge {
display: inline-block;
padding: 2px 8px;
border-radius: 10px;
font-size: 11px;
font-weight: 600;
background: rgba(239, 68, 68, 0.15);
color: #ef4444;
}
.exclude-badge.exclude-zero {
background: transparent;
color: #475569;
font-weight: 400;
}
/* === Klassifikations-Review === */
.sources-tab-badge {
@@ -1148,26 +1134,6 @@ input[type="date"].filter-select { padding: 6px 10px; }
letter-spacing: 0.3px;
text-transform: uppercase;
}
.alignment-chips { display: flex; flex-wrap: wrap; gap: 6px; }
.alignment-chip {
display: inline-flex;
align-items: center;
padding: 4px 10px;
border-radius: 999px;
font-size: 11px;
font-weight: 500;
background: transparent;
color: var(--text-secondary);
border: 1px solid var(--border);
cursor: pointer;
transition: all 0.12s ease;
}
.alignment-chip:hover { background: var(--bg-tertiary); color: var(--text-primary); }
.alignment-chip.active {
background: var(--accent);
color: var(--bg-primary);
border-color: var(--accent);
}
/* === Theme-Umschalter (goldener Schiebeschalter, wie im Monitor) === */

Datei anzeigen

@@ -6,7 +6,7 @@
<title>AegisSight Monitor-Verwaltung</title>
<link rel="icon" type="image/svg+xml" href="/static/favicon.svg">
<link rel="apple-touch-icon" href="/static/favicon.svg">
<link rel="stylesheet" href="/static/css/style.css?v=20260725g">
<link rel="stylesheet" href="/static/css/style.css?v=20260725h">
<script>(function(){var t=localStorage.getItem('portal_theme');if(t==='light')document.documentElement.setAttribute('data-theme','light');try{var a=JSON.parse(localStorage.getItem('osint_a11y')||'{}');Object.keys(a).forEach(function(k){if(a[k])document.documentElement.setAttribute('data-a11y-'+k,'true');});}catch(e){}})()</script>
<style>
@@ -329,7 +329,6 @@
<th class="sortable" data-sort="source_type" onclick="sortGlobalSources('source_type')">Typ <span class="sort-icon"></span></th>
<th class="sortable" data-sort="article_count" onclick="sortGlobalSources('article_count')">Artikel <span class="sort-icon"></span></th>
<th class="sortable" data-sort="articles_30d" onclick="sortGlobalSources('articles_30d')">Aktivität <span class="sort-icon"></span></th>
<th class="sortable" data-sort="tenant_excluded_count" onclick="sortGlobalSources('tenant_excluded_count')">Sperren <span class="sort-icon"></span></th>
<th class="sortable" data-sort="language" onclick="sortGlobalSources('language')">Sprache <span class="sort-icon"></span></th>
<th>Bias</th>
<th class="sortable" data-sort="last_seen_at" onclick="sortGlobalSources('last_seen_at')">Letzter Treffer <span class="sort-icon"></span></th>
@@ -760,38 +759,11 @@
<div style="display:grid;grid-template-columns:1fr 1fr;gap:12px;">
<div class="form-group">
<label for="sourceType">Typ</label>
<select id="sourceType">
<option value="rss_feed">RSS-Feed</option>
<option value="web_source">Webquelle</option>
<option value="telegram_channel">Telegram-Kanal</option>
<option value="podcast_feed">Podcast-Feed</option>
<option value="excluded">Ausgeschlossen</option>
<option value="pdf_document" disabled>PDF-Dokument (nur Upload)</option>
</select>
<select id="sourceType"></select>
</div>
<div class="form-group">
<label for="sourceCategory">Kategorie</label>
<select id="sourceCategory">
<option value="nachrichtenagentur">Nachrichtenagentur</option>
<option value="oeffentlich-rechtlich">Öffentlich-Rechtlich</option>
<option value="qualitaetszeitung">Qualitätszeitung</option>
<option value="behoerde">Behörde</option>
<option value="fachmedien">Fachmedien</option>
<option value="think-tank">Think-Tank</option>
<option value="international">International</option>
<option value="regional">Regional</option>
<option value="boulevard">Boulevard</option>
<option value="sonstige" selected>Sonstige</option>
<option value="cybercrime">Cybercrime / Hacktivismus</option>
<option value="cybercrime-leaks">Cybercrime / Leaks</option>
<option value="ukraine-russland-krieg">Ukraine-Russland-Krieg</option>
<option value="irankonflikt">Irankonflikt</option>
<option value="osint-international">OSINT International</option>
<option value="extremismus-deutschland">Extremismus Deutschland</option>
<option value="russische-staatspropaganda">Russische Staatspropaganda</option>
<option value="russische-opposition">Russische Opposition / Exilmedien</option>
<option value="syrien-nahost">Syrien / Nahost</option>
</select>
<select id="sourceCategory"></select>
</div>
</div>
<div style="display:grid;grid-template-columns:1fr 1fr;gap:12px;">
@@ -896,23 +868,6 @@
</label>
</div>
</div>
<div class="form-group" style="margin-top:8px;">
<label>Geopolitische Nähe (Mehrfachauswahl)</label>
<div id="sourceAlignmentChips" class="alignment-chips" onclick="handleAlignmentChipClick(event)">
<button type="button" class="alignment-chip" data-alignment="prorussisch">prorussisch</button>
<button type="button" class="alignment-chip" data-alignment="proiranisch">proiranisch</button>
<button type="button" class="alignment-chip" data-alignment="prowestlich">prowestlich</button>
<button type="button" class="alignment-chip" data-alignment="proukrainisch">proukrainisch</button>
<button type="button" class="alignment-chip" data-alignment="prochinesisch">prochinesisch</button>
<button type="button" class="alignment-chip" data-alignment="projapanisch">projapanisch</button>
<button type="button" class="alignment-chip" data-alignment="proisraelisch">proisraelisch</button>
<button type="button" class="alignment-chip" data-alignment="propalaestinensisch">propalästinensisch</button>
<button type="button" class="alignment-chip" data-alignment="protuerkisch">protürkisch</button>
<button type="button" class="alignment-chip" data-alignment="panarabisch">panarabisch</button>
<button type="button" class="alignment-chip" data-alignment="neutral">neutral</button>
<button type="button" class="alignment-chip" data-alignment="sonstige">sonstige</button>
</div>
</div>
</div>
<div id="sourceError" class="error-msg" style="display:none"></div>
@@ -950,13 +905,7 @@
<div style="display:grid;grid-template-columns:1fr 1fr;gap:12px;">
<div class="form-group">
<label for="pdfCategory">Kategorie</label>
<select id="pdfCategory">
<option value="sonstige" selected>Sonstige</option>
<option value="behoerde">Behörde</option>
<option value="think-tank">Think-Tank</option>
<option value="fachmedien">Fachmedien</option>
<option value="international">International</option>
</select>
<select id="pdfCategory"></select>
</div>
<div class="form-group">
<label for="pdfLanguage">Sprache (optional)</label>
@@ -1106,9 +1055,9 @@
<script src="/static/js/a11y.js?v=20260725a"></script>
<script src="/static/js/app.js?v=20260725r"></script>
<script src="/static/js/sources.js?v=20260522x2"></script>
<script src="/static/js/sources.js?v=20260725a"></script>
<script src="/static/js/x-scraper.js?v=20260522a"></script>
<script src="/static/js/source-health.js?v=20260509l"></script>
<script src="/static/js/source-health.js?v=20260725a"></script>
<script src="/static/js/audit.js?v=20260509d"></script>
<div id="toastContainer" class="toast-container" aria-live="polite" aria-atomic="true"></div>
</body>

Datei anzeigen

@@ -49,13 +49,6 @@ const LUCIDE_ICONS = {
};
// --- Init ---
function setupHealthTab() {
const tab = document.querySelector('#sourceSubTabs .nav-tab[data-subtab="source-health"]');
if (tab) {
tab.addEventListener("click", () => loadHealthData());
}
}
// Sub-Sub-Tabs innerhalb von Quellen-Health: Vorschläge / Health-Status / Verlauf.
function setupHealthSubTabs() {
document.querySelectorAll("#healthSubTabs .nav-tab").forEach((tab) => {
@@ -72,7 +65,6 @@ function setupHealthSubTabs() {
}
document.addEventListener("DOMContentLoaded", () => {
setupHealthTab();
setupHealthSubTabs();
});
@@ -102,8 +94,6 @@ async function loadHealthData(force = false) {
renderHealthDashboard();
} catch (err) {
console.error("Health-Daten laden fehlgeschlagen:", err);
document.getElementById("healthContent").innerHTML =
'<div class="text-muted" style="padding:20px;">Fehler beim Laden der Health-Daten.</div>';
}
}
@@ -260,23 +250,6 @@ function renderHealthDashboard() {
return `<span class="${cssClass}" title="${esc(detail)}">${total} ${label}</span> <span class="text-secondary" style="font-size:11px;">(${esc(detail)})</span>`;
}
// Trend-Delta zum vorletzten Run (healthHistoryCache[1]). Index 0 ist
// typischerweise der aktuelle Stand, Index 1 der davor archivierte Run.
// Wenn weniger als 2 Runs in der History: kein Delta anzeigen.
const prevRun = (healthHistoryCache && healthHistoryCache.length > 1) ? healthHistoryCache[1] : null;
function deltaBadge(currentValue, prevValue, badIsUp) {
if (prevValue == null) return "";
const d = currentValue - prevValue;
if (d === 0) return ` <span class="text-secondary" style="font-size:11px;" title="unverändert seit letztem Run">(±0)</span>`;
const sign = d > 0 ? "+" : "";
// badIsUp=true: Anstieg = schlecht (rot), Abnahme = gut (grün). Umgekehrt für OK.
const cls = (badIsUp ? (d > 0) : (d < 0)) ? "text-danger" : "text-success";
return ` <span class="${cls}" style="font-size:11px;" title="seit letztem Run">(${sign}${d})</span>`;
}
const dErr = prevRun ? deltaBadge(healthData.errors, prevRun.errors, true) : "";
const dWarn = prevRun ? deltaBadge(healthData.warnings, prevRun.warnings, true) : "";
const dOk = prevRun ? deltaBadge(okCount, prevRun.ok, false) : "";
healthHtml = `
<div class="card">
<div class="card-header">
@@ -284,9 +257,9 @@ function renderHealthDashboard() {
<span class="text-secondary" style="font-size:13px;">
Letzter Check: ${healthData.last_check ? formatDateTime(healthData.last_check) : "Noch nie"}
&nbsp;|&nbsp;
${breakdownLine("error", "text-danger") || `<span class="text-danger">0 Fehler</span>`}${dErr} &nbsp;
${breakdownLine("warning", "text-warning") || `<span class="text-warning">0 Warnungen</span>`}${dWarn} &nbsp;
<span class="text-success">${okCount} OK</span>${dOk}
${breakdownLine("error", "text-danger") || `<span class="text-danger">0 Fehler</span>`} &nbsp;
${breakdownLine("warning", "text-warning") || `<span class="text-warning">0 Warnungen</span>`} &nbsp;
<span class="text-success">${okCount} OK</span>
</span>
</div>
<div class="action-bar" style="border-bottom:1px solid var(--border, rgba(255,255,255,0.08));">
@@ -456,107 +429,6 @@ async function handleSuggestion(id, accept) {
}
}
// --- Health-Check manuell starten ---
async function runHealthCheck() {
const btn = document.getElementById("runHealthCheckBtn");
if (!btn) return;
btn.disabled = true;
// Fortschrittsanzeige erstellen
let progressEl = document.getElementById("healthProgress");
if (!progressEl) {
progressEl = document.createElement("div");
progressEl.id = "healthProgress";
progressEl.style.cssText = "display:flex;align-items:center;gap:12px;padding:12px 16px;background:var(--bg-secondary);border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px;font-size:13px;";
btn.parentElement.after(progressEl);
}
progressEl.style.display = "flex";
function updateProgress(data) {
if (data.phase === "check") {
const pct = data.total > 0 ? Math.round((data.checked / data.total) * 100) : 0;
const statusIcon = data.status === "error" ? "\u2717" : data.status === "warning" ? "\u26A0" : "\u2713";
btn.textContent = data.checked + "/" + data.total;
progressEl.innerHTML =
'<div style="flex:1;">' +
'<div style="display:flex;justify-content:space-between;margin-bottom:4px;">' +
'<span>' + (data.current ? statusIcon + " " + esc(data.current) : "Starte...") + '</span>' +
'<span class="text-secondary">' + pct + '%</span>' +
'</div>' +
'<div style="height:4px;background:var(--bg-tertiary);border-radius:2px;overflow:hidden;">' +
'<div style="height:100%;width:' + pct + '%;background:var(--accent);transition:width 0.3s;"></div>' +
'</div>' +
'</div>';
} else if (data.phase === "suggestions") {
progressEl.innerHTML = '<span class="text-secondary">Generiere Vorschl\u00e4ge...</span>';
} else if (data.phase === "done") {
progressEl.innerHTML = '<span class="text-success">' + data.checked + ' gepr\u00fcft, ' + data.issues + ' Probleme, ' + data.suggestions + ' Vorschl\u00e4ge</span>';
setTimeout(function() { progressEl.style.display = "none"; }, 5000);
}
}
try {
const headers = { "Content-Type": "application/json" };
if (API.token) headers["Authorization"] = "Bearer " + API.token;
const response = await fetch("/api/sources/health/run-stream", {
method: "POST",
headers: headers,
});
if (!response.ok) {
throw new Error("HTTP " + response.status);
}
const reader = response.body.getReader();
const decoder = new TextDecoder();
let buffer = "";
while (true) {
const { done, value } = await reader.read();
if (done) break;
buffer += decoder.decode(value, { stream: true });
const lines = buffer.split("\n");
buffer = lines.pop() || "";
for (const line of lines) {
if (line.startsWith("data: ")) {
try {
const data = JSON.parse(line.slice(6));
updateProgress(data);
} catch (_) {}
}
}
}
loadHealthData(true);
} catch (err) {
progressEl.innerHTML = '<span class="text-danger">Fehler: ' + esc(err.message) + '</span>';
} finally {
btn.disabled = false;
btn.textContent = "Jetzt pr\u00fcfen";
}
}
// --- Hilfsfunktionen ---
function formatDateTime(dateStr) {
if (!dateStr) return "-";
try {
const d = new Date(dateStr);
return d.toLocaleDateString("de-DE", {
day: "2-digit",
month: "2-digit",
year: "numeric",
hour: "2-digit",
minute: "2-digit",
});
} catch (_) {
return dateStr;
}
}
// --- Sonnet-Recherche für kaputte Quelle ---
async function searchFix(btn) {
const sourceId = btn.dataset.sourceId;

Datei anzeigen

@@ -14,6 +14,34 @@ let globalSortAsc = true;
// CATEGORY_LABELS jetzt global (aus app.js loadMeta)
// TYPE_LABELS jetzt global (aus app.js loadMeta)
// META muss geladen sein, bevor Dropdowns befüllt werden (loadMeta läuft async beim Start)
async function ensureMeta() {
if (!window.META || !window.META.categories || !window.META.categories.length) {
if (typeof loadMeta === "function") await loadMeta();
}
}
// Modal-Selects ohne Leer-Option aus META befüllen
function fillSelect(el, items, opts = {}) {
if (!el) return;
el.innerHTML = "";
(items || []).forEach((it) => {
const o = document.createElement("option");
o.value = it.key;
o.textContent = it.label + (opts.disabledKeys && opts.disabledKeys.includes(it.key) && opts.disabledHint ? " " + opts.disabledHint : "");
if (opts.disabledKeys && opts.disabledKeys.includes(it.key)) o.disabled = true;
el.appendChild(o);
});
if (opts.value !== undefined) el.value = opts.value;
}
function fillSourceModalSelects() {
fillSelect(document.getElementById("sourceType"), (window.META && window.META.types) || [],
{ disabledKeys: ["pdf_document"], disabledHint: "(nur Upload)" });
fillSelect(document.getElementById("sourceCategory"), (window.META && window.META.categories) || []);
}
// --- Init ---
document.addEventListener("DOMContentLoaded", () => {
setupSourceSubTabs();
@@ -46,11 +74,11 @@ function setupSourceSubTabs() {
// --- Grundquellen ---
async function loadGlobalSources() {
try {
await ensureMeta();
// Kategorien/Typen-Dropdowns aus META befüllen (idempotent)
if (window.META && window.META.categories && window.META.categories.length) {
populateSelect(document.getElementById("globalFilterCategory"), window.META.categories, "Alle Kategorien");
populateSelect(document.getElementById("globalFilterType"),
(window.META.types || []).filter(t => t.key !== "excluded"), "Alle Typen");
populateSelect(document.getElementById("globalFilterType"), window.META.types || [], "Alle Typen");
}
const [list, stats, languages] = await Promise.all([
API.get("/api/sources/global"),
@@ -145,7 +173,6 @@ function renderGlobalStats(stats) {
const parts = [];
parts.push(`<span class="sources-stat-item"><span class="sources-stat-value">${stats.total || 0}</span> Quellen gesamt</span>`);
for (const t of types) {
if (t.key === "excluded") continue;
const v = stats.by_type[t.key] || { count: 0, articles: 0 };
parts.push(`<span class="sources-stat-item"><span class="sources-stat-value">${v.count}</span> ${esc(t.label)}</span>`);
}
@@ -161,7 +188,7 @@ function renderGlobalStats(stats) {
function renderGlobalSources(sources) {
const tbody = document.getElementById("globalSourceTable");
const cols = 13;
const cols = 12;
if (sources.length === 0) {
tbody.innerHTML = `<tr><td colspan="${cols}" class="text-muted">Keine Grundquellen</td></tr>`;
return;
@@ -200,7 +227,6 @@ function renderGlobalSources(sources) {
<td>${typeLabel(s.source_type)}</td>
<td class="text-right">${s.article_count || 0}</td>
<td class="${(s.articles_30d || 0) === 0 ? "activity-cell activity-zero" : "activity-cell"}" title="7 Tage / 30 Tage"><strong>${s.articles_7d || 0}</strong> / ${s.articles_30d || 0}</td>
<td class="text-right"><span class="${(s.tenant_excluded_count || 0) === 0 ? "exclude-badge exclude-zero" : "exclude-badge"}">${s.tenant_excluded_count || 0}</span></td>
<td class="text-secondary">${esc(s.language || "-")}</td>
<td class="text-secondary" style="max-width:200px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="${esc(s.bias || "")}">${esc(s.bias || "-")}</td>
<td class="text-secondary">${lastSeen}</td>
@@ -252,7 +278,7 @@ function filterGlobalSources() {
filtered.sort((a, b) => {
let va = a[globalSortField] ?? "";
let vb = b[globalSortField] ?? "";
const NUMERIC_FIELDS = ["article_count", "articles_7d", "articles_30d", "tenant_excluded_count"];
const NUMERIC_FIELDS = ["article_count", "articles_7d", "articles_30d"];
if (NUMERIC_FIELDS.includes(globalSortField)) {
va = parseInt(va) || 0;
vb = parseInt(vb) || 0;
@@ -278,18 +304,23 @@ function sortGlobalSources(field) {
}
// --- Grundquelle erstellen/bearbeiten ---
function openNewGlobalSource() {
async function openNewGlobalSource() {
editingSourceId = null;
document.getElementById("sourceModalTitle").textContent = "Neue Grundquelle";
document.getElementById("sourceForm").reset();
setAlignmentChips([]);
await ensureMeta();
fillSourceModalSelects();
document.getElementById("sourceType").value = "rss_feed";
document.getElementById("sourceCategory").value = "sonstige";
openModal("modalSource");
}
function editGlobalSource(id) {
async function editGlobalSource(id) {
const s = globalSourcesCache.find((x) => x.id === id);
if (!s) return;
editingSourceId = id;
await ensureMeta();
fillSourceModalSelects();
document.getElementById("sourceModalTitle").textContent = "Grundquelle bearbeiten";
document.getElementById("sourceName").value = s.name;
document.getElementById("sourceUrl").value = s.url || "";
@@ -306,7 +337,6 @@ function editGlobalSource(id) {
document.getElementById("sourceReliability").value = s.reliability || "";
document.getElementById("sourceCountryCode").value = s.country_code || "";
document.getElementById("sourceStateAffiliated").checked = !!s.state_affiliated;
setAlignmentChips(s.alignments || []);
openModal("modalSource");
}
@@ -330,14 +360,18 @@ function setupSourceForms() {
name: document.getElementById("sourceName").value,
url: document.getElementById("sourceUrl").value || null,
domain: document.getElementById("sourceDomain").value || null,
source_type: document.getElementById("sourceType").value,
category: document.getElementById("sourceCategory").value,
status: document.getElementById("sourceStatus").value,
notes: document.getElementById("sourceNotes").value || null,
language: document.getElementById("sourceLanguage").value || null,
bias: document.getElementById("sourceBias").value || null,
fetch_strategy: document.getElementById("sourceFetchStrategy").value || "default",
};
// Leere Werte NIE mitsenden. Ein leerer Select-Wert (z.B. META noch nicht
// geladen) würde sonst die Kategorie serverseitig still überschreiben.
const st = document.getElementById("sourceType").value;
if (st) body.source_type = st;
const cat = document.getElementById("sourceCategory").value;
if (cat) body.category = cat;
const pol = document.getElementById("sourcePolitical")?.value;
if (pol) body.political_orientation = pol;
@@ -349,7 +383,6 @@ function setupSourceForms() {
if (cc) body.country_code = cc;
if (editingSourceId) {
body.state_affiliated = !!document.getElementById("sourceStateAffiliated")?.checked;
body.alignments = getAlignmentChips();
}
try {
@@ -408,8 +441,7 @@ async function loadTenantSources() {
function populateTenantFilters() {
// Typ + Kategorie aus META, Org aus Cache
if (window.META && window.META.types) {
populateSelect(document.getElementById("tenantFilterType"),
window.META.types.filter(t => t.key !== "excluded"), "Alle Typen");
populateSelect(document.getElementById("tenantFilterType"), window.META.types, "Alle Typen");
}
if (window.META && window.META.categories) {
populateSelect(document.getElementById("tenantFilterCategory"),
@@ -693,34 +725,6 @@ const MEDIA_TYPE_LABELS = {
ngo: "NGO", behoerde: "Behörde", staatsmedium: "Staatsmedium",
fachmedium: "Fachmedium", sonstige: "Sonstige",
};
const ALIGNMENT_LABELS = {
prorussisch: "prorussisch", proiranisch: "proiranisch", prowestlich: "prowestlich",
proukrainisch: "proukrainisch", prochinesisch: "prochinesisch", projapanisch: "projapanisch",
proisraelisch: "proisraelisch", propalaestinensisch: "propalästinensisch",
protuerkisch: "protürkisch", panarabisch: "panarabisch", neutral: "neutral", sonstige: "sonstige",
};
function setAlignmentChips(active) {
const chips = document.querySelectorAll("#sourceAlignmentChips .alignment-chip");
const set = new Set((active || []).map((a) => (a || "").toLowerCase()));
chips.forEach((chip) => {
if (set.has(chip.dataset.alignment)) chip.classList.add("active");
else chip.classList.remove("active");
});
}
function getAlignmentChips() {
return Array.from(document.querySelectorAll("#sourceAlignmentChips .alignment-chip.active"))
.map((chip) => chip.dataset.alignment);
}
function handleAlignmentChipClick(e) {
const chip = e.target.closest(".alignment-chip");
if (!chip) return;
e.preventDefault();
chip.classList.toggle("active");
}
async function refreshClassificationStats() {
try {
const stats = await API.get("/api/sources/classification/stats");
@@ -761,8 +765,6 @@ function renderClassificationQueueItem(item) {
const relFmt = (v) => (v && v !== "na" ? RELIABILITY_LABELS[v] || v : "–");
const stateFmt = (v) => (v ? "ja" : "nein");
const ccFmt = (v) => v || "–";
const alignFmt = (v) =>
Array.isArray(v) && v.length > 0 ? v.map((a) => ALIGNMENT_LABELS[a] || a).join(", ") : "–";
const row = (label, c, p, fmt) => {
const cs = fmt(c);
@@ -796,7 +798,6 @@ function renderClassificationQueueItem(item) {
${row("Glaubwürdigkeit", cur.reliability, prop.reliability, relFmt)}
${row("Staatsnah", cur.state_affiliated, prop.state_affiliated, stateFmt)}
${row("Land", cur.country_code, prop.country_code, ccFmt)}
${row("Geopol. Nähe", cur.alignments, prop.alignments, alignFmt)}
</div>
${reasoning ? `<div class="review-card-reasoning"><strong>Begründung:</strong> ${reasoning}</div>` : ""}
<div class="review-card-actions">
@@ -885,9 +886,11 @@ function toggleSourceInfo(id) {
}
// --- PDF-Quellen-Upload ---
function openPdfUploadModal() {
async function openPdfUploadModal() {
const form = document.getElementById("pdfUploadForm");
if (form) form.reset();
await ensureMeta();
fillSelect(document.getElementById("pdfCategory"), (window.META && window.META.categories) || [], { value: "sonstige" });
const err = document.getElementById("pdfUploadError");
if (err) { err.style.display = "none"; err.textContent = ""; }
const prog = document.getElementById("pdfUploadProgress");

Datei anzeigen

@@ -49,17 +49,25 @@ def test_meta_types_have_required_fields(authed_client):
assert "label" in t
def test_meta_includes_specialized_categories(authed_client):
"""Phase 3b - die spezielleren Lagen-Themen muessen als Kategorien existieren."""
def test_meta_categories_exact_set(authed_client):
"""Bereinigte Kategorienliste - exakt gepinnt gegen stilles Driften.
telegram und x muessen drin sein (147 Quellen im Bestand), die frueher
definierten, aber nie belegten Lagen-Kategorien sind bewusst entfernt.
"""
r = authed_client.get("/api/sources/meta")
keys = {c["key"] for c in r.json()["categories"]}
assert "cybercrime" in keys
assert "ukraine-russland-krieg" in keys
assert "russische-staatspropaganda" in keys
assert keys == {
"nachrichtenagentur", "oeffentlich-rechtlich", "qualitaetszeitung",
"behoerde", "fachmedien", "think-tank", "international", "regional",
"boulevard", "telegram", "x", "sonstige",
}
assert "cybercrime" not in keys
assert "stimmungsbild" not in keys
def test_meta_includes_all_source_types(authed_client):
"""Alle 5 Source-Types muessen rauskommen."""
"""Alle 6 Source-Types muessen rauskommen (excluded ist bewusst entfernt)."""
r = authed_client.get("/api/sources/meta")
keys = {t["key"] for t in r.json()["types"]}
assert keys == {"rss_feed", "web_source", "telegram_channel", "podcast_feed", "excluded"}
assert keys == {"rss_feed", "web_source", "telegram_channel", "podcast_feed", "x_account", "pdf_document"}

Datei anzeigen

@@ -53,8 +53,6 @@ AUTH_PROTECTED = [
("GET", "/api/sources/health"),
("GET", "/api/sources/suggestions"),
("PUT", "/api/sources/suggestions/1"),
("POST", "/api/sources/health/run"),
("POST", "/api/sources/health/run-stream"),
("POST", "/api/sources/health/search-fix/1"),
("GET", "/api/statistik"),
("GET", "/api/token-usage/overview"),

Datei anzeigen

@@ -40,12 +40,17 @@ def test_category_label_lookup():
def test_type_label_lookup():
assert type_label("rss_feed") == "RSS-Feed"
assert type_label("telegram_channel") == "Telegram-Kanal"
assert type_label("x_account") == "X-Konto"
assert type_label("pdf_document") == "PDF-Dokument"
assert type_label("does-not-exist") == "does-not-exist"
def test_category_includes_aktuelle_themen():
"""Phase 3b: Lagen-spezifische Kategorien (cybercrime etc.) müssen drin sein."""
def test_category_covers_bestand():
"""telegram und x muessen als Kategorien existieren (belegter Bestand),
die nie belegten Lagen-Kategorien sind bewusst entfernt."""
keys = {c["key"] for c in SOURCE_CATEGORIES}
assert "cybercrime" in keys
assert "ukraine-russland-krieg" in keys
assert "russische-staatspropaganda" in keys
assert "telegram" in keys
assert "x" in keys
assert "cybercrime" not in keys
assert "ukraine-russland-krieg" not in keys
assert "russische-staatspropaganda" not in keys