feat: Discovery-Funktion in Grundquellen-Verwaltung integriert
- POST /api/sources/discover: URL analysieren, RSS-Feeds erkennen, Duplikate prüfen - POST /api/sources/discover/add: Erkannte Feeds als Grundquellen anlegen (inkl. Web-Source) - Erkennen-Button und Modal im Dashboard mit Feed-Auswahl per Checkbox - Duplikat-Erkennung zeigt bereits vorhandene Grundquellen an - source_rules aus Monitor importiert für Feed-Discovery und Claude-Bewertung - config.py um Discovery-Konfiguration erweitert Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
Dieser Commit ist enthalten in:
claude-dev
@@ -30,3 +30,8 @@ SMTP_USE_TLS = os.environ.get("SMTP_USE_TLS", "true").lower() == "true"
|
||||
# Magic Link Base URL (fuer OSINT-Monitor Einladungen)
|
||||
MAGIC_LINK_BASE_URL = os.environ.get("MAGIC_LINK_BASE_URL", "https://osint.intelsight.de")
|
||||
MAGIC_LINK_EXPIRE_MINUTES = 10
|
||||
|
||||
# Source Discovery (geteilte Config mit OSINT-Monitor)
|
||||
CLAUDE_PATH = os.environ.get("CLAUDE_PATH", "/home/claude-dev/.claude/local/claude")
|
||||
CLAUDE_TIMEOUT = 300
|
||||
MAX_FEEDS_PER_DOMAIN = 3
|
||||
|
||||
@@ -1,4 +1,10 @@
|
||||
"""Grundquellen-Verwaltung und Kundenquellen-Übersicht."""
|
||||
import sys
|
||||
import logging
|
||||
|
||||
# Monitor-Source-Rules verfügbar machen
|
||||
sys.path.insert(0, "/home/claude-dev/AegisSight-Monitor/src")
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Optional
|
||||
@@ -6,6 +12,17 @@ from auth import get_current_admin
|
||||
from database import db_dependency
|
||||
import aiosqlite
|
||||
|
||||
from source_rules import (
|
||||
discover_source,
|
||||
discover_all_feeds,
|
||||
evaluate_feeds_with_claude,
|
||||
_extract_domain,
|
||||
_detect_category,
|
||||
domain_to_display_name,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("verwaltung.sources")
|
||||
|
||||
router = APIRouter(prefix="/api/sources", tags=["sources"])
|
||||
|
||||
|
||||
@@ -165,3 +182,133 @@ async def promote_to_global(
|
||||
|
||||
cursor = await db.execute("SELECT * FROM sources WHERE id = ?", (source_id,))
|
||||
return dict(await cursor.fetchone())
|
||||
|
||||
|
||||
|
||||
@router.post("/discover")
|
||||
async def discover_source_endpoint(
|
||||
url: str,
|
||||
admin: dict = Depends(get_current_admin),
|
||||
db: aiosqlite.Connection = Depends(db_dependency),
|
||||
):
|
||||
"""URL analysieren: Domain, Kategorie und RSS-Feeds automatisch erkennen.
|
||||
|
||||
Findet alle Feeds einer Domain, bewertet sie mit Claude und gibt
|
||||
die relevanten zurueck. Prueft auf bereits vorhandene Grundquellen.
|
||||
"""
|
||||
try:
|
||||
multi = await discover_all_feeds(url)
|
||||
except Exception as e:
|
||||
logger.error(f"Discovery fehlgeschlagen: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Discovery fehlgeschlagen")
|
||||
|
||||
domain = multi["domain"]
|
||||
category = multi["category"]
|
||||
feeds = multi.get("feeds", [])
|
||||
|
||||
# Fallback auf Einzel-Discovery wenn keine Feeds gefunden
|
||||
if not feeds:
|
||||
try:
|
||||
single = await discover_source(url)
|
||||
if single.get("rss_url"):
|
||||
feeds = [{"name": single["name"], "url": single["rss_url"]}]
|
||||
domain = single.get("domain", domain)
|
||||
category = single.get("category", category)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not feeds:
|
||||
return {
|
||||
"domain": domain,
|
||||
"category": category,
|
||||
"feeds": [],
|
||||
"existing": [],
|
||||
"message": "Keine RSS-Feeds gefunden",
|
||||
}
|
||||
|
||||
# Mit Claude bewerten
|
||||
try:
|
||||
relevant_feeds = await evaluate_feeds_with_claude(domain, feeds)
|
||||
except Exception:
|
||||
relevant_feeds = feeds[:3]
|
||||
|
||||
# Bereits vorhandene Grundquellen pruefen
|
||||
cursor = await db.execute(
|
||||
"SELECT url FROM sources WHERE tenant_id IS NULL AND url IS NOT NULL"
|
||||
)
|
||||
existing_urls = {row["url"] for row in await cursor.fetchall()}
|
||||
|
||||
result_feeds = []
|
||||
existing = []
|
||||
for feed in relevant_feeds:
|
||||
info = {
|
||||
"name": feed.get("name", domain_to_display_name(domain)),
|
||||
"url": feed["url"],
|
||||
"domain": domain,
|
||||
"category": category,
|
||||
}
|
||||
if feed["url"] in existing_urls:
|
||||
existing.append(info)
|
||||
else:
|
||||
result_feeds.append(info)
|
||||
|
||||
return {
|
||||
"domain": domain,
|
||||
"category": category,
|
||||
"feeds": result_feeds,
|
||||
"existing": existing,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/discover/add")
|
||||
async def add_discovered_sources(
|
||||
feeds: list[dict],
|
||||
admin: dict = Depends(get_current_admin),
|
||||
db: aiosqlite.Connection = Depends(db_dependency),
|
||||
):
|
||||
"""Erkannte Feeds als Grundquellen anlegen.
|
||||
|
||||
Erwartet eine Liste von {name, url, domain, category}.
|
||||
Ueberspringt bereits vorhandene URLs.
|
||||
"""
|
||||
cursor = await db.execute(
|
||||
"SELECT url FROM sources WHERE tenant_id IS NULL AND url IS NOT NULL"
|
||||
)
|
||||
existing_urls = {row["url"] for row in await cursor.fetchall()}
|
||||
|
||||
added = 0
|
||||
skipped = 0
|
||||
for feed in feeds:
|
||||
if not feed.get("url"):
|
||||
continue
|
||||
if feed["url"] in existing_urls:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
domain = feed.get("domain", "")
|
||||
await db.execute(
|
||||
"""INSERT INTO sources (name, url, domain, source_type, category, status, added_by, tenant_id)
|
||||
VALUES (?, ?, ?, 'rss_feed', ?, 'active', 'system', NULL)""",
|
||||
(feed["name"], feed["url"], domain, feed.get("category", "sonstige")),
|
||||
)
|
||||
existing_urls.add(feed["url"])
|
||||
added += 1
|
||||
|
||||
# Web-Source für die Domain anlegen wenn noch nicht vorhanden
|
||||
if feeds and feeds[0].get("domain"):
|
||||
domain = feeds[0]["domain"]
|
||||
cursor = await db.execute(
|
||||
"SELECT id FROM sources WHERE LOWER(domain) = ? AND source_type = 'web_source' AND tenant_id IS NULL",
|
||||
(domain.lower(),),
|
||||
)
|
||||
if not await cursor.fetchone():
|
||||
await db.execute(
|
||||
"""INSERT INTO sources (name, url, domain, source_type, category, status, added_by, tenant_id)
|
||||
VALUES (?, ?, ?, 'web_source', ?, 'active', 'system', NULL)""",
|
||||
(domain_to_display_name(domain), f"https://{domain}", domain,
|
||||
feeds[0].get("category", "sonstige")),
|
||||
)
|
||||
added += 1
|
||||
|
||||
await db.commit()
|
||||
return {"added": added, "skipped": skipped}
|
||||
|
||||
@@ -209,6 +209,7 @@
|
||||
<input type="text" class="search-input" id="globalSourceSearch" placeholder="Grundquelle suchen...">
|
||||
<span class="text-secondary" id="globalSourceCount"></span>
|
||||
</div>
|
||||
<button class="btn btn-secondary" id="discoverSourceBtn">Erkennen</button>
|
||||
<button class="btn btn-primary" id="newGlobalSourceBtn">+ Neue Grundquelle</button>
|
||||
</div>
|
||||
<div class="card">
|
||||
@@ -421,6 +422,33 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Modal: Discover Sources -->
|
||||
<div class="modal-overlay" id="modalDiscover">
|
||||
<div class="modal" style="max-width:600px;">
|
||||
<div class="modal-header">
|
||||
<h3>Quellen erkennen</h3>
|
||||
<button class="modal-close" onclick="closeModal('modalDiscover')">×</button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<div class="form-group">
|
||||
<label for="discoverUrl">Website-URL</label>
|
||||
<div style="display:flex;gap:8px;">
|
||||
<input type="url" id="discoverUrl" placeholder="https://www.example.de" style="flex:1;" required>
|
||||
<button class="btn btn-primary" id="discoverBtn" onclick="runDiscover()">Erkennen</button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="discoverStatus" style="display:none;padding:12px 0;color:var(--text-secondary);font-size:13px;"></div>
|
||||
<div id="discoverResults" style="display:none;">
|
||||
<div id="discoverExisting" style="display:none;margin-bottom:12px;"></div>
|
||||
<div id="discoverFeeds"></div>
|
||||
<div style="margin-top:12px;display:flex;justify-content:flex-end;">
|
||||
<button class="btn btn-primary" id="addDiscoveredBtn" style="display:none;" onclick="addDiscoveredFeeds()">Ausgewählte hinzufügen</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Modal: Confirm -->
|
||||
<div class="modal-overlay" id="modalConfirm">
|
||||
<div class="modal" style="max-width: 400px;">
|
||||
|
||||
@@ -123,6 +123,12 @@ function editGlobalSource(id) {
|
||||
|
||||
function setupSourceForms() {
|
||||
document.getElementById("newGlobalSourceBtn").addEventListener("click", openNewGlobalSource);
|
||||
document.getElementById("discoverSourceBtn").addEventListener("click", () => {
|
||||
document.getElementById("discoverUrl").value = "";
|
||||
document.getElementById("discoverStatus").style.display = "none";
|
||||
document.getElementById("discoverResults").style.display = "none";
|
||||
openModal("modalDiscover");
|
||||
});
|
||||
|
||||
document.getElementById("sourceForm").addEventListener("submit", async (e) => {
|
||||
e.preventDefault();
|
||||
@@ -167,7 +173,7 @@ function setupSourceForms() {
|
||||
function confirmDeleteGlobalSource(id, name) {
|
||||
showConfirm(
|
||||
"Grundquelle loeschen",
|
||||
`Soll die Grundquelle "${name}" endgueltig geloescht werden? Sie wird fuer alle Monitore entfernt.`,
|
||||
`Soll die Grundquelle "${name}" endgültig gelöscht werden? Sie wird für alle Monitore entfernt.`,
|
||||
async () => {
|
||||
try {
|
||||
await API.del("/api/sources/global/" + id);
|
||||
@@ -229,7 +235,7 @@ document.addEventListener("DOMContentLoaded", () => {
|
||||
function promoteSource(id, name) {
|
||||
showConfirm(
|
||||
"Zur Grundquelle machen",
|
||||
`Soll "${name}" als Grundquelle uebernommen werden? Sie wird dann fuer alle Monitore verfuegbar.`,
|
||||
`Soll "${name}" als Grundquelle übernommen werden? Sie wird dann für alle Monitore verfügbar.`,
|
||||
async () => {
|
||||
try {
|
||||
await API.post("/api/sources/tenant/" + id + "/promote");
|
||||
@@ -240,3 +246,97 @@ function promoteSource(id, name) {
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
// --- Discovery ---
|
||||
let discoveredFeeds = [];
|
||||
|
||||
async function runDiscover() {
|
||||
const url = document.getElementById("discoverUrl").value.trim();
|
||||
if (!url) return;
|
||||
|
||||
const btn = document.getElementById("discoverBtn");
|
||||
const statusEl = document.getElementById("discoverStatus");
|
||||
const resultsEl = document.getElementById("discoverResults");
|
||||
|
||||
btn.disabled = true;
|
||||
btn.textContent = "Suche...";
|
||||
statusEl.style.display = "block";
|
||||
statusEl.textContent = "Analysiere Website und suche RSS-Feeds...";
|
||||
resultsEl.style.display = "none";
|
||||
|
||||
try {
|
||||
const data = await API.post("/api/sources/discover?url=" + encodeURIComponent(url));
|
||||
discoveredFeeds = data.feeds || [];
|
||||
|
||||
if (discoveredFeeds.length === 0 && (!data.existing || data.existing.length === 0)) {
|
||||
statusEl.textContent = data.message || "Keine RSS-Feeds gefunden für " + data.domain;
|
||||
return;
|
||||
}
|
||||
|
||||
statusEl.style.display = "none";
|
||||
resultsEl.style.display = "block";
|
||||
|
||||
// Bereits vorhandene anzeigen
|
||||
const existingEl = document.getElementById("discoverExisting");
|
||||
if (data.existing && data.existing.length > 0) {
|
||||
existingEl.style.display = "block";
|
||||
existingEl.innerHTML = '<div class="text-secondary" style="font-size:12px;margin-bottom:8px;">Bereits als Grundquelle vorhanden:</div>' +
|
||||
data.existing.map(f => '<div style="padding:4px 0;font-size:13px;color:var(--text-tertiary);">✓ ' + esc(f.name) + '</div>').join("");
|
||||
} else {
|
||||
existingEl.style.display = "none";
|
||||
}
|
||||
|
||||
// Neue Feeds mit Checkboxen
|
||||
const feedsEl = document.getElementById("discoverFeeds");
|
||||
if (discoveredFeeds.length > 0) {
|
||||
feedsEl.innerHTML = '<div class="text-secondary" style="font-size:12px;margin-bottom:8px;">Neue Feeds gefunden (' + data.domain + ', ' + (CATEGORY_LABELS[data.category] || data.category) + '):</div>' +
|
||||
discoveredFeeds.map((f, i) => `
|
||||
<label style="display:flex;align-items:center;gap:8px;padding:6px 0;font-size:13px;cursor:pointer;">
|
||||
<input type="checkbox" checked data-idx="${i}">
|
||||
<span>${esc(f.name)}</span>
|
||||
<span class="text-secondary" style="font-size:11px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;max-width:250px;" title="${esc(f.url)}">${esc(f.url)}</span>
|
||||
</label>
|
||||
`).join("");
|
||||
document.getElementById("addDiscoveredBtn").style.display = "";
|
||||
} else {
|
||||
feedsEl.innerHTML = '<div class="text-muted" style="font-size:13px;">Alle Feeds dieser Domain sind bereits als Grundquellen vorhanden.</div>';
|
||||
document.getElementById("addDiscoveredBtn").style.display = "none";
|
||||
}
|
||||
} catch (err) {
|
||||
statusEl.textContent = "Fehler: " + err.message;
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = "Erkennen";
|
||||
}
|
||||
}
|
||||
|
||||
async function addDiscoveredFeeds() {
|
||||
const checkboxes = document.querySelectorAll("#discoverFeeds input[type=checkbox]:checked");
|
||||
const selected = [];
|
||||
checkboxes.forEach(cb => {
|
||||
const idx = parseInt(cb.dataset.idx);
|
||||
if (discoveredFeeds[idx]) selected.push(discoveredFeeds[idx]);
|
||||
});
|
||||
|
||||
if (selected.length === 0) {
|
||||
alert("Keine Feeds ausgewaehlt");
|
||||
return;
|
||||
}
|
||||
|
||||
const btn = document.getElementById("addDiscoveredBtn");
|
||||
btn.disabled = true;
|
||||
btn.textContent = "Wird hinzugefügt...";
|
||||
|
||||
try {
|
||||
const result = await API.post("/api/sources/discover/add", selected);
|
||||
closeModal("modalDiscover");
|
||||
loadGlobalSources();
|
||||
alert(result.added + " Grundquelle(n) hinzugefügt" + (result.skipped ? ", " + result.skipped + " übersprungen" : ""));
|
||||
} catch (err) {
|
||||
alert("Fehler: " + err.message);
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = "Ausgewählte hinzufügen";
|
||||
}
|
||||
}
|
||||
|
||||
In neuem Issue referenzieren
Einen Benutzer sperren