AegisSight/AegisSight-Monitor
3
0
Fork 0
Code Issues Pull-Requests Actions Pakete Projekte Releases Wiki Aktivität

feat(studio): Phase 2 - Studio-Bausteine, Backend-Kern (Sammeln/Analyse/Faktencheck)

Quellcode durchsuchen 
Backend fuer die modularen Studio-Pipeline-Bausteine, alles ueber Claude.

DB (additiv, idempotent):
- Tabellen incident_events + fact_check_runs (+ Indizes)
- Funktion log_incident_event()
- Spalten incidents.summary_at, incidents.executive_summary, articles.geoparsed_at

agents/stage_runners.py (neu): isolierte Bausteine analyze + factcheck auf dem
vorhandenen Bestand, komplett neu, Historie bleibt (Snapshot/fact_check_runs).

orchestrator.py: collect_only in die Lane-Multi-Tenant-Queue eingebaut (enqueue_refresh
4-Tupel, _worker-Entpacken, Multi-Pass-Gate, _run_refresh-Signatur + Analyse/FC-Skip).
Zusaetzlich summary_at beim Schreiben des Lagebilds gestempelt (Studio-Freshness).

incidents.py: Endpunkte POST /{id}/run/{stage} (collect|analyze|factcheck),
GET /{id}/run-status, GET /{id}/freshness, GET /{id}/factcheck-runs[/{run_id}].
Busy-Check auf Online-Orchestrator (_current_tasks), Tenant-Zugriffspruefung.

Kein lokaler LLM-Unterbau, kein lokales Modell. /ask + /events folgen in Phase 3.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Dieser Commit ist enthalten in:
claude-dev
2026-07-24 23:18:15 +02:00
Ursprung 4bc842ca1c
Commit b686685f43
4 geänderte Dateien mit 538 neuen und 11 gelöschten Zeilen
Patch-Datei herunterladen Vergleichs-Datei herunterladen Alle Dateien ausklappen Alle Dateien einklappen

34
src/agents/orchestrator.py
Datei anzeigen

@@ -458,9 +458,13 @@ class AgentOrchestrator:
task.cancel()
logger.info("Agenten-Orchestrator gestoppt")
async def enqueue_refresh(self, incident_id: int, trigger_type: str = "manual", user_id: int = None) -> bool:
async def enqueue_refresh(self, incident_id: int, trigger_type: str = "manual", user_id: int = None, collect_only: bool = False) -> bool:
"""Refresh-Auftrag in die Lane der Organisation stellen. Gibt False zurueck
wenn die Lage dort bereits wartet oder gerade laeuft."""
wenn die Lage dort bereits wartet oder gerade laeuft.
collect_only=True (Studio-Baustein "Sammeln"): sammelt nur Artikel, ueberspringt
Analyse und Faktencheck.
"""
if incident_id in self._queued_ids or incident_id in self._current_tasks:
logger.info(f"Refresh fuer Lage {incident_id} uebersprungen: bereits aktiv/in Queue")
return False
@@ -476,7 +480,7 @@ class AgentOrchestrator:
self._lane_workers[lane_key] = asyncio.create_task(self._worker(lane_key, queue))
logger.info(f"Neue Worker-Lane fuer Organisation {lane_key} gestartet")
self._queued_ids.add(incident_id)
queue.put_nowait((incident_id, trigger_type, user_id))
queue.put_nowait((incident_id, trigger_type, user_id, collect_only))
queue_size = queue.qsize()
logger.info(f"Refresh fuer Lage {incident_id} eingereiht (Lane {lane_key}, Queue: {queue_size}, Trigger: {trigger_type})")
@@ -582,11 +586,15 @@ class AgentOrchestrator:
return
continue
if len(item) == 3:
if len(item) == 4:
incident_id, trigger_type, user_id, collect_only = item
elif len(item) == 3:
incident_id, trigger_type, user_id = item
collect_only = False
else:
incident_id, trigger_type = item
user_id = None
collect_only = False
self._queued_ids.discard(incident_id)
# Session-Start EINMAL setzen — bleibt ueber Multi-Pass/Retry hinweg stabil
self._current_tasks[incident_id] = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
@@ -610,14 +618,14 @@ class AgentOrchestrator:
try:
# Research-Lagen: Automatisch 3 Durchläufe nur beim ersten Refresh
incident_type, has_summary = await self._get_incident_info(incident_id)
use_multi_pass = incident_type == "research" and not has_summary
use_multi_pass = incident_type == "research" and not has_summary and not collect_only
for attempt in range(3):
try:
if use_multi_pass:
await self._run_research_multi_pass(incident_id, trigger_type=trigger_type, user_id=user_id)
else:
await self._run_refresh(incident_id, trigger_type=trigger_type, retry_count=attempt, user_id=user_id)
await self._run_refresh(incident_id, trigger_type=trigger_type, retry_count=attempt, user_id=user_id, collect_only=collect_only)
last_error = None
break # Erfolg
except asyncio.CancelledError:
@@ -783,7 +791,7 @@ class AgentOrchestrator:
await db.close()
return visibility, created_by, tenant_id
async def _run_refresh(self, incident_id: int, trigger_type: str = "manual", retry_count: int = 0, user_id: int = None, _suppress_complete: bool = False, _pass_info: dict = None):
async def _run_refresh(self, incident_id: int, trigger_type: str = "manual", retry_count: int = 0, user_id: int = None, _suppress_complete: bool = False, _pass_info: dict = None, collect_only: bool = False):
"""Führt einen kompletten Refresh-Zyklus durch."""
import aiosqlite
from database import get_db
@@ -1433,7 +1441,8 @@ class AgentOrchestrator:
logger.warning(f"Quellen-Statistiken konnten nicht aktualisiert werden: {e}")
# Schritt 3+4: Analyse und Faktencheck PARALLEL
if new_count > 0 or not previous_summary:
# collect_only (Studio-Baustein "Sammeln"): ueberspringt Analyse/Faktencheck.
if (new_count > 0 or not previous_summary) and not collect_only:
is_first_summary = not previous_summary
# Snapshot des alten Lagebilds sichern BEVOR parallele Verarbeitung startet
@@ -1775,9 +1784,14 @@ class AgentOrchestrator:
pass
sources_json = json.dumps(sources, ensure_ascii=False) if sources else previous_sources_json
# summary_at haelt fest, WANN das Lagebild entstand (Studio-Freshness).
# JETZT stempeln, nicht 'now' vom Lauf-Beginn: sonst saehen die in diesem
# Lauf gesammelten Artikel neuer aus als der Bericht, der sie schon enthaelt.
summary_now = datetime.now(TIMEZONE).strftime('%Y-%m-%d %H:%M:%S')
await db.execute(
"UPDATE incidents SET summary = ?, sources_json = ?, executive_summary = NULL, updated_at = ? WHERE id = ?",
(new_summary, sources_json, now, incident_id),
"UPDATE incidents SET summary = ?, sources_json = ?, executive_summary = NULL, "
"updated_at = ?, summary_at = ? WHERE id = ?",
(new_summary, sources_json, summary_now, summary_now, incident_id),
)
# Beim ersten Refresh: Snapshot des neuen Lagebilds erstellen

257
src/agents/stage_runners.py Normale Datei
Datei anzeigen

@@ -0,0 +1,257 @@
"""
Modulare Pipeline-Bausteine fuers Studio.
Fuehrt einzelne Pipeline-Stufen ISOLIERT auf dem vorhandenen DB-Bestand aus,
statt des durchlaufenden orchestrator._run_refresh. Das grosse _run_refresh
bleibt unangetastet (dient weiter als "kompletter Lauf").
Nutzt die bereits reinen Agenten (AnalyzerAgent/FactCheckerAgent) und bildet nur
die Lade-/Persistenz-Logik aus _run_refresh nach.
Verhalten: KOMPLETT NEU (ersetzt das aktive Ergebnis), Historie bleibt einsehbar:
- Analyse -> altes Lagebild wird als incident_snapshots archiviert, dann neu gesetzt
- Faktencheck -> alter Faktenstand wird als fact_check_runs archiviert, dann ersetzt
Phase 1: analyze, factcheck. (Spaeter: collect, geoparse, network.)
"""
from __future__ import annotations
import asyncio
import json
import logging
from datetime import datetime
from config import TIMEZONE
from database import get_db
logger = logging.getLogger("osint.stages")
# Im Studio einzeln startbare Bausteine (Phase 1)
STAGES = {"analyze", "factcheck"}
# Status-Registry je Incident (Single-Flight) + Task-Referenzen (GC-Schutz)
_STATE: dict[int, dict] = {}
_TASKS: set = set()
def _now() -> str:
return datetime.now(TIMEZONE).strftime("%Y-%m-%d %H:%M:%S")
def get_state(incident_id: int) -> dict | None:
return _STATE.get(incident_id)
def is_running(incident_id: int) -> bool:
st = _STATE.get(incident_id)
return bool(st and st.get("status") == "running")
async def _load_incident(db, incident_id: int) -> dict | None:
cur = await db.execute("SELECT * FROM incidents WHERE id = ?", (incident_id,))
row = await cur.fetchone()
return dict(row) if row else None
async def _output_language(db, tenant_id) -> str:
from services.org_settings import get_org_language, language_display
iso = await get_org_language(db, tenant_id) if tenant_id else "de"
return language_display(iso)
async def _count(db, sql: str, params) -> int:
row = await (await db.execute(sql, params)).fetchone()
return (row[0] if row else 0) or 0
# ---------------------------------------------------------------------------
# Baustein: Analyse (Lagebild / Recherchebericht) — komplett neu
# ---------------------------------------------------------------------------
async def _run_analysis(db, inc: dict, user_id) -> dict:
from agents.analyzer import AnalyzerAgent, build_fact_context_block
from services.license_service import charge_usage_to_tenant
incident_id = inc["id"]
tenant_id = inc.get("tenant_id")
incident_type = inc.get("type") or "adhoc"
title = inc.get("title") or ""
description = inc.get("description") or ""
prev_summary = inc.get("summary") or ""
prev_sources = inc.get("sources_json")
now = _now()
output_language = await _output_language(db, tenant_id)
# 1) Altes Lagebild als Snapshot archivieren (Historie), bevor es ersetzt wird
if prev_summary:
acnt = await _count(db, "SELECT COUNT(*) FROM articles WHERE incident_id = ?", (incident_id,))
fcnt = await _count(db, "SELECT COUNT(*) FROM fact_checks WHERE incident_id = ?", (incident_id,))
await db.execute(
"""INSERT INTO incident_snapshots
(incident_id, summary, sources_json, article_count, fact_check_count,
refresh_log_id, created_at, tenant_id)
VALUES (?,?,?,?,?,?,?,?)""",
(incident_id, prev_summary, prev_sources, acnt, fcnt, None, now, tenant_id),
)
await db.commit()
# 2) Alle Artikel + bestehende Fakten laden (Frische-Bias bei adhoc)
order = "published_at IS NULL, published_at DESC" if incident_type == "adhoc" else "collected_at DESC"
arts = [dict(r) for r in await (await db.execute(
f"SELECT * FROM articles WHERE incident_id = ? ORDER BY {order}", (incident_id,)
)).fetchall()]
if not arts:
raise ValueError("Keine Artikel vorhanden — bitte zuerst sammeln.")
facts = [dict(r) for r in await (await db.execute(
"SELECT id, claim, status, sources_count, evidence FROM fact_checks WHERE incident_id = ?",
(incident_id,),
)).fetchall()]
fact_ctx = ""
try:
fact_ctx = build_fact_context_block(facts, [], incident_type)
except Exception as e:
logger.warning(f"Faktenkontext fuer Analyse fehlgeschlagen: {e}")
# 3) Analyse komplett neu ueber ALLE Artikel
analyzer = AnalyzerAgent()
analysis, usage = await analyzer.analyze(
title, description, arts, incident_type,
fact_context_block=fact_ctx, output_language=output_language,
)
if usage:
try:
await charge_usage_to_tenant(db, tenant_id, usage, source="analysis")
except Exception:
pass
if not analysis or not (analysis.get("summary") or "").strip():
raise ValueError("Analyse lieferte kein Lagebild.")
summary = analysis.get("summary") or ""
sources = analysis.get("sources") or []
for s in sources:
if isinstance(s.get("nr"), str):
try:
s["nr"] = int(s["nr"])
except ValueError:
pass
sources_json = json.dumps(sources, ensure_ascii=False) if sources else prev_sources
# summary_at = Entstehungszeit des Lagebilds. JETZT stempeln, nicht 'now' vom Beginn
# des Bausteins: die Analyse laeuft Minuten, und der Stempel muss den Stand abdecken,
# der tatsaechlich verarbeitet wurde. (updated_at wird auch beim Sammeln gesetzt und
# taugt als Bericht-Zeitpunkt ohnehin nicht.)
summary_now = _now()
await db.execute(
"UPDATE incidents SET summary = ?, sources_json = ?, executive_summary = NULL, "
"updated_at = ?, summary_at = ? WHERE id = ?",
(summary, sources_json, summary_now, summary_now, incident_id),
)
await db.commit()
return {"articles": len(arts), "summary_len": len(summary), "sources": len(sources)}
# ---------------------------------------------------------------------------
# Baustein: Faktencheck — komplett neu (alter Stand als Lauf archiviert)
# ---------------------------------------------------------------------------
async def _run_factcheck(db, inc: dict, user_id) -> dict:
from agents.factchecker import FactCheckerAgent, deduplicate_new_facts
from services.license_service import charge_usage_to_tenant
incident_id = inc["id"]
tenant_id = inc.get("tenant_id")
incident_type = inc.get("type") or "adhoc"
title = inc.get("title") or ""
now = _now()
output_language = await _output_language(db, tenant_id)
# 1) Aktuellen Faktenstand als Lauf archivieren (Historie)
cur_facts = [dict(r) for r in await (await db.execute(
"SELECT claim, status, sources_count, evidence, is_notification, checked_at, status_history "
"FROM fact_checks WHERE incident_id = ? ORDER BY id", (incident_id,)
)).fetchall()]
if cur_facts:
await db.execute(
"INSERT INTO fact_check_runs (incident_id, tenant_id, created_at, facts_json, fact_count) VALUES (?,?,?,?,?)",
(incident_id, tenant_id, now, json.dumps(cur_facts, ensure_ascii=False), len(cur_facts)),
)
await db.commit()
# 2) Alle Artikel laden
arts = [dict(r) for r in await (await db.execute(
"SELECT * FROM articles WHERE incident_id = ? ORDER BY collected_at DESC", (incident_id,)
)).fetchall()]
if not arts:
raise ValueError("Keine Artikel vorhanden — bitte zuerst sammeln.")
# 3) Faktencheck komplett neu
fc = FactCheckerAgent()
facts, usage = await fc.check(title, arts, incident_type, output_language=output_language)
if usage:
try:
await charge_usage_to_tenant(db, tenant_id, usage, source="factcheck")
except Exception:
pass
facts = deduplicate_new_facts(facts or [])
# 4) Bestehende Fakten ersetzen
await db.execute("DELETE FROM fact_checks WHERE incident_id = ?", (incident_id,))
for f in facts:
init_hist = json.dumps([{"status": f.get("status", "developing"), "at": now}])
await db.execute(
"""INSERT INTO fact_checks
(incident_id, claim, status, sources_count, evidence, is_notification, tenant_id, status_history, checked_at)
VALUES (?,?,?,?,?,?,?,?,?)""",
(incident_id, f.get("claim", ""), f.get("status", "developing"),
f.get("sources_count", 0), f.get("evidence"), f.get("is_notification", 0),
tenant_id, init_hist, now),
)
await db.commit()
return {"facts": len(facts), "archived": len(cur_facts), "articles": len(arts)}
# ---------------------------------------------------------------------------
# Orchestrierung: Start (Single-Flight) + Status
# ---------------------------------------------------------------------------
_RUNNERS = {"analyze": _run_analysis, "factcheck": _run_factcheck}
_LABELS = {"analyze": "Analyse", "factcheck": "Faktencheck"}
def start_stage(incident_id: int, stage: str, user_id) -> bool:
"""Startet einen Baustein im Hintergrund. False, wenn schon einer laeuft."""
if stage not in STAGES:
raise ValueError(f"Unbekannter Baustein: {stage}")
if is_running(incident_id):
return False
_STATE[incident_id] = {
"stage": stage, "label": _LABELS.get(stage, stage),
"status": "running", "started_at": _now(),
"finished_at": None, "error": None, "result": None,
}
t = asyncio.create_task(_execute(incident_id, stage, user_id))
_TASKS.add(t)
t.add_done_callback(_TASKS.discard)
return True
async def _execute(incident_id: int, stage: str, user_id):
db = await get_db()
started_at = _STATE.get(incident_id, {}).get("started_at")
try:
inc = await _load_incident(db, incident_id)
if not inc:
raise ValueError("Lage nicht gefunden")
res = await _RUNNERS[stage](db, inc, user_id)
_STATE[incident_id] = {
"stage": stage, "label": _LABELS.get(stage, stage), "status": "done",
"started_at": started_at, "finished_at": _now(), "error": None, "result": res,
}
logger.info(f"Baustein {stage} Lage {incident_id} fertig: {res}")
except Exception as e:
logger.warning(f"Baustein {stage} Lage {incident_id} Fehler: {e}", exc_info=True)
_STATE[incident_id] = {
"stage": stage, "label": _LABELS.get(stage, stage), "status": "error",
"started_at": started_at, "finished_at": _now(), "error": str(e)[:400], "result": None,
}
finally:
await db.close()