Initial commit: AegisSight-Monitor (OSINT-Monitoringsystem)
Dieser Commit ist enthalten in:
claude-dev
504
src/database.py
Normale Datei
504
src/database.py
Normale Datei
@@ -0,0 +1,504 @@
|
||||
"""SQLite Datenbank-Setup und Zugriff."""
|
||||
import aiosqlite
|
||||
import logging
|
||||
import os
|
||||
from config import DB_PATH, DATA_DIR
|
||||
|
||||
logger = logging.getLogger("osint.database")
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS organizations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
slug TEXT UNIQUE NOT NULL,
|
||||
is_active INTEGER DEFAULT 1,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS licenses (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
organization_id INTEGER NOT NULL REFERENCES organizations(id) ON DELETE CASCADE,
|
||||
license_type TEXT NOT NULL DEFAULT 'trial',
|
||||
max_users INTEGER NOT NULL DEFAULT 5,
|
||||
valid_from TIMESTAMP NOT NULL,
|
||||
valid_until TIMESTAMP,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
notes TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
email TEXT UNIQUE NOT NULL,
|
||||
username TEXT NOT NULL,
|
||||
password_hash TEXT,
|
||||
organization_id INTEGER NOT NULL REFERENCES organizations(id),
|
||||
role TEXT NOT NULL DEFAULT 'member',
|
||||
is_active INTEGER DEFAULT 1,
|
||||
last_login_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS magic_links (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
email TEXT NOT NULL,
|
||||
token TEXT UNIQUE NOT NULL,
|
||||
code TEXT NOT NULL,
|
||||
purpose TEXT NOT NULL DEFAULT 'login',
|
||||
user_id INTEGER REFERENCES users(id),
|
||||
is_used INTEGER DEFAULT 0,
|
||||
expires_at TIMESTAMP NOT NULL,
|
||||
ip_address TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS portal_admins (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username TEXT UNIQUE NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS incidents (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
status TEXT DEFAULT 'active',
|
||||
type TEXT DEFAULT 'adhoc',
|
||||
refresh_mode TEXT DEFAULT 'manual',
|
||||
refresh_interval INTEGER DEFAULT 15,
|
||||
retention_days INTEGER DEFAULT 0,
|
||||
visibility TEXT DEFAULT 'public',
|
||||
summary TEXT,
|
||||
sources_json TEXT,
|
||||
international_sources INTEGER DEFAULT 1,
|
||||
tenant_id INTEGER REFERENCES organizations(id),
|
||||
created_by INTEGER REFERENCES users(id),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS articles (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
incident_id INTEGER REFERENCES incidents(id) ON DELETE CASCADE,
|
||||
headline TEXT NOT NULL,
|
||||
headline_de TEXT,
|
||||
source TEXT NOT NULL,
|
||||
source_url TEXT,
|
||||
content_original TEXT,
|
||||
content_de TEXT,
|
||||
language TEXT DEFAULT 'de',
|
||||
published_at TIMESTAMP,
|
||||
collected_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
verification_status TEXT DEFAULT 'unverified',
|
||||
tenant_id INTEGER REFERENCES organizations(id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS fact_checks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
incident_id INTEGER REFERENCES incidents(id) ON DELETE CASCADE,
|
||||
claim TEXT NOT NULL,
|
||||
status TEXT DEFAULT 'developing',
|
||||
sources_count INTEGER DEFAULT 0,
|
||||
evidence TEXT,
|
||||
is_notification INTEGER DEFAULT 0,
|
||||
checked_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
tenant_id INTEGER REFERENCES organizations(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS refresh_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
incident_id INTEGER REFERENCES incidents(id) ON DELETE CASCADE,
|
||||
started_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
completed_at TIMESTAMP,
|
||||
articles_found INTEGER DEFAULT 0,
|
||||
status TEXT DEFAULT 'running',
|
||||
tenant_id INTEGER REFERENCES organizations(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS incident_snapshots (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
incident_id INTEGER REFERENCES incidents(id) ON DELETE CASCADE,
|
||||
summary TEXT,
|
||||
sources_json TEXT,
|
||||
article_count INTEGER DEFAULT 0,
|
||||
fact_check_count INTEGER DEFAULT 0,
|
||||
refresh_log_id INTEGER REFERENCES refresh_log(id),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
tenant_id INTEGER REFERENCES organizations(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sources (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
url TEXT,
|
||||
domain TEXT,
|
||||
source_type TEXT NOT NULL DEFAULT 'rss_feed',
|
||||
category TEXT NOT NULL DEFAULT 'sonstige',
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
notes TEXT,
|
||||
added_by TEXT,
|
||||
article_count INTEGER DEFAULT 0,
|
||||
last_seen_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
tenant_id INTEGER REFERENCES organizations(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS notifications (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
incident_id INTEGER REFERENCES incidents(id) ON DELETE CASCADE,
|
||||
type TEXT NOT NULL DEFAULT 'refresh_summary',
|
||||
title TEXT NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
icon TEXT DEFAULT 'info',
|
||||
is_read INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
tenant_id INTEGER REFERENCES organizations(id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_notifications_user_read ON notifications(user_id, is_read);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS incident_subscriptions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
incident_id INTEGER NOT NULL REFERENCES incidents(id) ON DELETE CASCADE,
|
||||
notify_email_summary INTEGER DEFAULT 0,
|
||||
notify_email_new_articles INTEGER DEFAULT 0,
|
||||
notify_email_status_change INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(user_id, incident_id)
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
async def get_db() -> aiosqlite.Connection:
|
||||
"""Erstellt eine neue Datenbankverbindung."""
|
||||
os.makedirs(DATA_DIR, exist_ok=True)
|
||||
db = await aiosqlite.connect(DB_PATH)
|
||||
db.row_factory = aiosqlite.Row
|
||||
await db.execute("PRAGMA journal_mode=WAL")
|
||||
await db.execute("PRAGMA foreign_keys=ON")
|
||||
return db
|
||||
|
||||
|
||||
async def init_db():
|
||||
"""Initialisiert die Datenbank mit dem Schema."""
|
||||
db = await get_db()
|
||||
try:
|
||||
await db.executescript(SCHEMA)
|
||||
await db.commit()
|
||||
|
||||
# --- Migrationen fuer bestehende Datenbanken ---
|
||||
|
||||
# Incidents-Spalten pruefen
|
||||
cursor = await db.execute("PRAGMA table_info(incidents)")
|
||||
columns = [row[1] for row in await cursor.fetchall()]
|
||||
|
||||
if "type" not in columns:
|
||||
await db.execute("ALTER TABLE incidents ADD COLUMN type TEXT DEFAULT 'adhoc'")
|
||||
await db.commit()
|
||||
|
||||
if "sources_json" not in columns:
|
||||
await db.execute("ALTER TABLE incidents ADD COLUMN sources_json TEXT")
|
||||
await db.commit()
|
||||
|
||||
if "international_sources" not in columns:
|
||||
await db.execute("ALTER TABLE incidents ADD COLUMN international_sources INTEGER DEFAULT 1")
|
||||
await db.commit()
|
||||
|
||||
if "visibility" not in columns:
|
||||
await db.execute("ALTER TABLE incidents ADD COLUMN visibility TEXT DEFAULT 'public'")
|
||||
await db.commit()
|
||||
|
||||
if "tenant_id" not in columns:
|
||||
await db.execute("ALTER TABLE incidents ADD COLUMN tenant_id INTEGER REFERENCES organizations(id)")
|
||||
await db.commit()
|
||||
logger.info("Migration: tenant_id zu incidents hinzugefuegt")
|
||||
|
||||
# Migration: E-Mail-Benachrichtigungs-Praeferenzen pro Lage
|
||||
if "notify_email_summary" not in columns:
|
||||
await db.execute("ALTER TABLE incidents ADD COLUMN notify_email_summary INTEGER DEFAULT 0")
|
||||
await db.execute("ALTER TABLE incidents ADD COLUMN notify_email_new_articles INTEGER DEFAULT 0")
|
||||
await db.execute("ALTER TABLE incidents ADD COLUMN notify_email_status_change INTEGER DEFAULT 0")
|
||||
await db.commit()
|
||||
logger.info("Migration: E-Mail-Benachrichtigungs-Spalten zu incidents hinzugefuegt")
|
||||
|
||||
# Migration: Token-Spalten fuer refresh_log
|
||||
cursor = await db.execute("PRAGMA table_info(refresh_log)")
|
||||
rl_columns = [row[1] for row in await cursor.fetchall()]
|
||||
if "input_tokens" not in rl_columns:
|
||||
await db.execute("ALTER TABLE refresh_log ADD COLUMN input_tokens INTEGER DEFAULT 0")
|
||||
await db.execute("ALTER TABLE refresh_log ADD COLUMN output_tokens INTEGER DEFAULT 0")
|
||||
await db.execute("ALTER TABLE refresh_log ADD COLUMN cache_creation_tokens INTEGER DEFAULT 0")
|
||||
await db.execute("ALTER TABLE refresh_log ADD COLUMN cache_read_tokens INTEGER DEFAULT 0")
|
||||
await db.execute("ALTER TABLE refresh_log ADD COLUMN total_cost_usd REAL DEFAULT 0.0")
|
||||
await db.execute("ALTER TABLE refresh_log ADD COLUMN api_calls INTEGER DEFAULT 0")
|
||||
await db.commit()
|
||||
|
||||
if "trigger_type" not in rl_columns:
|
||||
await db.execute("ALTER TABLE refresh_log ADD COLUMN trigger_type TEXT DEFAULT 'manual'")
|
||||
await db.commit()
|
||||
|
||||
if "retry_count" not in rl_columns:
|
||||
await db.execute("ALTER TABLE refresh_log ADD COLUMN retry_count INTEGER DEFAULT 0")
|
||||
await db.execute("ALTER TABLE refresh_log ADD COLUMN error_message TEXT")
|
||||
await db.commit()
|
||||
|
||||
if "tenant_id" not in rl_columns:
|
||||
await db.execute("ALTER TABLE refresh_log ADD COLUMN tenant_id INTEGER REFERENCES organizations(id)")
|
||||
await db.commit()
|
||||
|
||||
# Migration: reliability_score entfernen (falls noch vorhanden)
|
||||
cursor = await db.execute("PRAGMA table_info(incidents)")
|
||||
inc_columns = [row[1] for row in await cursor.fetchall()]
|
||||
if "reliability_score" in inc_columns:
|
||||
await db.execute("ALTER TABLE incidents DROP COLUMN reliability_score")
|
||||
await db.commit()
|
||||
|
||||
cursor = await db.execute("PRAGMA table_info(incident_snapshots)")
|
||||
snap_columns = [row[1] for row in await cursor.fetchall()]
|
||||
if "reliability_score" in snap_columns:
|
||||
await db.execute("ALTER TABLE incident_snapshots DROP COLUMN reliability_score")
|
||||
await db.commit()
|
||||
|
||||
# Migration: notifications-Tabelle (fuer bestehende DBs)
|
||||
cursor = await db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='notifications'")
|
||||
if not await cursor.fetchone():
|
||||
await db.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS notifications (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
incident_id INTEGER REFERENCES incidents(id) ON DELETE CASCADE,
|
||||
type TEXT NOT NULL DEFAULT 'refresh_summary',
|
||||
title TEXT NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
icon TEXT DEFAULT 'info',
|
||||
is_read INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
tenant_id INTEGER REFERENCES organizations(id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_notifications_user_read ON notifications(user_id, is_read);
|
||||
""")
|
||||
await db.commit()
|
||||
|
||||
# Migration: incident_subscriptions-Tabelle (fuer bestehende DBs)
|
||||
cursor = await db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='incident_subscriptions'")
|
||||
if not await cursor.fetchone():
|
||||
await db.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS incident_subscriptions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
incident_id INTEGER NOT NULL REFERENCES incidents(id) ON DELETE CASCADE,
|
||||
notify_email_summary INTEGER DEFAULT 0,
|
||||
notify_email_new_articles INTEGER DEFAULT 0,
|
||||
notify_email_status_change INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(user_id, incident_id)
|
||||
);
|
||||
""")
|
||||
await db.commit()
|
||||
logger.info("Migration: incident_subscriptions-Tabelle erstellt")
|
||||
else:
|
||||
# Migration: Spalte umbenennen contradiction -> new_articles
|
||||
cursor = await db.execute("PRAGMA table_info(incident_subscriptions)")
|
||||
sub_columns = [row[1] for row in await cursor.fetchall()]
|
||||
if "notify_email_contradiction" in sub_columns:
|
||||
await db.execute("ALTER TABLE incident_subscriptions RENAME COLUMN notify_email_contradiction TO notify_email_new_articles")
|
||||
await db.commit()
|
||||
logger.info("Migration: notify_email_contradiction -> notify_email_new_articles umbenannt")
|
||||
|
||||
# Migration: role-Spalte fuer users
|
||||
cursor = await db.execute("PRAGMA table_info(users)")
|
||||
user_columns = [row[1] for row in await cursor.fetchall()]
|
||||
if "role" not in user_columns:
|
||||
await db.execute("ALTER TABLE users ADD COLUMN role TEXT DEFAULT 'member'")
|
||||
await db.execute("UPDATE users SET role = 'org_admin'")
|
||||
await db.commit()
|
||||
logger.info("Migration: role-Spalte zu users hinzugefuegt")
|
||||
|
||||
# Migration: email, organization_id, is_active, last_login_at fuer users
|
||||
if "email" not in user_columns:
|
||||
await db.execute("ALTER TABLE users ADD COLUMN email TEXT")
|
||||
await db.commit()
|
||||
logger.info("Migration: email zu users hinzugefuegt")
|
||||
|
||||
if "organization_id" not in user_columns:
|
||||
await db.execute("ALTER TABLE users ADD COLUMN organization_id INTEGER REFERENCES organizations(id)")
|
||||
await db.commit()
|
||||
logger.info("Migration: organization_id zu users hinzugefuegt")
|
||||
|
||||
# Index erst nach Spalten-Migration erstellen
|
||||
try:
|
||||
await db.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_users_org_username ON users(organization_id, username)")
|
||||
await db.commit()
|
||||
except Exception:
|
||||
pass # Index existiert bereits oder Spalte fehlt noch
|
||||
|
||||
if "is_active" not in user_columns:
|
||||
await db.execute("ALTER TABLE users ADD COLUMN is_active INTEGER DEFAULT 1")
|
||||
await db.commit()
|
||||
|
||||
if "last_login_at" not in user_columns:
|
||||
await db.execute("ALTER TABLE users ADD COLUMN last_login_at TIMESTAMP")
|
||||
await db.commit()
|
||||
|
||||
# Migration: E-Mail-Benachrichtigungs-Praeferenzen fuer users
|
||||
if "notify_email_summary" not in user_columns:
|
||||
await db.execute("ALTER TABLE users ADD COLUMN notify_email_summary INTEGER DEFAULT 0")
|
||||
await db.execute("ALTER TABLE users ADD COLUMN notify_email_new_articles INTEGER DEFAULT 0")
|
||||
await db.execute("ALTER TABLE users ADD COLUMN notify_email_status_change INTEGER DEFAULT 0")
|
||||
await db.commit()
|
||||
logger.info("Migration: E-Mail-Benachrichtigungs-Spalten zu users hinzugefuegt")
|
||||
|
||||
# Migration: tenant_id fuer articles
|
||||
cursor = await db.execute("PRAGMA table_info(articles)")
|
||||
art_columns = [row[1] for row in await cursor.fetchall()]
|
||||
if "tenant_id" not in art_columns:
|
||||
await db.execute("ALTER TABLE articles ADD COLUMN tenant_id INTEGER REFERENCES organizations(id)")
|
||||
await db.commit()
|
||||
|
||||
# Migration: tenant_id fuer fact_checks
|
||||
cursor = await db.execute("PRAGMA table_info(fact_checks)")
|
||||
fc_columns = [row[1] for row in await cursor.fetchall()]
|
||||
if "tenant_id" not in fc_columns:
|
||||
await db.execute("ALTER TABLE fact_checks ADD COLUMN tenant_id INTEGER REFERENCES organizations(id)")
|
||||
await db.commit()
|
||||
|
||||
# Migration: tenant_id fuer incident_snapshots
|
||||
cursor = await db.execute("PRAGMA table_info(incident_snapshots)")
|
||||
snap_columns2 = [row[1] for row in await cursor.fetchall()]
|
||||
if "tenant_id" not in snap_columns2:
|
||||
await db.execute("ALTER TABLE incident_snapshots ADD COLUMN tenant_id INTEGER REFERENCES organizations(id)")
|
||||
await db.commit()
|
||||
|
||||
# Migration: tenant_id fuer sources
|
||||
cursor = await db.execute("PRAGMA table_info(sources)")
|
||||
src_columns = [row[1] for row in await cursor.fetchall()]
|
||||
if "tenant_id" not in src_columns:
|
||||
await db.execute("ALTER TABLE sources ADD COLUMN tenant_id INTEGER REFERENCES organizations(id)")
|
||||
await db.commit()
|
||||
|
||||
# Migration: tenant_id fuer notifications
|
||||
cursor = await db.execute("PRAGMA table_info(notifications)")
|
||||
notif_columns = [row[1] for row in await cursor.fetchall()]
|
||||
if "tenant_id" not in notif_columns:
|
||||
await db.execute("ALTER TABLE notifications ADD COLUMN tenant_id INTEGER REFERENCES organizations(id)")
|
||||
await db.commit()
|
||||
|
||||
# Indexes erstellen (nach Spalten-Migrationen)
|
||||
for idx_sql in [
|
||||
"CREATE INDEX IF NOT EXISTS idx_incidents_tenant_status ON incidents(tenant_id, status)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_articles_tenant_incident ON articles(tenant_id, incident_id)",
|
||||
]:
|
||||
try:
|
||||
await db.execute(idx_sql)
|
||||
await db.commit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Verwaiste running-Eintraege beim Start als error markieren (aelter als 15 Min)
|
||||
await db.execute(
|
||||
"""UPDATE refresh_log SET status = 'error', error_message = 'Verwaist beim Neustart',
|
||||
completed_at = CURRENT_TIMESTAMP
|
||||
WHERE status = 'running'
|
||||
AND started_at < datetime('now', '-15 minutes')"""
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
# Sources-Tabelle seeden (nur wenn leer)
|
||||
cursor = await db.execute("SELECT COUNT(*) as cnt FROM sources")
|
||||
row = await cursor.fetchone()
|
||||
if row["cnt"] == 0:
|
||||
await _seed_sources(db)
|
||||
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
async def _seed_sources(db: aiosqlite.Connection):
|
||||
"""Befuellt die sources-Tabelle aus der config.py-Konfiguration."""
|
||||
from config import RSS_FEEDS, EXCLUDED_SOURCES
|
||||
|
||||
category_map = {
|
||||
"tagesschau": "oeffentlich-rechtlich",
|
||||
"ZDF heute": "oeffentlich-rechtlich",
|
||||
"Deutsche Welle": "oeffentlich-rechtlich",
|
||||
"Spiegel": "qualitaetszeitung",
|
||||
"Zeit": "qualitaetszeitung",
|
||||
"FAZ": "qualitaetszeitung",
|
||||
"Süddeutsche": "qualitaetszeitung",
|
||||
"NZZ": "qualitaetszeitung",
|
||||
"Reuters": "nachrichtenagentur",
|
||||
"AP News": "nachrichtenagentur",
|
||||
"BBC World": "international",
|
||||
"Al Jazeera": "international",
|
||||
"France24": "international",
|
||||
"BMI": "behoerde",
|
||||
"Europol": "behoerde",
|
||||
}
|
||||
|
||||
for _rss_category, feeds in RSS_FEEDS.items():
|
||||
for feed in feeds:
|
||||
name = feed["name"]
|
||||
url = feed["url"]
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
domain = urlparse(url).netloc.lower().replace("www.", "")
|
||||
except Exception:
|
||||
domain = ""
|
||||
|
||||
category = category_map.get(name, "sonstige")
|
||||
await db.execute(
|
||||
"""INSERT INTO sources (name, url, domain, source_type, category, status, added_by, tenant_id)
|
||||
VALUES (?, ?, ?, 'rss_feed', ?, 'active', 'system', NULL)""",
|
||||
(name, url, domain, category),
|
||||
)
|
||||
|
||||
for excl in EXCLUDED_SOURCES:
|
||||
await db.execute(
|
||||
"""INSERT INTO sources (name, domain, source_type, category, status, added_by, tenant_id)
|
||||
VALUES (?, ?, 'excluded', 'sonstige', 'active', 'system', NULL)""",
|
||||
(excl, excl),
|
||||
)
|
||||
|
||||
await db.commit()
|
||||
await refresh_source_counts(db)
|
||||
|
||||
logger.info(f"Sources-Tabelle geseeded: {len(RSS_FEEDS.get('deutsch', []))+len(RSS_FEEDS.get('international', []))+len(RSS_FEEDS.get('behoerden', []))} RSS-Feeds, {len(EXCLUDED_SOURCES)} ausgeschlossene Quellen")
|
||||
|
||||
|
||||
async def refresh_source_counts(db: aiosqlite.Connection):
|
||||
"""Berechnet Artikelzaehler und last_seen_at fuer alle Quellen neu."""
|
||||
cursor = await db.execute("SELECT id, name, domain FROM sources WHERE source_type != 'excluded'")
|
||||
sources = await cursor.fetchall()
|
||||
|
||||
for source in sources:
|
||||
sid = source["id"]
|
||||
name = source["name"]
|
||||
domain = source["domain"] or ""
|
||||
|
||||
if domain:
|
||||
cursor = await db.execute(
|
||||
"""SELECT COUNT(*) as cnt, MAX(collected_at) as last_seen
|
||||
FROM articles WHERE source = ? OR source_url LIKE ?""",
|
||||
(name, f"%{domain}%"),
|
||||
)
|
||||
else:
|
||||
cursor = await db.execute(
|
||||
"SELECT COUNT(*) as cnt, MAX(collected_at) as last_seen FROM articles WHERE source = ?",
|
||||
(name,),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
await db.execute(
|
||||
"UPDATE sources SET article_count = ?, last_seen_at = ? WHERE id = ?",
|
||||
(row["cnt"], row["last_seen"], sid),
|
||||
)
|
||||
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def db_dependency():
|
||||
"""FastAPI Dependency fuer Datenbankverbindungen."""
|
||||
db = await get_db()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
await db.close()
|
||||
In neuem Issue referenzieren
Einen Benutzer sperren