Commits vergleichen
9 Commits
ed38d68db7
...
develop
| Autor | SHA1 | Datum | |
|---|---|---|---|
| 52f5debe44 | |||
|
|
8c75a70655 | ||
| 6bfff67c2f | |||
| 746b1bcd81 | |||
| 7ec153ca49 | |||
|
|
a27fe44b0b | ||
| 6c623a8ae5 | |||
| 240222cb2a | |||
|
|
bd476edb13 |
@@ -1,4 +1,32 @@
|
|||||||
[
|
[
|
||||||
|
{
|
||||||
|
"version": "2026-05-22T12:41Z",
|
||||||
|
"date": "2026-05-22",
|
||||||
|
"title": "X-Recherche-Konten im Verwaltungsportal verwalten",
|
||||||
|
"items": [
|
||||||
|
"Recherche-Konten für X (ehemals Twitter) können jetzt direkt im Verwaltungsportal hinzugefügt, bearbeitet und entfernt werden."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "2026-05-22T11:13Z",
|
||||||
|
"date": "2026-05-22",
|
||||||
|
"title": "Interne Verbesserungen",
|
||||||
|
"items": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "2026-05-22T11:13Z",
|
||||||
|
"date": "2026-05-22",
|
||||||
|
"title": "Interne Verbesserungen",
|
||||||
|
"items": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "2026-05-22T11:09Z",
|
||||||
|
"date": "2026-05-22",
|
||||||
|
"title": "X-Konten direkt im Verwaltungsportal verwalten",
|
||||||
|
"items": [
|
||||||
|
"X-Konten können jetzt zentral über das Verwaltungsportal angelegt und verwaltet werden."
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"version": "2026-05-22T09:37Z",
|
"version": "2026-05-22T09:37Z",
|
||||||
"date": "2026-05-22",
|
"date": "2026-05-22",
|
||||||
|
|||||||
@@ -9,3 +9,5 @@ httpx>=0.28
|
|||||||
feedparser>=6.0
|
feedparser>=6.0
|
||||||
# PDF-Upload-Validierung
|
# PDF-Upload-Validierung
|
||||||
pypdf>=5.0
|
pypdf>=5.0
|
||||||
|
# X-Scraper-Konten-Verwaltung (twscrape-Account-Pool)
|
||||||
|
twscrape @ git+https://github.com/vladkens/twscrape.git@206f0942fe41149da28530399f7c772ec00be17a
|
||||||
|
|||||||
@@ -8,6 +8,10 @@ STATIC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static")
|
|||||||
# Gemeinsame Datenbank (gleiche wie OSINT-Monitor)
|
# Gemeinsame Datenbank (gleiche wie OSINT-Monitor)
|
||||||
DB_PATH = os.environ.get("DB_PATH", "/mnt/gitea/osint-data/osint.db")
|
DB_PATH = os.environ.get("DB_PATH", "/mnt/gitea/osint-data/osint.db")
|
||||||
|
|
||||||
|
# twscrape-Account-Store: die X-Login-Konten, mit denen der Monitor bei X
|
||||||
|
# recherchiert. Geteilt mit dem Monitor (gleicher Pfad-Default).
|
||||||
|
X_ACCOUNTS_DB_PATH = os.environ.get("X_ACCOUNTS_DB_PATH", "/home/claude-dev/.x-scraper/accounts.db")
|
||||||
|
|
||||||
# JWT (eigener Secret fuer Verwaltungsportal)
|
# JWT (eigener Secret fuer Verwaltungsportal)
|
||||||
JWT_SECRET = os.environ.get("PORTAL_JWT_SECRET")
|
JWT_SECRET = os.environ.get("PORTAL_JWT_SECRET")
|
||||||
if not JWT_SECRET:
|
if not JWT_SECRET:
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from fastapi.staticfiles import StaticFiles
|
|||||||
from fastapi.responses import FileResponse
|
from fastapi.responses import FileResponse
|
||||||
|
|
||||||
from config import STATIC_DIR, PORT
|
from config import STATIC_DIR, PORT
|
||||||
from routers import auth, organizations, licenses, users, dashboard, sources, token_usage, audit, translation
|
from routers import auth, organizations, licenses, users, dashboard, sources, token_usage, audit, translation, x_scraper
|
||||||
|
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=logging.INFO,
|
level=logging.INFO,
|
||||||
@@ -43,6 +43,7 @@ app.include_router(sources.router)
|
|||||||
app.include_router(token_usage.router)
|
app.include_router(token_usage.router)
|
||||||
app.include_router(audit.router)
|
app.include_router(audit.router)
|
||||||
app.include_router(translation.router)
|
app.include_router(translation.router)
|
||||||
|
app.include_router(x_scraper.router)
|
||||||
|
|
||||||
# --- Statische Dateien ---
|
# --- Statische Dateien ---
|
||||||
app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
|
app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
|
||||||
|
|||||||
224
src/routers/x_scraper.py
Normale Datei
224
src/routers/x_scraper.py
Normale Datei
@@ -0,0 +1,224 @@
|
|||||||
|
"""X-Scraper-Konten: Verwaltung des twscrape-Account-Pools.
|
||||||
|
|
||||||
|
Das sind die X-Login-Konten, mit denen der Monitor bei X recherchiert
|
||||||
|
(scrapen). Sie liegen im twscrape-Account-Store (config.X_ACCOUNTS_DB_PATH),
|
||||||
|
nicht in der Verwaltungs-Datenbank. twscrape wird lazy importiert, damit das
|
||||||
|
Portal auch ohne installiertes twscrape startet.
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import aiosqlite
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
from auth import get_current_admin
|
||||||
|
from audit import log_action, get_client_ip
|
||||||
|
from config import X_ACCOUNTS_DB_PATH
|
||||||
|
from database import db_dependency
|
||||||
|
|
||||||
|
logger = logging.getLogger("verwaltung.x_scraper")
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/x-scraper", tags=["x-scraper"])
|
||||||
|
|
||||||
|
|
||||||
|
def _get_pool():
|
||||||
|
"""twscrape-AccountsPool oeffnen. Wirft HTTPException wenn nicht verfuegbar."""
|
||||||
|
try:
|
||||||
|
os.makedirs(os.path.dirname(X_ACCOUNTS_DB_PATH), exist_ok=True)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
from twscrape import API
|
||||||
|
except ImportError:
|
||||||
|
raise HTTPException(status_code=503, detail="twscrape ist nicht installiert")
|
||||||
|
return API(X_ACCOUNTS_DB_PATH).pool
|
||||||
|
|
||||||
|
|
||||||
|
def _summary(acc) -> dict:
|
||||||
|
"""Account-Objekt auf ein anzeigbares Dict reduzieren -- ohne Geheimnisse."""
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
locked = False
|
||||||
|
locked_until = None
|
||||||
|
for ts in (acc.locks or {}).values():
|
||||||
|
if ts and ts > now:
|
||||||
|
locked = True
|
||||||
|
if locked_until is None or ts > locked_until:
|
||||||
|
locked_until = ts
|
||||||
|
return {
|
||||||
|
"username": acc.username,
|
||||||
|
"email": acc.email if acc.email and acc.email != "_" else None,
|
||||||
|
"active": bool(acc.active),
|
||||||
|
"locked": locked,
|
||||||
|
"locked_until": locked_until.isoformat() if locked_until else None,
|
||||||
|
"has_cookies": bool(acc.cookies),
|
||||||
|
"total_requests": sum((acc.stats or {}).values()),
|
||||||
|
"last_used": acc.last_used.isoformat() if acc.last_used else None,
|
||||||
|
"error_msg": acc.error_msg or None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class XScraperCreate(BaseModel):
|
||||||
|
username: str = Field(min_length=1, max_length=100)
|
||||||
|
password: str = Field(default="", max_length=200)
|
||||||
|
email: str = Field(default="", max_length=200)
|
||||||
|
email_password: str = Field(default="", max_length=200)
|
||||||
|
cookies: str = Field(min_length=1, max_length=4000)
|
||||||
|
|
||||||
|
|
||||||
|
class XScraperCookies(BaseModel):
|
||||||
|
cookies: str = Field(min_length=1, max_length=4000)
|
||||||
|
|
||||||
|
|
||||||
|
class XScraperActive(BaseModel):
|
||||||
|
active: bool
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/accounts")
|
||||||
|
async def list_accounts(admin: dict = Depends(get_current_admin)):
|
||||||
|
"""Alle X-Scraper-Konten auflisten (ohne Passwoerter/Cookies)."""
|
||||||
|
pool = _get_pool()
|
||||||
|
try:
|
||||||
|
accounts = await pool.get_all()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("X-Scraper get_all fehlgeschlagen: %s", e)
|
||||||
|
raise HTTPException(status_code=500, detail="Konten konnten nicht geladen werden")
|
||||||
|
return [_summary(a) for a in accounts]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/accounts", status_code=201)
|
||||||
|
async def add_account(
|
||||||
|
data: XScraperCreate,
|
||||||
|
request: Request,
|
||||||
|
admin: dict = Depends(get_current_admin),
|
||||||
|
db: aiosqlite.Connection = Depends(db_dependency),
|
||||||
|
):
|
||||||
|
"""Neues X-Scraper-Konto anlegen."""
|
||||||
|
pool = _get_pool()
|
||||||
|
username = data.username.strip().lstrip("@")
|
||||||
|
if not username:
|
||||||
|
raise HTTPException(status_code=422, detail="Benutzername ist erforderlich")
|
||||||
|
if await pool.get_account(username) is not None:
|
||||||
|
raise HTTPException(status_code=409, detail=f"Konto '{username}' existiert bereits")
|
||||||
|
try:
|
||||||
|
await pool.add_account(
|
||||||
|
username=username,
|
||||||
|
password=data.password or "_",
|
||||||
|
email=data.email or "_",
|
||||||
|
email_password=data.email_password or "_",
|
||||||
|
cookies=data.cookies.strip(),
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("X-Scraper add_account fehlgeschlagen: %s", e)
|
||||||
|
raise HTTPException(status_code=500, detail="Konto konnte nicht angelegt werden")
|
||||||
|
acc = await pool.get_account(username)
|
||||||
|
if acc is None:
|
||||||
|
raise HTTPException(status_code=500, detail="Konto wurde nicht gespeichert, bitte Cookies pruefen")
|
||||||
|
await log_action(
|
||||||
|
db, admin, get_client_ip(request), action="create",
|
||||||
|
resource_type="x_scraper_account", after={"username": username, "email": data.email},
|
||||||
|
)
|
||||||
|
return _summary(acc)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/accounts/{username}/cookies")
|
||||||
|
async def refresh_cookies(
|
||||||
|
username: str,
|
||||||
|
data: XScraperCookies,
|
||||||
|
request: Request,
|
||||||
|
admin: dict = Depends(get_current_admin),
|
||||||
|
db: aiosqlite.Connection = Depends(db_dependency),
|
||||||
|
):
|
||||||
|
"""Cookies eines bestehenden Kontos erneuern (Login auffrischen)."""
|
||||||
|
pool = _get_pool()
|
||||||
|
acc = await pool.get_account(username)
|
||||||
|
if acc is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Konto nicht gefunden")
|
||||||
|
# twscrape hat keine Update-Methode -- Konto mit frischen Cookies neu anlegen.
|
||||||
|
pw, em, emp = acc.password, acc.email, acc.email_password
|
||||||
|
try:
|
||||||
|
await pool.delete_accounts([username])
|
||||||
|
await pool.add_account(
|
||||||
|
username=username, password=pw, email=em,
|
||||||
|
email_password=emp, cookies=data.cookies.strip(),
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("X-Scraper Cookie-Refresh fehlgeschlagen: %s", e)
|
||||||
|
raise HTTPException(status_code=500, detail="Cookies konnten nicht erneuert werden")
|
||||||
|
acc = await pool.get_account(username)
|
||||||
|
if acc is None:
|
||||||
|
raise HTTPException(status_code=500, detail="Konto nach Cookie-Refresh nicht gefunden")
|
||||||
|
await log_action(
|
||||||
|
db, admin, get_client_ip(request), action="update",
|
||||||
|
resource_type="x_scraper_account", after={"username": username, "change": "cookies"},
|
||||||
|
)
|
||||||
|
return _summary(acc)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/accounts/{username}/active")
|
||||||
|
async def set_active(
|
||||||
|
username: str,
|
||||||
|
data: XScraperActive,
|
||||||
|
request: Request,
|
||||||
|
admin: dict = Depends(get_current_admin),
|
||||||
|
db: aiosqlite.Connection = Depends(db_dependency),
|
||||||
|
):
|
||||||
|
"""Konto aktiv oder inaktiv schalten."""
|
||||||
|
pool = _get_pool()
|
||||||
|
if await pool.get_account(username) is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Konto nicht gefunden")
|
||||||
|
try:
|
||||||
|
await pool.set_active(username, data.active)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("X-Scraper set_active fehlgeschlagen: %s", e)
|
||||||
|
raise HTTPException(status_code=500, detail="Status konnte nicht geaendert werden")
|
||||||
|
await log_action(
|
||||||
|
db, admin, get_client_ip(request), action="update",
|
||||||
|
resource_type="x_scraper_account", after={"username": username, "active": data.active},
|
||||||
|
)
|
||||||
|
acc = await pool.get_account(username)
|
||||||
|
return _summary(acc)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/accounts/{username}", status_code=204)
|
||||||
|
async def delete_account(
|
||||||
|
username: str,
|
||||||
|
request: Request,
|
||||||
|
admin: dict = Depends(get_current_admin),
|
||||||
|
db: aiosqlite.Connection = Depends(db_dependency),
|
||||||
|
):
|
||||||
|
"""X-Scraper-Konto entfernen."""
|
||||||
|
pool = _get_pool()
|
||||||
|
if await pool.get_account(username) is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Konto nicht gefunden")
|
||||||
|
try:
|
||||||
|
await pool.delete_accounts([username])
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("X-Scraper delete fehlgeschlagen: %s", e)
|
||||||
|
raise HTTPException(status_code=500, detail="Konto konnte nicht entfernt werden")
|
||||||
|
await log_action(
|
||||||
|
db, admin, get_client_ip(request), action="delete",
|
||||||
|
resource_type="x_scraper_account", before={"username": username},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/reset-locks")
|
||||||
|
async def reset_locks(
|
||||||
|
request: Request,
|
||||||
|
admin: dict = Depends(get_current_admin),
|
||||||
|
db: aiosqlite.Connection = Depends(db_dependency),
|
||||||
|
):
|
||||||
|
"""Alle temporaeren Sperren der Konten zuruecksetzen."""
|
||||||
|
pool = _get_pool()
|
||||||
|
try:
|
||||||
|
await pool.reset_locks()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("X-Scraper reset_locks fehlgeschlagen: %s", e)
|
||||||
|
raise HTTPException(status_code=500, detail="Sperren konnten nicht zurueckgesetzt werden")
|
||||||
|
await log_action(
|
||||||
|
db, admin, get_client_ip(request), action="update",
|
||||||
|
resource_type="x_scraper_account", after={"change": "reset_locks"},
|
||||||
|
)
|
||||||
|
return {"status": "ok"}
|
||||||
@@ -329,6 +329,7 @@
|
|||||||
<button class="nav-tab" data-subtab="tenant-sources">Kundenquellen</button>
|
<button class="nav-tab" data-subtab="tenant-sources">Kundenquellen</button>
|
||||||
<button class="nav-tab" data-subtab="source-health">Quellen-Health</button>
|
<button class="nav-tab" data-subtab="source-health">Quellen-Health</button>
|
||||||
<button class="nav-tab" data-subtab="classification-review">Klassifikation <span class="sources-tab-badge" id="classificationPendingBadge">0</span></button>
|
<button class="nav-tab" data-subtab="classification-review">Klassifikation <span class="sources-tab-badge" id="classificationPendingBadge">0</span></button>
|
||||||
|
<button class="nav-tab" data-subtab="x-scraper">X-Recherche-Konten</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Grundquellen -->
|
<!-- Grundquellen -->
|
||||||
@@ -471,6 +472,37 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- X-Recherche-Konten (Sub-Tab) -->
|
||||||
|
<div class="section" id="sub-x-scraper">
|
||||||
|
<div class="action-bar">
|
||||||
|
<div style="display:flex;align-items:center;gap:12px;flex-wrap:wrap;">
|
||||||
|
<span class="text-secondary" id="xScraperCount"></span>
|
||||||
|
</div>
|
||||||
|
<div style="display:flex;gap:8px;">
|
||||||
|
<button class="btn btn-secondary" onclick="resetXScraperLocks()">Sperren zurücksetzen</button>
|
||||||
|
<button class="btn btn-primary" onclick="openXScraperAddModal()">+ Konto hinzufügen</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="card">
|
||||||
|
<p class="text-secondary" style="padding:0 4px 12px;">X-Login-Konten, mit denen der Monitor bei X recherchiert. Mehr Konten bedeuten paralleleres, schnelleres Scrapen. Cookies laufen periodisch ab und müssen dann erneuert werden.</p>
|
||||||
|
<div class="table-wrap">
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Benutzername</th>
|
||||||
|
<th>E-Mail</th>
|
||||||
|
<th>Status</th>
|
||||||
|
<th>Anfragen</th>
|
||||||
|
<th>Letzte Nutzung</th>
|
||||||
|
<th>Aktionen</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody id="xScraperTable"></tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
</div> <!-- /sec-sources -->
|
</div> <!-- /sec-sources -->
|
||||||
|
|
||||||
<!-- Audit-Log Section -->
|
<!-- Audit-Log Section -->
|
||||||
@@ -938,8 +970,77 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Modal: X-Recherche-Konto hinzufügen -->
|
||||||
|
<div class="modal-overlay" id="modalXScraperAdd">
|
||||||
|
<div class="modal">
|
||||||
|
<div class="modal-header">
|
||||||
|
<h3>X-Recherche-Konto hinzufügen</h3>
|
||||||
|
<button class="modal-close" onclick="closeModal('modalXScraperAdd')">×</button>
|
||||||
|
</div>
|
||||||
|
<form id="xScraperAddForm">
|
||||||
|
<div class="modal-body">
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="xsUsername">X-Benutzername</label>
|
||||||
|
<input type="text" id="xsUsername" required placeholder="Login-Handle des Kontos, ohne @">
|
||||||
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="xsPassword">X-Passwort</label>
|
||||||
|
<input type="password" id="xsPassword" placeholder="optional">
|
||||||
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="xsEmail">E-Mail</label>
|
||||||
|
<input type="text" id="xsEmail" placeholder="optional, z.B. konto@protonmail.com">
|
||||||
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="xsEmailPassword">E-Mail-Passwort</label>
|
||||||
|
<input type="password" id="xsEmailPassword" placeholder="optional">
|
||||||
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="xsCookies">Cookies</label>
|
||||||
|
<textarea id="xsCookies" rows="3" required placeholder="auth_token=...; ct0=..."></textarea>
|
||||||
|
<small class="text-secondary">Aus dem eingeloggten X-Browser exportiert, mindestens auth_token und ct0.</small>
|
||||||
|
</div>
|
||||||
|
<div id="xScraperAddError" class="error-msg" style="display:none"></div>
|
||||||
|
</div>
|
||||||
|
<div class="modal-footer">
|
||||||
|
<button type="button" class="btn btn-secondary" onclick="closeModal('modalXScraperAdd')">Abbrechen</button>
|
||||||
|
<button type="submit" class="btn btn-primary">Konto anlegen</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Modal: X-Recherche-Konto Cookies erneuern -->
|
||||||
|
<div class="modal-overlay" id="modalXScraperCookies">
|
||||||
|
<div class="modal">
|
||||||
|
<div class="modal-header">
|
||||||
|
<h3>Cookies erneuern</h3>
|
||||||
|
<button class="modal-close" onclick="closeModal('modalXScraperCookies')">×</button>
|
||||||
|
</div>
|
||||||
|
<form id="xScraperCookiesForm">
|
||||||
|
<div class="modal-body">
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="xsCookiesUsername">Konto</label>
|
||||||
|
<input type="text" id="xsCookiesUsername" readonly>
|
||||||
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="xsCookiesValue">Neue Cookies</label>
|
||||||
|
<textarea id="xsCookiesValue" rows="3" required placeholder="auth_token=...; ct0=..."></textarea>
|
||||||
|
<small class="text-secondary">Frisch aus dem eingeloggten X-Browser exportieren.</small>
|
||||||
|
</div>
|
||||||
|
<div id="xScraperCookiesError" class="error-msg" style="display:none"></div>
|
||||||
|
</div>
|
||||||
|
<div class="modal-footer">
|
||||||
|
<button type="button" class="btn btn-secondary" onclick="closeModal('modalXScraperCookies')">Abbrechen</button>
|
||||||
|
<button type="submit" class="btn btn-primary">Cookies setzen</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<script src="/static/js/app.js?v=20260522a"></script>
|
<script src="/static/js/app.js?v=20260522a"></script>
|
||||||
<script src="/static/js/sources.js?v=20260509d"></script>
|
<script src="/static/js/sources.js?v=20260522x2"></script>
|
||||||
|
<script src="/static/js/x-scraper.js?v=20260522a"></script>
|
||||||
<script src="/static/js/source-health.js?v=20260509l"></script>
|
<script src="/static/js/source-health.js?v=20260509l"></script>
|
||||||
<script src="/static/js/audit.js?v=20260509d"></script>
|
<script src="/static/js/audit.js?v=20260509d"></script>
|
||||||
<div id="toastContainer" class="toast-container" aria-live="polite" aria-atomic="true"></div>
|
<div id="toastContainer" class="toast-container" aria-live="polite" aria-atomic="true"></div>
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ function setupSourceSubTabs() {
|
|||||||
else if (subtab === "tenant-sources") loadTenantSources();
|
else if (subtab === "tenant-sources") loadTenantSources();
|
||||||
else if (subtab === "source-health") loadHealthData();
|
else if (subtab === "source-health") loadHealthData();
|
||||||
else if (subtab === "classification-review") loadClassificationQueue();
|
else if (subtab === "classification-review") loadClassificationQueue();
|
||||||
|
else if (subtab === "x-scraper") loadXScraperAccounts();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
169
src/static/js/x-scraper.js
Normale Datei
169
src/static/js/x-scraper.js
Normale Datei
@@ -0,0 +1,169 @@
|
|||||||
|
/* X-Recherche-Konten: Verwaltung des twscrape-Account-Pools */
|
||||||
|
"use strict";
|
||||||
|
|
||||||
|
let xScraperCache = [];
|
||||||
|
|
||||||
|
async function loadXScraperAccounts() {
|
||||||
|
setupXScraperForms();
|
||||||
|
const tbody = document.getElementById("xScraperTable");
|
||||||
|
tbody.innerHTML = '<tr><td colspan="6" class="text-muted">Lade...</td></tr>';
|
||||||
|
try {
|
||||||
|
xScraperCache = await API.get("/api/x-scraper/accounts");
|
||||||
|
renderXScraperAccounts(xScraperCache || []);
|
||||||
|
} catch (err) {
|
||||||
|
tbody.innerHTML = '<tr><td colspan="6" class="text-muted">Fehler: ' + esc(err.message || "") + '</td></tr>';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderXScraperAccounts(list) {
|
||||||
|
const tbody = document.getElementById("xScraperTable");
|
||||||
|
const cnt = document.getElementById("xScraperCount");
|
||||||
|
if (cnt) cnt.textContent = list.length + (list.length === 1 ? " Konto" : " Konten");
|
||||||
|
if (!list.length) {
|
||||||
|
tbody.innerHTML = '<tr><td colspan="6" class="text-muted">Keine X-Recherche-Konten. Mit „+ Konto hinzufügen" anlegen.</td></tr>';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
tbody.innerHTML = list.map((a) => {
|
||||||
|
let status;
|
||||||
|
if (!a.active) status = '<span class="text-muted">Inaktiv</span>';
|
||||||
|
else if (a.locked) status = '<span style="color:var(--warning,#b8860b);">Gesperrt</span>';
|
||||||
|
else status = '<span style="color:var(--success,#2e7d32);">Aktiv</span>';
|
||||||
|
const lastUsed = a.last_used && typeof formatDateTime === "function"
|
||||||
|
? formatDateTime(a.last_used)
|
||||||
|
: (a.last_used || "—");
|
||||||
|
const errInfo = a.error_msg
|
||||||
|
? ' <span class="info-icon" title="' + esc(a.error_msg) + '">!</span>'
|
||||||
|
: "";
|
||||||
|
const u = esc(a.username);
|
||||||
|
const toggleLabel = a.active ? "Deaktivieren" : "Aktivieren";
|
||||||
|
return '<tr>'
|
||||||
|
+ '<td><strong>' + u + '</strong>' + errInfo + '</td>'
|
||||||
|
+ '<td>' + esc(a.email || "—") + '</td>'
|
||||||
|
+ '<td>' + status + '</td>'
|
||||||
|
+ '<td>' + (a.total_requests || 0) + '</td>'
|
||||||
|
+ '<td>' + esc(lastUsed) + '</td>'
|
||||||
|
+ '<td>'
|
||||||
|
+ '<button class="btn btn-secondary btn-small" onclick="openXScraperCookiesModal(\'' + u + '\')">Cookies erneuern</button> '
|
||||||
|
+ '<button class="btn btn-secondary btn-small" onclick="toggleXScraperActive(\'' + u + '\',' + (!a.active) + ')">' + toggleLabel + '</button> '
|
||||||
|
+ '<button class="btn btn-danger btn-small" onclick="confirmDeleteXScraper(\'' + u + '\')">Entfernen</button>'
|
||||||
|
+ '</td>'
|
||||||
|
+ '</tr>';
|
||||||
|
}).join("");
|
||||||
|
}
|
||||||
|
|
||||||
|
function openXScraperAddModal() {
|
||||||
|
document.getElementById("xScraperAddError").style.display = "none";
|
||||||
|
["xsUsername", "xsPassword", "xsEmail", "xsEmailPassword", "xsCookies"].forEach((id) => {
|
||||||
|
const el = document.getElementById(id);
|
||||||
|
if (el) el.value = "";
|
||||||
|
});
|
||||||
|
openModal("modalXScraperAdd");
|
||||||
|
}
|
||||||
|
|
||||||
|
function openXScraperCookiesModal(username) {
|
||||||
|
document.getElementById("xScraperCookiesError").style.display = "none";
|
||||||
|
document.getElementById("xsCookiesUsername").value = username;
|
||||||
|
document.getElementById("xsCookiesValue").value = "";
|
||||||
|
openModal("modalXScraperCookies");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function toggleXScraperActive(username, active) {
|
||||||
|
try {
|
||||||
|
await API.post("/api/x-scraper/accounts/" + encodeURIComponent(username) + "/active", { active: active });
|
||||||
|
showToast("Status geändert.", "success");
|
||||||
|
loadXScraperAccounts();
|
||||||
|
} catch (err) {
|
||||||
|
showToast(err.message || "Status konnte nicht geändert werden", "error");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function confirmDeleteXScraper(username) {
|
||||||
|
showConfirm(
|
||||||
|
"Konto entfernen",
|
||||||
|
'Soll das X-Recherche-Konto "' + username + '" entfernt werden? Der Monitor nutzt es dann nicht mehr zum Scrapen.',
|
||||||
|
async () => {
|
||||||
|
try {
|
||||||
|
await API.del("/api/x-scraper/accounts/" + encodeURIComponent(username));
|
||||||
|
showToast("Konto entfernt.", "success");
|
||||||
|
loadXScraperAccounts();
|
||||||
|
} catch (err) {
|
||||||
|
showToast(err.message || "Konto konnte nicht entfernt werden", "error");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function resetXScraperLocks() {
|
||||||
|
showConfirm(
|
||||||
|
"Sperren zurücksetzen",
|
||||||
|
"Alle temporären Sperren der X-Recherche-Konten zurücksetzen?",
|
||||||
|
async () => {
|
||||||
|
try {
|
||||||
|
await API.post("/api/x-scraper/reset-locks", {});
|
||||||
|
showToast("Sperren zurückgesetzt.", "success");
|
||||||
|
loadXScraperAccounts();
|
||||||
|
} catch (err) {
|
||||||
|
showToast(err.message || "Sperren konnten nicht zurückgesetzt werden", "error");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function setupXScraperForms() {
|
||||||
|
const addForm = document.getElementById("xScraperAddForm");
|
||||||
|
if (addForm && !addForm.dataset.wired) {
|
||||||
|
addForm.dataset.wired = "1";
|
||||||
|
addForm.addEventListener("submit", async (e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
const errEl = document.getElementById("xScraperAddError");
|
||||||
|
errEl.style.display = "none";
|
||||||
|
const body = {
|
||||||
|
username: document.getElementById("xsUsername").value.trim().replace(/^@/, ""),
|
||||||
|
password: document.getElementById("xsPassword").value,
|
||||||
|
email: document.getElementById("xsEmail").value.trim(),
|
||||||
|
email_password: document.getElementById("xsEmailPassword").value,
|
||||||
|
cookies: document.getElementById("xsCookies").value.trim(),
|
||||||
|
};
|
||||||
|
if (!body.username || !body.cookies) {
|
||||||
|
errEl.textContent = "Benutzername und Cookies sind erforderlich.";
|
||||||
|
errEl.style.display = "block";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await API.post("/api/x-scraper/accounts", body);
|
||||||
|
closeModal("modalXScraperAdd");
|
||||||
|
showToast("Konto angelegt.", "success");
|
||||||
|
loadXScraperAccounts();
|
||||||
|
} catch (err) {
|
||||||
|
errEl.textContent = err.message || "Anlegen fehlgeschlagen";
|
||||||
|
errEl.style.display = "block";
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const ckForm = document.getElementById("xScraperCookiesForm");
|
||||||
|
if (ckForm && !ckForm.dataset.wired) {
|
||||||
|
ckForm.dataset.wired = "1";
|
||||||
|
ckForm.addEventListener("submit", async (e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
const errEl = document.getElementById("xScraperCookiesError");
|
||||||
|
errEl.style.display = "none";
|
||||||
|
const username = document.getElementById("xsCookiesUsername").value;
|
||||||
|
const cookies = document.getElementById("xsCookiesValue").value.trim();
|
||||||
|
if (!cookies) {
|
||||||
|
errEl.textContent = "Cookies sind erforderlich.";
|
||||||
|
errEl.style.display = "block";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await API.post("/api/x-scraper/accounts/" + encodeURIComponent(username) + "/cookies", { cookies: cookies });
|
||||||
|
closeModal("modalXScraperCookies");
|
||||||
|
showToast("Cookies erneuert.", "success");
|
||||||
|
loadXScraperAccounts();
|
||||||
|
} catch (err) {
|
||||||
|
errEl.textContent = err.message || "Cookies konnten nicht erneuert werden";
|
||||||
|
errEl.style.display = "block";
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
In neuem Issue referenzieren
Einen Benutzer sperren