025ddfcebd
- noindex/nofollow von 12 indexierbaren Seiten entfernt (Hauptseiten DE/EN, 3 Lagen DE/EN, Legal DE/EN) - robots.txt scharf geschaltet: Crawling allgemein erlaubt, Live-Search-AI-Bots (OAI-SearchBot, ChatGPT-User, ClaudeBot, PerplexityBot) erlaubt, Training-Bots (GPTBot, CCBot, anthropic-ai, Google-Extended, Applebot-Extended, Bytespider, ...) geblockt - sitemap.xml: Inhalt aus sitemap-launch.xml uebernommen, mit Sitemap-Verweis in robots.txt - Lagen-Seiten (3 DE + 3 EN): description, canonical, Open Graph, Twitter Card und Schema.org Article ergaenzt - Lagen-Hero: Topic-Default in <p id="incident-title"> als Crawler-Fallback (JS ueberschreibt mit Datum bei Lade) - CLAUDE.md CHANGE_LOG ergaenzt
113 Zeilen
2.4 KiB
Plaintext
113 Zeilen
2.4 KiB
Plaintext
# robots.txt - AegisSight UG
|
|
# Crawling allgemein erlaubt, ausser API/interne Pfade
|
|
# Keine Trainingsdaten-Verwendung durch AI-Crawler (Training-Bots geblockt)
|
|
# Live-Search-AI-Bots (OAI-SearchBot, ChatGPT-User, ClaudeBot, PerplexityBot) sind erlaubt
|
|
|
|
User-agent: *
|
|
Allow: /
|
|
Disallow: /api/
|
|
Disallow: /_archiv/
|
|
Disallow: /insights/
|
|
|
|
# Sitemap
|
|
Sitemap: https://aegis-sight.de/sitemap.xml
|
|
|
|
# ----------------------------------------------------------------------
|
|
# AI-Training-Crawler -- BLOCKED (kein Training auf unseren Inhalten)
|
|
# ----------------------------------------------------------------------
|
|
User-agent: GPTBot
|
|
Disallow: /
|
|
|
|
User-agent: CCBot
|
|
Disallow: /
|
|
|
|
User-agent: anthropic-ai
|
|
Disallow: /
|
|
|
|
User-agent: Claude-Web
|
|
Disallow: /
|
|
|
|
User-agent: Google-Extended
|
|
Disallow: /
|
|
|
|
User-agent: Applebot-Extended
|
|
Disallow: /
|
|
|
|
User-agent: Meta-ExternalAgent
|
|
Disallow: /
|
|
|
|
User-agent: Bytespider
|
|
Disallow: /
|
|
|
|
User-agent: cohere-ai
|
|
Disallow: /
|
|
|
|
User-agent: FacebookBot
|
|
Disallow: /
|
|
|
|
User-agent: ImagesiftBot
|
|
Disallow: /
|
|
|
|
User-agent: Diffbot
|
|
Disallow: /
|
|
|
|
User-agent: Omgilibot
|
|
Disallow: /
|
|
|
|
# ----------------------------------------------------------------------
|
|
# AI-Live-Search-Crawler -- ALLOWED (Sichtbarkeit in KI-Antworten)
|
|
# OAI-SearchBot, ChatGPT-User, ClaudeBot, PerplexityBot werden NICHT
|
|
# blockiert. Sie crawlen fuer Live-Antworten, nicht fuer Training.
|
|
# ----------------------------------------------------------------------
|
|
|
|
# ----------------------------------------------------------------------
|
|
# Archiv-Bots
|
|
# ----------------------------------------------------------------------
|
|
User-agent: ia_archiver
|
|
Disallow: /
|
|
|
|
User-agent: archive.org_bot
|
|
Disallow: /
|
|
|
|
# ----------------------------------------------------------------------
|
|
# SEO-/Spam-Crawler
|
|
# ----------------------------------------------------------------------
|
|
User-agent: AhrefsBot
|
|
Disallow: /
|
|
|
|
User-agent: SemrushBot
|
|
Disallow: /
|
|
|
|
User-agent: MJ12bot
|
|
Disallow: /
|
|
|
|
User-agent: DotBot
|
|
Disallow: /
|
|
|
|
User-agent: SEOkicks-Robot
|
|
Disallow: /
|
|
|
|
User-agent: MauiBot
|
|
Disallow: /
|
|
|
|
User-agent: Majestic-12
|
|
Disallow: /
|
|
|
|
User-agent: BLEXBot
|
|
Disallow: /
|
|
|
|
User-agent: SerendeputyBot
|
|
Disallow: /
|
|
|
|
# ----------------------------------------------------------------------
|
|
# Download-Manager
|
|
# ----------------------------------------------------------------------
|
|
User-agent: HTTrack
|
|
Disallow: /
|
|
|
|
User-agent: SiteSnagger
|
|
Disallow: /
|
|
|
|
User-agent: WebCopier
|
|
Disallow: /
|