# robots.txt file for http://www.goyellow.de/
# mail info@goyellow.de for constructive criticism

User-agent: *
Disallow: /sendMail/
Disallow: /ox/
Disallow: /IDA2/
Disallow: /schnellsuche/
Disallow: /upgrade/
Disallow: /branchenbuch/angebote/
Disallow: /telefonnummer/
Disallow: /trackbrowser.jsp
Disallow: /suche/*vw=map
Disallow: /suche/*st=topbewertungen
Disallow: /suche/*st=inderumgebung
Disallow: /suche/*st=jetztoffen

User-agent: Sogou web spider
Disallow: /

User-agent: WinHttp.WinHttpRequest
Disallow: /

User-agent: YandexBot/3.0
Disallow: /

user-agent: yacybot
disallow: / 

user-agent: unisterbot
disallow: / 

user-agent: ia_archiver
disallow: / 

user-agent: mail.ru_bot/2.0
disallow: / 

# Untersagt KI-Training durch bestimmte Bots

# — On-Demand / Assistenz-Bots erlauben (keine Trainingssammlung) ---
# OpenAI Live-Fetch
User-agent: ChatGPT-User
Allow: /

# Anthropic Live-Fetch
User-agent: Claude-User
Allow: /

# Perplexity Live-Fetch (on-demand, not for training)
User-agent: Perplexity-User
Allow: /

# Mistral Live-Fetch (on-demand, not for training)
User-agent: MistralAI-User
Allow: /

# (Optional) Anthropic Search-Bot für bessere Zitationen zulassen
User-agent: Claude-SearchBot
Allow: /

# — KI-TRAINING und Massenscrapes untersagen ---
# OpenAI Trainingscrawler
User-agent: GPTBot
Disallow: /

# Google Opt-out fürs KI-Training (kein Crawler; SEO bleibt)
User-agent: Google-Extended
Disallow: /

# Apple Opt-out fürs KI-Training (kein Crawler; SEO bleibt)
User-agent: Applebot-Extended
Disallow: /

# Anthropic Trainingscrawler
User-agent: ClaudeBot
Disallow: /

# Common Crawl (häufige Quelle für Trainingsdaten)
User-agent: CCBot
Disallow: /

# Meta Trainings-/Index-Bot für AI
User-agent: meta-externalagent
Disallow: /

# Perplexity allgemeiner Crawler (nicht der on-demand 'Perplexity-User')
User-agent: PerplexityBot
Disallow: /