# ============================================
# robots.txt voor vinro.nl
# Laatste update: april 2026
# ============================================

# ============================================
# Algemene regels (fallback voor alle overige bots)
# ============================================
User-agent: *
Disallow: /private/
Disallow: /tmp/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /search/
Disallow: /tag/
Disallow: /author/
Disallow: /404-error/
Disallow: /*?s=*
Disallow: /*?p=*
Disallow: /*&p=*
Disallow: /*&preview=*
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 10

# ============================================
# Grote zoekmachines
# ============================================

# Google - negeert Crawl-delay, regelt snelheid via Search Console
User-agent: Googlebot
Disallow: /private/
Disallow: /tmp/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /search/
Disallow: /tag/
Disallow: /author/
Disallow: /404-error/
Disallow: /*?s=*
Disallow: /*?p=*
Disallow: /*&p=*
Disallow: /*&preview=*
Allow: /wp-admin/admin-ajax.php

# Google afbeeldingen
User-agent: Googlebot-Image
Disallow: /private/
Disallow: /tmp/
Allow: /

# Bing - voedt ook DuckDuckGo, Yahoo en ChatGPT Search
User-agent: Bingbot
Disallow: /private/
Disallow: /tmp/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /search/
Disallow: /tag/
Disallow: /author/
Disallow: /404-error/
Disallow: /*?s=*
Disallow: /*?p=*
Disallow: /*&p=*
Disallow: /*&preview=*
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 5

# DuckDuckGo - voor instant-answer functies
User-agent: DuckDuckBot
Disallow: /private/
Disallow: /tmp/
Allow: /
Crawl-delay: 10

# ============================================
# AI-zoekbots (real-time citaties in chat)
# ============================================

# ChatGPT Search - indexering voor zoekresultaten
User-agent: OAI-SearchBot
Disallow: /private/
Disallow: /tmp/
Allow: /

# ChatGPT - haalt live data op bij gebruikersvragen
User-agent: ChatGPT-User
Disallow: /private/
Disallow: /tmp/
Allow: /

# Claude (Anthropic) - hoofdcrawler
User-agent: ClaudeBot
Disallow: /private/
Disallow: /tmp/
Allow: /

# Claude - real-time fetches bij gebruikersvragen
User-agent: Claude-User
Disallow: /private/
Disallow: /tmp/
Allow: /

# Claude Search - indexering voor zoekresultaten
User-agent: Claude-SearchBot
Disallow: /private/
Disallow: /tmp/
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Disallow: /private/
Disallow: /tmp/
Allow: /

# ============================================
# AI-trainingsbots (toegestaan, maar beperkt)
# ============================================

# OpenAI GPTBot - voor training van toekomstige modellen
User-agent: GPTBot
Disallow: /private/
Disallow: /tmp/
Crawl-delay: 20

# Google-Extended - voor training van Gemini
User-agent: Google-Extended
Disallow: /private/
Disallow: /tmp/
Crawl-delay: 15

# Common Crawl - dataset gebruikt door veel AI-modellen
User-agent: CCBot
Disallow: /private/
Disallow: /tmp/
Crawl-delay: 20

# ============================================
# Sitemap
# ============================================
Sitemap: https://www.vinro.nl/sitemap_index.xml
Sitemap: https://www.vinro.nl/page-sitemap.xml