# robots.txt for djbobcarpenter.com
# Purpose: Be Google-friendly and explicitly allow ChatGPT/OpenAI crawlers.
# Last updated: 2025-08-28

##### OpenAI / ChatGPT crawlers
# ChatGPT Search indexer
User-agent: OAI-SearchBot
Allow: /
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /admin/
Disallow: /private/
Disallow: /vendor/
Disallow: /scripts/
Disallow: /includes/

# OpenAI training crawler
User-agent: GPTBot
Allow: /
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /admin/
Disallow: /private/
Disallow: /vendor/
Disallow: /scripts/
Disallow: /includes/

# ChatGPT web-browsing agent
User-agent: ChatGPT-User
Allow: /
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /admin/
Disallow: /private/
Disallow: /vendor/
Disallow: /scripts/
Disallow: /includes/

##### Major search engines
User-agent: Googlebot
Allow: /
Disallow: /includes/
Disallow: /cgi-bin/
User-agent: Googlebot-Image
Allow: /
User-agent: Googlebot-Video
Allow: /
User-agent: AdsBot-Google
Allow: /
User-agent: Bingbot
Allow: /
Disallow: /includes/
User-agent: Applebot
Allow: /
User-agent: DuckDuckBot
Allow: /

##### Catch-all (everything else)
User-agent: *
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /admin/
Disallow: /private/
Disallow: /vendor/
Disallow: /scripts/
Disallow: /includes/
Allow: /

##### Sitemaps
Sitemap: https://www.djbobcarpenter.com/sitemap.xml
Sitemap: https://www.djbobcarpenter.com/sitemap.php