# General rules for all crawlers
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/
Disallow: /private/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 10
# Sitemap location
Sitemap: https://yourdomain.com/sitemap.xml

# Specific rules for major search engines
User-agent: Googlebot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 5

User-agent: Bingbot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 10

User-agent: Applebot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 15

User-agent: AhrefsBot
Disallow: /
Crawl-delay: 30

User-agent: GPTBot
Disallow: /private/
Crawl-delay: 15

User-agent: AdsBot-Google
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 5