# Allow all crawlers
User-agent: *

# Allow access to all content
Allow: /

# Disallow admin or private areas
Disallow: /admin/
Disallow: /private/
Disallow: /api/

# Sitemap location
Sitemap: https://arpejio.com/sitemap.xml

# Crawl-delay for all bots (optional)
Crawl-delay: 10

# Specific rules for Google bot
User-agent: Googlebot
Allow: /
Allow: /*.js$
Allow: /*.css$

# Specific rules for Bing bot
User-agent: Bingbot
Allow: /
Allow: /*.js$
Allow: /*.css$

# Block specific bots known for aggressive crawling
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /