# Robots.txt for Aragon Metal Traders
# This file tells search engines which pages to crawl and which to avoid

# Rules for all bots
User-agent: *
Allow: /
Allow: /sitemap.html
Allow: /amp.html

# Disallow admin and private areas
Disallow: /admin/
Disallow: /private/
Disallow: /api/
Disallow: /uploads/temp/
Disallow: /admin-resources/

# Disallow sensitive pages
Disallow: /account/
Disallow: /user/profile/
Disallow: /checkout/
Disallow: /cart/

# Allow search engines to crawl your static assets
Allow: /*.js
Allow: /*.css
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.png
Allow: /*.gif
Allow: /*.webp
Allow: /*.svg

# Slow down aggressive crawlers
User-agent: AhrefsBot
Crawl-delay: 10
Request-rate: 1/10s

User-agent: SemrushBot
Crawl-delay: 10
Request-rate: 1/10s

# Specific rules for Googlebot
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Specific rules for Bingbot
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Sitemap location
Sitemap: https://www.aragonmetaltraders.com/sitemap.xml