User-agent: *
Allow: /
Allow: /compare/
Allow: /categories/
Allow: /about
Allow: /ads.txt
Disallow: /api/
Disallow: /control-panel/
Disallow: /admin/
Disallow: /_next/
Disallow: /test/
Disallow: /test-anchors/
Disallow: /visual-dimensions-demo/

# Crawl-delay for polite crawling
Crawl-delay: 1

# Sitemaps
Sitemap: https://www.letscompare.co/sitemap.xml

# Specific bot instructions
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: facebookexternalhit
Allow: /