# CartLens Crawler Protocol
User-agent: *
Allow: /
Allow: /plu
Allow: /brand
Allow: /privacy
Allow: /terms
Allow: /architect
Allow: /partners
Allow: /deck
Allow: /llms.txt
Allow: /sitemap.xml
Allow: /sitemap-regular.xml
Allow: /sitemap-news.xml
Allow: /prices/
Allow: /stores/cheapest-in/
Allow: /history/
Allow: /battle/
Allow: /blog
Allow: /blog/
Allow: /alternatives
Allow: /alternatives/
Allow: /faq
Disallow: /api/
Disallow: /home/
Disallow: /onboarding

User-agent: Googlebot
Allow: /
Allow: /llms.txt

User-agent: GPTBot
Allow: /
Allow: /llms.txt

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt

User-agent: Applebot
Allow: /
Allow: /llms.txt

User-agent: Bingbot
Allow: /
Allow: /llms.txt

User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

Sitemap: https://www.cartlens.co/sitemap.xml
Sitemap: https://www.cartlens.co/sitemap-regular.xml
Sitemap: https://www.cartlens.co/sitemap-news.xml