# robots.txt Sluurpy # Google: accesso libero # Altri bot: limiti forti Sitemap: https://www.sluurpy.com/sitemap.xml # ============================================ # GOOGLE - Accesso completo senza limiti # ============================================ User-agent: Googlebot Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-News Allow: / User-agent: Googlebot-Video Allow: / # ============================================ # BING - Limitato # ============================================ User-agent: bingbot Crawl-delay: 30 Disallow: /search Disallow: /*?*sort= Disallow: /*?*price= Disallow: /*?*rating= Disallow: /*?*cuisine= Disallow: /*?*establishment= Disallow: /*?*meal_type= User-agent: msnbot Crawl-delay: 30 Disallow: /search Disallow: /*?* # ============================================ # AMAZON - Molto limitato (causa alto traffico) # ============================================ User-agent: Amazonbot Crawl-delay: 60 Disallow: /search Disallow: /*?*sort= Disallow: /*?*price= Disallow: /*?*rating= Disallow: /*?*cuisine= Disallow: /*?*establishment= Disallow: /*?*meal_type= Disallow: /*/photos/* Disallow: /*/reviews Disallow: /*/request-menu # ============================================ # ALTRI BOT COMUNI - Limitati # ============================================ User-agent: YandexBot Crawl-delay: 30 Disallow: /*?* User-agent: Baiduspider Crawl-delay: 60 Disallow: /*?* User-agent: DuckDuckBot Crawl-delay: 20 User-agent: facebookexternalhit Crawl-delay: 10 User-agent: Twitterbot Crawl-delay: 10 User-agent: LinkedInBot Crawl-delay: 10 User-agent: Slurp Crawl-delay: 30 Disallow: /*?* User-agent: Sogou Crawl-delay: 60 Disallow: / User-agent: Exabot Crawl-delay: 60 Disallow: /*?* User-agent: ia_archiver Crawl-delay: 30 User-agent: AhrefsBot Crawl-delay: 60 Disallow: / User-agent: SemrushBot Crawl-delay: 60 Disallow: / User-agent: MJ12bot Crawl-delay: 60 Disallow: / User-agent: DotBot Crawl-delay: 60 Disallow: / User-agent: BLEXBot Crawl-delay: 60 Disallow: / User-agent: PetalBot Crawl-delay: 60 Disallow: / User-agent: DataForSeoBot Crawl-delay: 60 Disallow: / User-agent: GPTBot Crawl-delay: 60 Disallow: / User-agent: CCBot Crawl-delay: 60 Disallow: / User-agent: anthropic-ai Crawl-delay: 60 Disallow: / User-agent: Claude-Web Crawl-delay: 60 Disallow: / User-agent: Bytespider Crawl-delay: 60 Disallow: / # ============================================ # BOT MALEVOLI / SCRAPER - Bloccati # ============================================ User-agent: AhrefsBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: SemrushBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: DotBot Disallow: / User-agent: megaindex Disallow: / User-agent: BacklinkCrawler Disallow: / User-agent: omgili Disallow: / User-agent: Screaming Frog Disallow: / # ============================================ # REGOLA DEFAULT - Altri bot non specificati # ============================================ User-agent: * Crawl-delay: 20 Disallow: /search Disallow: /api/ Disallow: /admin/ Disallow: /*?*sort= Disallow: /*?*price= Disallow: /*?*rating= # ============================================ # SITEMAP (declared at the top) # ============================================