# For all robots User-agent: * # Block access to specific groups of pages Disallow: /stock/* Disallow: /bag Disallow: /checkout Disallow: /my-account Disallow: //?event= Disallow: //?brandid= Disallow: //?categoryID= Disallow: //?categoryid= Disallow: /bin Disallow: /t/* Disallow: /t/ Disallow: /index.cfm Disallow: /product/ Disallow: */zoomImages Disallow: /styleguide Disallow: /search # Disallow favourite sorted pages (removed LH 020517) # Disallow favourite PPC pages Disallow: /ppc/* # Allow search crawlers to discover the sitemap Sitemap: https://www.beaverbrooks.co.uk/sitemap-product.xml Sitemap: https://www.beaverbrooks.co.uk/sitemap-category.xml Sitemap: https://www.beaverbrooks.co.uk/sitemap-static.xml Sitemap: https://www.beaverbrooks.co.uk/sitemap-image.xml # Block CazoodleBot as it does not present correct accept content headers User-agent: CazoodleBot Disallow: / # Block MJ12bot as it is just noise User-agent: MJ12bot Disallow: / # Block dotbot as it cannot parse base urls properly User-agent: dotbot/1.0 Disallow: / # Block Gigabot User-agent: Gigabot Disallow: /