2025-11-04

containers/conf/angie/snippets/bots.conf-example

@@ -0,0 +1,20 @@
+
+	# Use this with maps.conf in nginx.conf	
+	# include this in your vhost/location
+
+	# Return Code 410 is special, it says the content is deleted, indexers *should* their content.
+
+        #see maps.conf
+        if ($bad_client) { return 410; }
+
+        # Tailored for Magento 1 and layered navigation
+        ## Dont let search bots to drill your website | exclude some args from indexing
+        #if ($args ~ ^(brand|cat|color|dir|from|limit|price|type|mode|size|manufacturer)=.+) { set $filters A; }
+        #if ($http_user_agent ~* "http|bot|crawl|spider|Google|bing|Yahoo|Yandex") { set $filters "${filters}B"; }
+        #if ($filters = AB) { return 410; }
+
+	#uncomment this if you don't want to be indexed by google/yahoo/yandex/bing etc
+        #if ($http_user_agent ~* "Bot|Google|Bing|Yahoo|Yandex") {
+        #        return 410;
+        #}
+