From ff54b2f125dada0899ac2b12bf4fa9f496d03762 Mon Sep 17 00:00:00 2001 From: Mischa POSLAWSKY Date: Tue, 2 May 2017 23:35:02 +0200 Subject: [PATCH] robots: host directive to reduce mirror crawling Preferred domain to indicate main site for at least Yandex bots: --- robots.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/robots.txt b/robots.txt index 4ba4148..15b6ecb 100644 --- a/robots.txt +++ b/robots.txt @@ -2,3 +2,4 @@ User-agent: * Disallow: /source/*::* Sitemap: http://sheet.shiar.nl/sitemap.xml +Host: sheet.shiar.nl -- 2.30.0