From aa4350596e37ab5fab01eb3e47a771f704b31a17 Mon Sep 17 00:00:00 2001 From: Dagan Bronstein Date: Sun, 27 Sep 2015 15:56:58 +0300 Subject: [PATCH] Added Crawl-Delay --- public/robots.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/public/robots.txt b/public/robots.txt index 9e60f970..f2aa1616 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,2 +1,3 @@ User-agent: * Disallow: +Crawl-Delay: 10 \ No newline at end of file