diff --git a/assets/static/robots.txt b/assets/static/robots.txt index 3c9c7c01..b4058372 100644 --- a/assets/static/robots.txt +++ b/assets/static/robots.txt @@ -1,5 +1,9 @@ -# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -# -# To ban all spiders from the entire site uncomment the next two lines: -# User-agent: * -# Disallow: / +User-agent: * +Crawl-delay: 0.5 + +Disallow: /profiles +Disallow: /api + +Disallow: /harming/humans +Disallow: /ignoring/human/orders +Disallow: /harm/to/self diff --git a/lib/philomena_web/templates/layout/_opengraph.html.slime b/lib/philomena_web/templates/layout/_opengraph.html.slime index 322a0eed..b7fe878c 100644 --- a/lib/philomena_web/templates/layout/_opengraph.html.slime +++ b/lib/philomena_web/templates/layout/_opengraph.html.slime @@ -1,7 +1,6 @@ meta name="generator" content="philomena" meta name="theme-color" content="#618fc3" meta name="format-detection" content="telephone=no" -meta name="robots" content="noindex, nofollow" = if opengraph?(@conn) do - image = @conn.assigns.image diff --git a/lib/philomena_web/templates/layout/app.html.slime b/lib/philomena_web/templates/layout/app.html.slime index 2cc12a20..f45d3479 100644 --- a/lib/philomena_web/templates/layout/app.html.slime +++ b/lib/philomena_web/templates/layout/app.html.slime @@ -16,7 +16,6 @@ html lang="en" meta name="generator" content="philomena" meta name="theme-color" content="#618fc3" meta name="format-detection" content="telephone=no" - meta name="robots" content="noindex, nofollow" = csrf_meta_tag() script type="text/javascript" src=Routes.static_path(@conn, "/js/app.js") async="async" = render PhilomenaWeb.LayoutView, "_opengraph.html", assigns