robots config

assets/static/robots.txt

@@ -1,5 +1,9 @@
-# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
-#
-# To ban all spiders from the entire site uncomment the next two lines:
-# User-agent: *
-# Disallow: /
+User-agent: *
+Crawl-delay: 0.5
+
+Disallow: /profiles
+Disallow: /api
+
+Disallow: /harming/humans
+Disallow: /ignoring/human/orders
+Disallow: /harm/to/self

lib/philomena_web/templates/layout/_opengraph.html.slime

@@ -1,7 +1,6 @@
 meta name="generator" content="philomena"
 meta name="theme-color" content="#618fc3"
 meta name="format-detection" content="telephone=no"
-meta name="robots" content="noindex, nofollow"
 
 = if opengraph?(@conn) do
   - image = @conn.assigns.image

lib/philomena_web/templates/layout/app.html.slime

@@ -16,7 +16,6 @@ html lang="en"
     meta name="generator" content="philomena"
     meta name="theme-color" content="#618fc3"
     meta name="format-detection" content="telephone=no"
-    meta name="robots" content="noindex, nofollow"
     = csrf_meta_tag()
     script type="text/javascript" src=Routes.static_path(@conn, "/js/app.js") async="async"
     = render PhilomenaWeb.LayoutView, "_opengraph.html", assigns