From d6ba37c8825d47cbb223397c6b1244a9ecd46e4f Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Fri, 29 Nov 2019 17:45:44 -0500
Subject: [PATCH] stats page

---
 config/aggregation.json                       | 89 +++++++++++++++++++
 config/config.exs                             |  1 +
 lib/philomena/elasticsearch.ex                | 17 ++--
 .../controllers/stat_controller.ex            | 65 ++++++++++++++
 lib/philomena_web/router.ex                   |  1 +
 .../templates/stat/index.html.slime           | 76 ++++++++++++++++
 lib/philomena_web/views/stat_view.ex          |  3 +
 .../controllers/stat_controller_test.exs      | 88 ++++++++++++++++++
 8 files changed, 334 insertions(+), 6 deletions(-)
 create mode 100644 config/aggregation.json
 create mode 100644 lib/philomena_web/controllers/stat_controller.ex
 create mode 100644 lib/philomena_web/templates/stat/index.html.slime
 create mode 100644 lib/philomena_web/views/stat_view.ex
 create mode 100644 test/philomena_web/controllers/stat_controller_test.exs

diff --git a/config/aggregation.json b/config/aggregation.json
new file mode 100644
index 00000000..33e006b3
--- /dev/null
+++ b/config/aggregation.json
@@ -0,0 +1,89 @@
+{
+  "comments": {
+    "aggs": {
+      "deleted": {
+        "filter": {
+          "term": {
+            "hidden_from_users": true
+          }
+        }
+      },
+      "last_24h": {
+        "filter": {
+          "range": {
+            "posted_at": {
+              "gt": "now-24h"
+            }
+          }
+        }
+      }
+    }
+  },
+  "images": {
+    "aggs": {
+      "deleted": {
+        "filter": {
+          "term": {
+            "hidden_from_users": true
+          }
+        }
+      },
+      "non_deleted": {
+        "aggs": {
+          "all_time": {
+            "date_histogram": {
+              "field": "created_at",
+              "interval": "day"
+            }
+          },
+          "avg_comments": {
+            "avg": {
+              "field": "comment_count"
+            }
+          },
+          "faves_gt_0": {
+            "filter": {
+              "range": {
+                "faves": {
+                  "gt": 0
+                }
+              }
+            }
+          },
+          "last_24h": {
+            "filter": {
+              "range": {
+                "created_at": {
+                  "gt": "now-24h"
+                }
+              }
+            }
+          },
+          "score_gt_0": {
+            "filter": {
+              "range": {
+                "score": {
+                  "gt": 0
+                }
+              }
+            }
+          },
+          "score_lt_0": {
+            "filter": {
+              "range": {
+                "score": {
+                  "lt": 0
+                }
+              }
+            }
+          }
+        },
+        "filter": {
+          "term": {
+            "hidden_from_users": false
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/config/config.exs b/config/config.exs
index 71946c3d..6b119132 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -22,6 +22,7 @@ config :philomena,
   cdn_host: "",
   proxy_host: nil,
   quick_tags_json: File.read!("config/quick_tag_table.json"),
+  aggregation_json: File.read!("config/aggregation.json"),
   footer_json: File.read!("config/footer.json")
 
 config :philomena, :pow,
diff --git a/lib/philomena/elasticsearch.ex b/lib/philomena/elasticsearch.ex
index 717efc28..a326597b 100644
--- a/lib/philomena/elasticsearch.ex
+++ b/lib/philomena/elasticsearch.ex
@@ -82,19 +82,24 @@ defmodule Philomena.Elasticsearch do
         reindex(ecto_query, batch_size, ids)
       end
 
-      def search_results(elastic_query, pagination_params \\ %{}) do
-        page_number = pagination_params[:page_number] || 1
-        page_size = pagination_params[:page_size] || 25
-        elastic_query = Map.merge(elastic_query, %{from: (page_number - 1) * page_size, size: page_size, _source: false})
-
+      def search(query_body) do
         {:ok, %{body: results, status_code: 200}} =
           Elastix.Search.search(
             unquote(elastic_url),
             unquote(index_name),
             [unquote(doc_type)],
-            elastic_query
+            query_body
           )
 
+        results
+      end
+
+      def search_results(elastic_query, pagination_params \\ %{}) do
+        page_number = pagination_params[:page_number] || 1
+        page_size = pagination_params[:page_size] || 25
+        elastic_query = Map.merge(elastic_query, %{from: (page_number - 1) * page_size, size: page_size, _source: false})
+
+        results = search(elastic_query)
         time = results["took"]
         count = results["hits"]["total"]
         entries = results["hits"]["hits"] |> Enum.map(&String.to_integer(&1["_id"]))
diff --git a/lib/philomena_web/controllers/stat_controller.ex b/lib/philomena_web/controllers/stat_controller.ex
new file mode 100644
index 00000000..b7210034
--- /dev/null
+++ b/lib/philomena_web/controllers/stat_controller.ex
@@ -0,0 +1,65 @@
+defmodule PhilomenaWeb.StatController do
+  use PhilomenaWeb, :controller
+
+  alias Philomena.Images.Image
+  alias Philomena.Comments.Comment
+  alias Philomena.Topics.Topic
+  alias Philomena.Forums.Forum
+  alias Philomena.Posts.Post
+  alias Philomena.Users.User
+  alias Philomena.Repo
+  import Ecto.Query
+
+  def index(conn, _params) do
+    {image_aggs, comment_aggs } = aggregations()
+    {forums, topics, posts} = forums()
+    {users, users_24h} = users()
+
+    render(
+      conn,
+      "index.html",
+      image_aggs: image_aggs,
+      comment_aggs: comment_aggs,
+      forums_count: forums,
+      topics_count: topics,
+      posts_count: posts,
+      users_count: users,
+      users_24h: users_24h
+    )
+  end
+
+  defp aggregations do
+    data =
+      Application.get_env(:philomena, :aggregation_json)
+      |> Jason.decode!()
+
+    {Image.search(data["images"]), Comment.search(data["comments"])}
+  end
+
+  defp forums do
+    forums =
+      Forum
+      |> where(access_level: "normal")
+      |> Repo.aggregate(:count, :id)
+
+    first_topic = Repo.one(first(Topic))
+    last_topic = Repo.one(last(Topic))
+    first_post = Repo.one(first(Post))
+    last_post = Repo.one(last(Post))
+
+    {forums, last_topic.id - first_topic.id, last_post.id - first_post.id}
+  end
+
+  defp users do
+    first_user = Repo.one(first(User))
+    last_user = Repo.one(last(User))
+    time = DateTime.utc_now() |> DateTime.add(-86400, :second)
+
+    last_24h =
+      User
+      |> where([u], u.created_at > ^time)
+      |> Repo.aggregate(:count, :id)
+
+    {last_user.id - first_user.id, last_24h}
+  end
+end
diff --git a/lib/philomena_web/router.ex b/lib/philomena_web/router.ex
index 86f70353..28736d53 100644
--- a/lib/philomena_web/router.ex
+++ b/lib/philomena_web/router.ex
@@ -131,6 +131,7 @@ defmodule PhilomenaWeb.Router do
     resources "/pages", PageController, only: [:show]
     resources "/dnp", DnpEntryController, only: [:index, :show]
     resources "/staff", StaffController, only: [:index]
+    resources "/stats", StatController, only: [:index]
 
     get "/:id", ImageController, :show
     # get "/:forum_id", ForumController, :show # impossible to do without constraints
diff --git a/lib/philomena_web/templates/stat/index.html.slime b/lib/philomena_web/templates/stat/index.html.slime
new file mode 100644
index 00000000..aae4ec9a
--- /dev/null
+++ b/lib/philomena_web/templates/stat/index.html.slime
@@ -0,0 +1,76 @@
+h1 Site Statistics
+
+elixir:
+  img_bucket = @image_aggs["aggregations"]
+  cmt_bucket = @comment_aggs["aggregations"]
+
+.walloftext
+  h3 Images
+  p
+    ' There are
+    span.stat>
+      = number_with_delimiter(img_bucket["non_deleted"]["doc_count"])
+    ' non-deleted images total in our database. Of these,
+    span.stat>
+      = number_with_delimiter(img_bucket["non_deleted"]["last_24h"]["doc_count"])
+    ' images were uploaded in the last 24 hours.
+  p
+    ' This net total excludes the
+    => number_with_delimiter(img_bucket["deleted"]["doc_count"])
+    ' images that have been deleted or marked as duplicates.
+  
+  h3 Comments
+  p
+    ' There are
+    span.stat>
+      = number_with_delimiter(@comment_aggs["hits"]["total"])
+    ' comments on the site. Of these,
+    => number_with_delimiter(cmt_bucket["deleted"]["doc_count"])
+    ' have been deleted.
+  p
+    ' In the last 24 hours,
+    span.stat>
+      = number_with_delimiter(cmt_bucket["last_24h"]["doc_count"])
+    ' comments have been posted.
+  p
+    ' There are, on average,
+    span.stat>
+      = number_with_delimiter(trunc(img_bucket["non_deleted"]["avg_comments"]["value"]))
+    ' comments on each image on the site.
+  
+  h3 Votes
+  p
+    ' Out of
+    => number_with_delimiter(img_bucket["doc_count"])
+    ' images,
+    span.stat>
+      = number_with_delimiter(img_bucket["score_gt_0"]["doc_count"])
+    ' images have a score above 0, and
+    span.stat>
+      = number_with_delimiter(img_bucket["score_lt_0"]["doc_count"])
+    ' images have a score below 0.
+    span.stat>
+      = number_with_delimiter(img_bucket["faves_gt_0"]["doc_count"])
+    ' images have been faved by at least one user.
+
+  h3 Forums
+  p
+    ' In our
+    => @forums_count
+    ' forums there have been
+    span.stat>
+      = number_with_delimiter(@topics_count)
+    ' topics started. There have been
+    span.stat>
+      = number_with_delimiter(@posts_count)
+    ' replies to topics in total.
+
+  h3 Users
+  p
+    ' There are
+    span.stat>
+      = number_with_delimiter(@users_count)
+    ' users on the site. Of these,
+    span.stat>
+      = number_with_delimiter(@users_24h)
+    ' have joined in the last 24 hours.
\ No newline at end of file
diff --git a/lib/philomena_web/views/stat_view.ex b/lib/philomena_web/views/stat_view.ex
new file mode 100644
index 00000000..6ae6adc7
--- /dev/null
+++ b/lib/philomena_web/views/stat_view.ex
@@ -0,0 +1,3 @@
+defmodule PhilomenaWeb.StatView do
+  use PhilomenaWeb, :view
+end
diff --git a/test/philomena_web/controllers/stat_controller_test.exs b/test/philomena_web/controllers/stat_controller_test.exs
new file mode 100644
index 00000000..708ea20d
--- /dev/null
+++ b/test/philomena_web/controllers/stat_controller_test.exs
@@ -0,0 +1,88 @@
+defmodule PhilomenaWeb.StatControllerTest do
+  use PhilomenaWeb.ConnCase
+
+  alias Philomena.Stats
+
+  @create_attrs %{}
+  @update_attrs %{}
+  @invalid_attrs %{}
+
+  def fixture(:stat) do
+    {:ok, stat} = Stats.create_stat(@create_attrs)
+    stat
+  end
+
+  describe "index" do
+    test "lists all stats", %{conn: conn} do
+      conn = get(conn, Routes.stat_path(conn, :index))
+      assert html_response(conn, 200) =~ "Listing Stats"
+    end
+  end
+
+  describe "new stat" do
+    test "renders form", %{conn: conn} do
+      conn = get(conn, Routes.stat_path(conn, :new))
+      assert html_response(conn, 200) =~ "New Stat"
+    end
+  end
+
+  describe "create stat" do
+    test "redirects to show when data is valid", %{conn: conn} do
+      conn = post(conn, Routes.stat_path(conn, :create), stat: @create_attrs)
+
+      assert %{id: id} = redirected_params(conn)
+      assert redirected_to(conn) == Routes.stat_path(conn, :show, id)
+
+      conn = get(conn, Routes.stat_path(conn, :show, id))
+      assert html_response(conn, 200) =~ "Show Stat"
+    end
+
+    test "renders errors when data is invalid", %{conn: conn} do
+      conn = post(conn, Routes.stat_path(conn, :create), stat: @invalid_attrs)
+      assert html_response(conn, 200) =~ "New Stat"
+    end
+  end
+
+  describe "edit stat" do
+    setup [:create_stat]
+
+    test "renders form for editing chosen stat", %{conn: conn, stat: stat} do
+      conn = get(conn, Routes.stat_path(conn, :edit, stat))
+      assert html_response(conn, 200) =~ "Edit Stat"
+    end
+  end
+
+  describe "update stat" do
+    setup [:create_stat]
+
+    test "redirects when data is valid", %{conn: conn, stat: stat} do
+      conn = put(conn, Routes.stat_path(conn, :update, stat), stat: @update_attrs)
+      assert redirected_to(conn) == Routes.stat_path(conn, :show, stat)
+
+      conn = get(conn, Routes.stat_path(conn, :show, stat))
+      assert html_response(conn, 200)
+    end
+
+    test "renders errors when data is invalid", %{conn: conn, stat: stat} do
+      conn = put(conn, Routes.stat_path(conn, :update, stat), stat: @invalid_attrs)
+      assert html_response(conn, 200) =~ "Edit Stat"
+    end
+  end
+
+  describe "delete stat" do
+    setup [:create_stat]
+
+    test "deletes chosen stat", %{conn: conn, stat: stat} do
+      conn = delete(conn, Routes.stat_path(conn, :delete, stat))
+      assert redirected_to(conn) == Routes.stat_path(conn, :index)
+      assert_error_sent 404, fn ->
+        get(conn, Routes.stat_path(conn, :show, stat))
+      end
+    end
+  end
+
+  defp create_stat(_) do
+    stat = fixture(:stat)
+    {:ok, stat: stat}
+  end
+end