Merge pull request #341 from philomena-dev/deadweight-elimination-v2

Remove referrer, user_agent, and other unused fields

index/posts.mk

@@ -21,8 +21,8 @@ metadata: post_search_json
 			'body', p.body,
 			'subject', t.title,
 			'ip', p.ip,
-			'user_agent', p.user_agent,
-			'referrer', p.referrer,
+			'user_agent', '',
+			'referrer', '',
 			'fingerprint', p.fingerprint,
 			'topic_position', p.topic_position,
 			'forum', f.short_name,

lib/philomena/artist_links/artist_link.ex

@@ -13,8 +13,6 @@ defmodule Philomena.ArtistLinks.ArtistLink do
 
     field :aasm_state, :string, default: "unverified"
     field :uri, :string
-    field :hostname, :string
-    field :path, :string
     field :verification_code, :string
     field :public, :boolean, default: true
     field :next_check_at, :utc_datetime
@@ -35,7 +33,6 @@ defmodule Philomena.ArtistLinks.ArtistLink do
     |> cast(attrs, [:uri, :public])
     |> put_change(:tag_id, nil)
     |> validate_required([:user, :uri, :public])
-    |> parse_uri()
   end
 
   def edit_changeset(artist_link, attrs, tag) do
@@ -43,7 +40,6 @@ defmodule Philomena.ArtistLinks.ArtistLink do
     |> cast(attrs, [:uri, :public])
     |> put_change(:tag_id, tag.id)
     |> validate_required([:user, :uri, :public])
-    |> parse_uri()
   end
 
   def creation_changeset(artist_link, attrs, user, tag) do
@@ -55,7 +51,6 @@ defmodule Philomena.ArtistLinks.ArtistLink do
     |> validate_required([:tag], message: "must exist")
     |> validate_format(:uri, ~r|\Ahttps?://|)
     |> validate_category()
-    |> parse_uri()
     |> put_verification_code()
     |> put_next_check_at()
     |> unique_constraint([:uri, :tag_id, :user_id],
@@ -95,14 +90,6 @@ defmodule Philomena.ArtistLinks.ArtistLink do
     |> put_change(:aasm_state, "contacted")
   end
 
-  defp parse_uri(changeset) do
-    string_uri = get_field(changeset, :uri) |> to_string()
-    uri = URI.parse(string_uri)
-
-    changeset
-    |> change(hostname: uri.host, path: uri.path)
-  end
-
   defp put_verification_code(changeset) do
     code = :crypto.strong_rand_bytes(5) |> Base.encode16()
     change(changeset, verification_code: "DERPI-LINKVALIDATION-#{code}")

lib/philomena/channels/channel.ex

@@ -12,7 +12,6 @@ defmodule Philomena.Channels.Channel do
 
     field :short_name, :string
     field :title, :string, default: ""
-    field :tags, :string
     field :viewers, :integer, default: 0
     field :nsfw, :boolean, default: false
     field :is_live, :boolean, default: false
@@ -20,16 +19,6 @@ defmodule Philomena.Channels.Channel do
     field :next_check_at, :utc_datetime
     field :last_live_at, :utc_datetime
 
-    field :viewer_minutes_today, :integer, default: 0
-    field :viewer_minutes_thisweek, :integer, default: 0
-    field :viewer_minutes_thismonth, :integer, default: 0
-    field :total_viewer_minutes, :integer, default: 0
-
-    field :banner_image, :string
-    field :channel_image, :string
-    field :remote_stream_id, :integer
-    field :thumbnail_url, :string, default: ""
-
     timestamps(inserted_at: :created_at, type: :utc_datetime)
   end
 

lib/philomena/comments/comment.ex

@@ -14,15 +14,12 @@ defmodule Philomena.Comments.Comment do
     field :body, :string
     field :ip, EctoNetwork.INET
     field :fingerprint, :string
-    field :user_agent, :string, default: ""
-    field :referrer, :string, default: ""
     field :anonymous, :boolean, default: false
     field :hidden_from_users, :boolean, default: false
     field :edit_reason, :string
     field :edited_at, :utc_datetime
     field :deletion_reason, :string, default: ""
     field :destroyed_content, :boolean, default: false
-    field :name_at_post_time, :string
     field :approved, :boolean
 
     timestamps(inserted_at: :created_at, type: :utc_datetime)
@@ -35,7 +32,6 @@ defmodule Philomena.Comments.Comment do
     |> validate_required([:body])
     |> validate_length(:body, min: 1, max: 300_000, count: :bytes)
     |> change(attribution)
-    |> put_name_at_post_time(attribution[:user])
     |> Approval.maybe_put_approval(attribution[:user])
     |> Approval.maybe_strip_images(attribution[:user])
   end
@@ -74,7 +70,4 @@ defmodule Philomena.Comments.Comment do
     change(comment)
     |> put_change(:approved, true)
   end
-
-  defp put_name_at_post_time(changeset, nil), do: changeset
-  defp put_name_at_post_time(changeset, user), do: change(changeset, name_at_post_time: user.name)
 end

lib/philomena/images.ex

@@ -95,11 +95,6 @@ defmodule Philomena.Images do
 
     Multi.new()
     |> Multi.insert(:image, image)
-    |> Multi.run(:name_caches, fn repo, %{image: image} ->
-      image
-      |> Image.cache_changeset()
-      |> repo.update()
-    end)
     |> Multi.run(:added_tag_count, fn repo, %{image: image} ->
       tag_ids = image.added_tags |> Enum.map(& &1.id)
       tags = Tag |> where([t], t.id in ^tag_ids)
@@ -385,8 +380,6 @@ defmodule Philomena.Images do
       updated_at: now,
       ip: attribution[:ip],
       fingerprint: attribution[:fingerprint],
-      user_agent: attribution[:user_agent],
-      referrer: attribution[:referrer],
       added: added
     }
   end
@@ -522,8 +515,6 @@ defmodule Philomena.Images do
       tag_name_cache: tag.name,
       ip: attribution[:ip],
       fingerprint: attribution[:fingerprint],
-      user_agent: attribution[:user_agent],
-      referrer: attribution[:referrer],
       added: added
     }
   end

lib/philomena/images/image.ex

@@ -2,7 +2,6 @@ defmodule Philomena.Images.Image do
   use Ecto.Schema
 
   import Ecto.Changeset
-  import Ecto.Query
 
   alias Philomena.ImageIntensities.ImageIntensity
   alias Philomena.ImageVotes.ImageVote
@@ -59,14 +58,11 @@ defmodule Philomena.Images.Image do
     field :image_is_animated, :boolean, source: :is_animated
     field :ip, EctoNetwork.INET
     field :fingerprint, :string
-    field :user_agent, :string, default: ""
-    field :referrer, :string, default: ""
     field :anonymous, :boolean, default: false
     field :score, :integer, default: 0
     field :faves_count, :integer, default: 0
     field :upvotes_count, :integer, default: 0
     field :downvotes_count, :integer, default: 0
-    field :votes_count, :integer, default: 0
     field :source_url, :string
     field :description, :string, default: ""
     field :image_sha512_hash, :string
@@ -88,11 +84,6 @@ defmodule Philomena.Images.Image do
     field :hides_count, :integer, default: 0
     field :approved, :boolean
 
-    # todo: can probably remove these now
-    field :tag_list_cache, :string
-    field :tag_list_plus_alias_cache, :string
-    field :file_name_cache, :string
-
     field :removed_tags, {:array, :any}, default: [], virtual: true
     field :added_tags, {:array, :any}, default: [], virtual: true
     field :removed_sources, {:array, :any}, default: [], virtual: true
@@ -228,7 +219,6 @@ defmodule Philomena.Images.Image do
     |> cast(attrs, [])
     |> TagDiffer.diff_input(old_tags, new_tags, excluded_tags)
     |> TagValidator.validate_tags()
-    |> cache_changeset()
   end
 
   def locked_tags_changeset(image, attrs, locked_tags) do
@@ -345,53 +335,6 @@ defmodule Philomena.Images.Image do
     |> put_change(:first_seen_at, DateTime.utc_now(:second))
   end
 
-  def cache_changeset(image) do
-    changeset = change(image)
-    image = apply_changes(changeset)
-
-    {tag_list_cache, tag_list_plus_alias_cache, file_name_cache} =
-      create_caches(image.id, image.tags)
-
-    changeset
-    |> put_change(:tag_list_cache, tag_list_cache)
-    |> put_change(:tag_list_plus_alias_cache, tag_list_plus_alias_cache)
-    |> put_change(:file_name_cache, file_name_cache)
-  end
-
-  defp create_caches(image_id, tags) do
-    tags = Tag.display_order(tags)
-
-    tag_list_cache =
-      tags
-      |> Enum.map_join(", ", & &1.name)
-
-    tag_ids = tags |> Enum.map(& &1.id)
-
-    aliases =
-      Tag
-      |> where([t], t.aliased_tag_id in ^tag_ids)
-      |> Repo.all()
-
-    tag_list_plus_alias_cache =
-      (tags ++ aliases)
-      |> Tag.display_order()
-      |> Enum.map_join(", ", & &1.name)
-
-    # Truncate filename to 150 characters, making room for the path + filename on Windows
-    # https://stackoverflow.com/questions/265769/maximum-filename-length-in-ntfs-windows-xp-and-windows-vista
-    file_name_slug_fragment =
-      tags
-      |> Enum.map_join("_", & &1.slug)
-      |> String.to_charlist()
-      |> Enum.filter(&(&1 in ?a..?z or &1 in ~c"0123456789_-"))
-      |> List.to_string()
-      |> String.slice(0..150)
-
-    file_name_cache = "#{image_id}__#{file_name_slug_fragment}"
-
-    {tag_list_cache, tag_list_plus_alias_cache, file_name_cache}
-  end
-
   defp create_key do
     Base.encode16(:crypto.strong_rand_bytes(6), case: :lower)
   end

lib/philomena/polls/poll.ex

@@ -3,20 +3,16 @@ defmodule Philomena.Polls.Poll do
   import Ecto.Changeset
 
   alias Philomena.Topics.Topic
-  alias Philomena.Users.User
   alias Philomena.PollOptions.PollOption
 
   schema "polls" do
     belongs_to :topic, Topic
-    belongs_to :deleted_by, User
     has_many :options, PollOption
 
     field :title, :string
     field :vote_method, :string
     field :active_until, PhilomenaQuery.Ecto.RelativeDate
     field :total_votes, :integer, default: 0
-    field :hidden_from_users, :boolean, default: false
-    field :deletion_reason, :string, default: ""
 
     timestamps(inserted_at: :created_at, type: :utc_datetime)
   end

lib/philomena/posts/post.ex

@@ -15,15 +15,12 @@ defmodule Philomena.Posts.Post do
     field :edit_reason, :string
     field :ip, EctoNetwork.INET
     field :fingerprint, :string
-    field :user_agent, :string, default: ""
-    field :referrer, :string, default: ""
     field :topic_position, :integer
     field :hidden_from_users, :boolean, default: false
     field :anonymous, :boolean, default: false
     field :edited_at, :utc_datetime
     field :deletion_reason, :string, default: ""
     field :destroyed_content, :boolean, default: false
-    field :name_at_post_time, :string
     field :approved, :boolean, default: false
 
     timestamps(inserted_at: :created_at, type: :utc_datetime)
@@ -47,7 +44,6 @@ defmodule Philomena.Posts.Post do
     |> validate_required([:body])
     |> validate_length(:body, min: 1, max: 300_000, count: :bytes)
     |> change(attribution)
-    |> put_name_at_post_time(attribution[:user])
     |> Approval.maybe_put_approval(attribution[:user])
     |> Approval.maybe_strip_images(attribution[:user])
   end
@@ -61,7 +57,6 @@ defmodule Philomena.Posts.Post do
     |> validate_length(:body, min: 1, max: 300_000, count: :bytes)
     |> change(attribution)
     |> change(topic_position: 0)
-    |> put_name_at_post_time(attribution[:user])
     |> Approval.maybe_put_approval(attribution[:user])
     |> Approval.maybe_strip_images(attribution[:user])
   end
@@ -90,7 +85,4 @@ defmodule Philomena.Posts.Post do
     change(post)
     |> put_change(:approved, true)
   end
-
-  defp put_name_at_post_time(changeset, nil), do: changeset
-  defp put_name_at_post_time(changeset, user), do: change(changeset, name_at_post_time: user.name)
 end

lib/philomena/posts/search_index.ex

@@ -52,8 +52,8 @@ defmodule Philomena.Posts.SearchIndex do
       author: if(!!post.user and !post.anonymous, do: String.downcase(post.user.name)),
       subject: post.topic.title,
       ip: post.ip |> to_string(),
-      user_agent: post.user_agent,
-      referrer: post.referrer,
+      user_agent: "",
+      referrer: "",
       fingerprint: post.fingerprint,
       topic_position: post.topic_position,
       forum: post.topic.forum.short_name,

lib/philomena/reports.ex

@@ -139,9 +139,7 @@ defmodule Philomena.Reports do
     attributes = %{
       system: true,
       ip: %Postgrex.INET{address: {127, 0, 0, 1}, netmask: 32},
-      fingerprint: "ffff",
-      user_agent:
-        "Mozilla/5.0 (X11; Philomena; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0"
+      fingerprint: "ffff"
     }
 
     %Report{reportable_type: reportable_type, reportable_id: reportable_id}

lib/philomena/reports/report.ex

@@ -11,7 +11,6 @@ defmodule Philomena.Reports.Report do
     field :ip, EctoNetwork.INET
     field :fingerprint, :string
     field :user_agent, :string, default: ""
-    field :referrer, :string, default: ""
     field :reason, :string
     field :state, :string, default: "open"
     field :open, :boolean, default: true
@@ -61,8 +60,9 @@ defmodule Philomena.Reports.Report do
   @doc false
   def creation_changeset(report, attrs, attribution) do
     report
-    |> cast(attrs, [:category, :reason])
+    |> cast(attrs, [:category, :reason, :user_agent])
     |> validate_length(:reason, max: 10_000, count: :bytes)
+    |> validate_length(:user_agent, max: 1000, count: :bytes)
     |> merge_category()
     |> change(attribution)
     |> validate_required([

lib/philomena/roles/role.ex

@@ -4,12 +4,7 @@ defmodule Philomena.Roles.Role do
 
   schema "roles" do
     field :name, :string
-
-    # fixme: rails polymorphic relation
-    field :resource_id, :integer
     field :resource_type, :string
-
-    timestamps(inserted_at: :created_at, type: :utc_datetime)
   end
 
   @doc false

lib/philomena/source_changes/source_change.ex

@@ -8,8 +8,6 @@ defmodule Philomena.SourceChanges.SourceChange do
 
     field :ip, EctoNetwork.INET
     field :fingerprint, :string
-    field :user_agent, :string, default: ""
-    field :referrer, :string, default: ""
     field :value, :string
     field :added, :boolean
 

lib/philomena/tag_changes/tag_change.ex

@@ -9,8 +9,6 @@ defmodule Philomena.TagChanges.TagChange do
 
     field :ip, EctoNetwork.INET
     field :fingerprint, :string
-    field :user_agent, :string, default: ""
-    field :referrer, :string, default: ""
     field :added, :boolean
     field :tag_name_cache, :string, default: ""
 

lib/philomena/users/eraser.ex

@@ -115,9 +115,7 @@ defmodule Philomena.Users.Eraser do
     attribution = [
       user: user,
       ip: @wipe_ip,
-      fingerprint: @wipe_fp,
-      user_agent: "",
-      referrer: ""
+      fingerprint: @wipe_fp
     ]
 
     {:ok, _} = Images.update_sources(source_change.image, attribution, attrs)

lib/philomena/users/user.ex

@@ -109,7 +109,6 @@ defmodule Philomena.Users.User do
     field :watched_tag_list, :string, virtual: true
 
     # Other stuff
-    field :last_donation_at, :utc_datetime
     field :last_renamed_at, :utc_datetime
     field :deleted_at, :utc_datetime
     field :scratchpad, :string

lib/philomena_web/controllers/admin/batch/tag_controller.ex

@@ -37,8 +37,6 @@ defmodule PhilomenaWeb.Admin.Batch.TagController do
     attributes = %{
       ip: attributes[:ip],
       fingerprint: attributes[:fingerprint],
-      user_agent: attributes[:user_agent],
-      referrer: attributes[:referrer],
       user_id: attributes[:user].id
     }
 

lib/philomena_web/controllers/tag_change/full_revert_controller.ex

@@ -11,8 +11,6 @@ defmodule PhilomenaWeb.TagChange.FullRevertController do
     attributes = %{
       ip: to_string(attributes[:ip]),
       fingerprint: attributes[:fingerprint],
-      referrer: attributes[:referrer],
-      user_agent: attributes[:referrer],
       user_id: attributes[:user].id,
       batch_size: attributes[:batch_size] || 100
     }

lib/philomena_web/controllers/tag_change/revert_controller.ex

@@ -13,8 +13,6 @@ defmodule PhilomenaWeb.TagChange.RevertController do
     attributes = %{
       ip: attributes[:ip],
       fingerprint: attributes[:fingerprint],
-      referrer: attributes[:referrer],
-      user_agent: attributes[:referrer],
       user_id: attributes[:user].id
     }
 

lib/philomena_web/plugs/user_attribution_plug.ex

@@ -24,9 +24,7 @@ defmodule PhilomenaWeb.UserAttributionPlug do
     attributes = [
       ip: remote_ip,
       fingerprint: fingerprint(conn, conn.path_info),
-      referrer: referrer(conn.assigns.referrer),
-      user: user,
-      user_agent: user_agent(conn)
+      user: user
     ]
 
     conn
@@ -47,7 +45,4 @@ defmodule PhilomenaWeb.UserAttributionPlug do
   defp fingerprint(conn, _) do
     conn.cookies["_ses"]
   end
-
-  defp referrer(nil), do: nil
-  defp referrer(r), do: String.slice(r, 0, 255)
 end

lib/philomena_web/templates/report/new.html.slime

@@ -47,6 +47,7 @@ p
     .block
       = render PhilomenaWeb.MarkdownView, "_input.html", conn: @conn, f: f, placeholder: "Provide anything else we should know here.", name: :reason, required: false
 
+    = hidden_input f, :user_agent, value: get_user_agent(@conn)
     = render PhilomenaWeb.CaptchaView, "_captcha.html", name: "report", conn: @conn
 
     = submit "Send Report", class: "button"

lib/philomena_web/templates/topic/poll/_display.html.slime

@@ -6,27 +6,13 @@
     = link "Administrate", to: "#", data: [click_tab: "administration"]
 
 .block__tab data-tab="voting"
-  = cond do
-    - @poll.hidden_from_users ->
-      .walloftext
-      .block.block--fixed.block--warning
-        h1 This poll has been deleted
-        p
-          ' Reason:
-          strong
-            = @poll.deletion_reason || "Unknown (likely deleted in error). Please contact a moderator."
-
-    - @poll_active and not @voted and not is_nil(@conn.assigns.current_user) ->
   .poll
     .poll-area
+      = if @poll_active and not @voted and not is_nil(@conn.assigns.current_user) do
         = render PhilomenaWeb.Topic.PollView, "_vote_form.html", assigns
-
-    - true ->
-      .poll
-        .poll-area
+      - else
         = render PhilomenaWeb.Topic.PollView, "_results.html", assigns
 
-
 = if can?(@conn, :hide, @topic) do
   .block__tab.hidden data-tab="voters"
     p Loading…

lib/philomena_web/views/report_view.ex

@@ -79,4 +79,11 @@ defmodule PhilomenaWeb.ReportView do
   def link_to_reported_thing(_reportable) do
     "Reported item permanently destroyed."
   end
+
+  def get_user_agent(conn) do
+    case Plug.Conn.get_req_header(conn, "user-agent") do
+      [ua] -> ua
+      _ -> ""
+    end
+  end
 end

lib/philomena_web/views/topic/poll_view.ex

@@ -13,7 +13,7 @@ defmodule PhilomenaWeb.Topic.PollView do
   end
 
   def active?(poll) do
-    not poll.hidden_from_users and DateTime.diff(poll.active_until, DateTime.utc_now()) > 0
+    DateTime.diff(poll.active_until, DateTime.utc_now()) > 0
   end
 
   def require_answer?(%{vote_method: vote_method}), do: vote_method == "single"

priv/repo/seeds_development.exs

@@ -40,8 +40,6 @@ pleb = Repo.get_by!(User, name: "Pleb")
 request_attributes = [
   fingerprint: "c1836832948",
   ip: ip,
-  user_agent: "Hopefully not IE",
-  referrer: "localhost",
   user_id: pleb.id,
   user: pleb
 ]