From 0e5de7aaa2f5cfa727104f8bed82ab348237293c Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Sun, 24 Nov 2019 13:36:21 -0500 Subject: [PATCH] source/tag change model setup --- lib/philomena/images.ex | 94 +++++++++++++--- lib/philomena/images/image.ex | 29 ++++- lib/philomena/images/tag_differ.ex | 91 +++++++++++++++ lib/philomena/images/tag_validator.ex | 92 ++++++++++++++++ lib/philomena/source_changes.ex | 13 --- lib/philomena/source_changes/source_change.ex | 9 ++ lib/philomena/tags.ex | 63 +++++++++-- lib/philomena/tags/tag.ex | 104 ++++++++++++++++++ .../controllers/image/source_controller.ex | 32 ++++++ .../controllers/image/tag_controller.ex | 32 ++++++ lib/philomena_web/plugs/captcha_plug.ex | 26 +++++ 11 files changed, 547 insertions(+), 38 deletions(-) create mode 100644 lib/philomena/images/tag_differ.ex create mode 100644 lib/philomena/images/tag_validator.ex create mode 100644 lib/philomena_web/controllers/image/source_controller.ex create mode 100644 lib/philomena_web/controllers/image/tag_controller.ex create mode 100644 lib/philomena_web/plugs/captcha_plug.ex diff --git a/lib/philomena/images.ex b/lib/philomena/images.ex index 2b5031c4..91a92b06 100644 --- a/lib/philomena/images.ex +++ b/lib/philomena/images.ex @@ -4,22 +4,14 @@ defmodule Philomena.Images do """ import Ecto.Query, warn: false + + alias Ecto.Multi alias Philomena.Repo alias Philomena.Images.Image - - @doc """ - Returns the list of images. - - ## Examples - - iex> list_images() - [%Image{}, ...] - - """ - def list_images do - Repo.all(Image |> where(hidden_from_users: false) |> order_by(desc: :created_at) |> limit(25)) - end + alias Philomena.SourceChanges.SourceChange + alias Philomena.TagChanges.TagChange + alias Philomena.Tags @doc """ Gets a single image. @@ -75,6 +67,82 @@ defmodule Philomena.Images do |> Repo.update() end + def update_source(%Image{} = image, attribution, attrs) do + image_changes = + image + |> Image.source_changeset(attrs) + + source_changes = + Ecto.build_assoc(image, :source_changes) + |> SourceChange.creation_changeset(attrs, attribution) + + Multi.new + |> Multi.update(:image, image_changes) + |> Multi.insert(:source_change, source_changes) + |> Repo.isolated_transaction(:serializable) + end + + def update_tags(%Image{} = image, attribution, attrs) do + old_tags = Tags.get_or_create_tags(attrs["old_tag_input"]) + new_tags = Tags.get_or_create_tags(attrs["tag_input"]) + + Multi.new + |> Multi.run(:image, fn repo, _chg -> + image + |> repo.preload(:tags, force: true) + |> Image.tag_changeset(%{}, old_tags, new_tags) + |> repo.update() + |> case do + {:ok, image} -> + {:ok, {image, image.added_tags, image.removed_tags}} + + error -> + error + end + end) + |> Multi.run(:added_tag_changes, fn repo, %{image: {image, added_tags, _removed}} -> + tag_changes = + added_tags + |> Enum.map(&tag_change_attributes(attribution, image, &1, true, attribution[:user])) + + {count, nil} = repo.insert_all(TagChange, tag_changes) + + {:ok, count} + end) + |> Multi.run(:removed_tag_changes, fn repo, %{image: {image, _added, removed_tags}} -> + tag_changes = + removed_tags + |> Enum.map(&tag_change_attributes(attribution, image, &1, false, attribution[:user])) + + {count, nil} = repo.insert_all(TagChange, tag_changes) + + {:ok, count} + end) + |> Repo.isolated_transaction(:serializable) + end + + defp tag_change_attributes(attribution, image, tag, added, user) do + now = DateTime.utc_now() |> DateTime.truncate(:second) + user_id = + case user do + nil -> nil + user -> user.id + end + + %{ + image_id: image.id, + tag_id: tag.id, + user_id: user_id, + created_at: now, + tag_name_cache: tag.name, + ip: attribution[:ip], + fingerprint: attribution[:fingerprint], + user_agent: attribution[:user_agent], + referrer: attribution[:referrer], + added: added + } + end + @doc """ Deletes a Image. diff --git a/lib/philomena/images/image.ex b/lib/philomena/images/image.ex index 4dfa90e0..0d4e8819 100644 --- a/lib/philomena/images/image.ex +++ b/lib/philomena/images/image.ex @@ -13,9 +13,14 @@ defmodule Philomena.Images.Image do alias Philomena.ImageHides.ImageHide alias Philomena.Images.Subscription alias Philomena.Users.User - alias Philomena.Images.Tagging + alias Philomena.Tags.Tag alias Philomena.Galleries alias Philomena.Comments.Comment + alias Philomena.SourceChanges.SourceChange + alias Philomena.TagChanges.TagChange + + alias Philomena.Images.TagDiffer + alias Philomena.Images.TagValidator schema "images" do belongs_to :user, User @@ -25,14 +30,15 @@ defmodule Philomena.Images.Image do has_many :downvotes, ImageVote, where: [up: false] has_many :faves, ImageFave has_many :hides, ImageHide - has_many :taggings, Tagging has_many :gallery_interactions, Galleries.Interaction has_many :subscriptions, Subscription - has_many :tags, through: [:taggings, :tag] + has_many :source_changes, SourceChange + has_many :tag_changes, TagChange has_many :upvoters, through: [:upvotes, :user] has_many :downvoters, through: [:downvotes, :user] has_many :favers, through: [:faves, :user] has_many :hiders, through: [:hides, :user] + many_to_many :tags, Tag, join_through: "image_taggings", on_replace: :delete field :image, :string field :image_name, :string @@ -78,6 +84,9 @@ defmodule Philomena.Images.Image do field :tag_list_plus_alias_cache, :string field :file_name_cache, :string + field :removed_tags, {:array, :any}, default: [], virtual: true + field :added_tags, {:array, :any}, default: [], virtual: true + timestamps(inserted_at: :created_at) end @@ -96,4 +105,18 @@ defmodule Philomena.Images.Image do downvotes: image.downvotes_count } end + + def source_changeset(image, attrs) do + image + |> cast(attrs, [:source_url]) + |> validate_required(:source_url) + |> validate_format(:source_url, ~r/\Ahttps?:\/\//) + end + + def tag_changeset(image, attrs, old_tags, new_tags) do + image + |> cast(attrs, []) + |> TagDiffer.diff_input(old_tags, new_tags) + |> TagValidator.validate_tags() + end end diff --git a/lib/philomena/images/tag_differ.ex b/lib/philomena/images/tag_differ.ex new file mode 100644 index 00000000..17dd0ea0 --- /dev/null +++ b/lib/philomena/images/tag_differ.ex @@ -0,0 +1,91 @@ +defmodule Philomena.Images.TagDiffer do + import Ecto.Changeset + import Ecto.Query + + alias Philomena.Tags.Tag + alias Philomena.Repo + + def diff_input(changeset, old_tags, new_tags) do + old_set = to_set(old_tags) + new_set = to_set(new_tags) + + tags = changeset |> get_field(:tags) + added_tags = added_set(old_set, new_set) + removed_tags = removed_set(old_set, new_set) + + {tags, actually_added, actually_removed} = + apply_changes(tags, added_tags, removed_tags) + + changeset + |> put_change(:added_tags, actually_added) + |> put_change(:removed_tags, actually_removed) + |> put_assoc(:tags, tags) + end + + defp added_set(old_set, new_set) do + # new_tags - old_tags + added_set = + new_set + |> Map.drop(Map.keys(old_set)) + + implied_set = + added_set + |> Enum.flat_map(fn {_k, v} -> v.implied_tags end) + |> List.flatten() + |> to_set() + + added_and_implied_set = + Map.merge(added_set, implied_set) + + oc_set = + added_and_implied_set + |> Enum.filter(fn {_k, v} -> v.namespace == "oc" end) + |> get_oc_tag() + + Map.merge(added_and_implied_set, oc_set) + end + + defp removed_set(old_set, new_set) do + # old_tags - new_tags + old_set |> Map.drop(Map.keys(new_set)) + end + + defp get_oc_tag([]), do: Map.new() + defp get_oc_tag(_any_oc_tag) do + Tag + |> where(name: "oc") + |> Repo.all() + |> to_set() + end + + defp to_set(tags) do + tags |> Map.new(&{&1.id, &1}) + end + + defp to_tag_list(set) do + set |> Enum.map(fn {_k, v} -> v end) + end + + defp apply_changes(tags, added_set, removed_set) do + tag_set = tags |> to_set() + + desired_tags = + tag_set + |> Map.drop(Map.keys(removed_set)) + |> Map.merge(added_set) + + actually_added = + desired_tags + |> Map.drop(Map.keys(tag_set)) + + actually_removed = + tag_set + |> Map.drop(Map.keys(desired_tags)) + + tags = desired_tags |> to_tag_list() + actually_added = actually_added |> to_tag_list() + actually_removed = actually_removed |> to_tag_list() + + {tags, actually_added, actually_removed} + end +end \ No newline at end of file diff --git a/lib/philomena/images/tag_validator.ex b/lib/philomena/images/tag_validator.ex new file mode 100644 index 00000000..88de8145 --- /dev/null +++ b/lib/philomena/images/tag_validator.ex @@ -0,0 +1,92 @@ +defmodule Philomena.Images.TagValidator do + import Ecto.Changeset + + @safe_rating MapSet.new(["safe"]) + @sexual_ratings MapSet.new(["suggestive", "questionable", "explicit"]) + @horror_ratings MapSet.new(["semi-grimdark", "grimdark"]) + @gross_rating MapSet.new(["grotesque"]) + @empty MapSet.new() + + def validate_tags(changeset) do + tags = changeset |> get_field(:tags) + + validate_tag_input(changeset, tags) + end + + defp validate_tag_input(changeset, tags) do + tag_set = extract_names(tags) + rating_set = ratings(tag_set) + + changeset + |> validate_number_of_tags(tag_set, 3) + |> validate_has_rating(rating_set) + |> validate_safe(rating_set) + |> validate_sexual_exclusion(rating_set) + |> validate_horror_exclusion(rating_set) + end + + defp ratings(%MapSet{} = tag_set) do + safe = MapSet.intersection(tag_set, @safe_rating) + sexual = MapSet.intersection(tag_set, @sexual_ratings) + horror = MapSet.intersection(tag_set, @horror_ratings) + gross = MapSet.intersection(tag_set, @gross_rating) + + %{ + safe: safe, + sexual: sexual, + horror: horror, + gross: gross + } + end + + defp validate_number_of_tags(changeset, tag_set, num) do + cond do + MapSet.size(tag_set) < num -> + changeset + |> add_error(:tag_input, "must contain at least #{num} tags") + + true -> + changeset + end + end + + defp validate_has_rating(changeset, %{safe: s, sexual: x, horror: h, gross: g}) when s == @empty and x == @empty and h == @empty and g == @empty do + changeset + |> add_error(:tag_input, "must contain at least one rating tag") + end + defp validate_has_rating(changeset, _ratings), do: changeset + + defp validate_safe(changeset, %{safe: s, sexual: x, horror: h, gross: g}) when s != @empty and (x != @empty or h != @empty or g != @empty) do + changeset + |> add_error(:tag_input, "may not contain any other rating if safe") + end + defp validate_safe(changeset, _ratings), do: changeset + + defp validate_sexual_exclusion(changeset, %{sexual: x}) do + cond do + MapSet.size(x) > 1 -> + changeset + |> add_error(:tag_input, "may contain at most one sexual rating") + + true -> + changeset + end + end + + defp validate_horror_exclusion(changeset, %{horror: h}) do + cond do + MapSet.size(h) > 1 -> + changeset + |> add_error(:tag_input, "may contain at most one grim rating") + + true -> + changeset + end + end + + defp extract_names(tags) do + tags + |> Enum.map(& &1.name) + |> MapSet.new() + end +end \ No newline at end of file diff --git a/lib/philomena/source_changes.ex b/lib/philomena/source_changes.ex index 30e6b057..ebbb6e5a 100644 --- a/lib/philomena/source_changes.ex +++ b/lib/philomena/source_changes.ex @@ -8,19 +8,6 @@ defmodule Philomena.SourceChanges do alias Philomena.SourceChanges.SourceChange - @doc """ - Returns the list of source_changes. - - ## Examples - - iex> list_source_changes() - [%SourceChange{}, ...] - - """ - def list_source_changes do - Repo.all(SourceChange) - end - @doc """ Gets a single source_change. diff --git a/lib/philomena/source_changes/source_change.ex b/lib/philomena/source_changes/source_change.ex index f46383d3..e9c97990 100644 --- a/lib/philomena/source_changes/source_change.ex +++ b/lib/philomena/source_changes/source_change.ex @@ -13,6 +13,8 @@ defmodule Philomena.SourceChanges.SourceChange do field :new_value, :string field :initial, :boolean, default: false + field :source_url, :string, source: :new_value + timestamps(inserted_at: :created_at) end @@ -22,4 +24,11 @@ defmodule Philomena.SourceChanges.SourceChange do |> cast(attrs, []) |> validate_required([]) end + + @doc false + def creation_changeset(source_change, attrs, attribution) do + source_change + |> cast(attrs, [:source_url]) + |> change(attribution) + end end diff --git a/lib/philomena/tags.ex b/lib/philomena/tags.ex index 5b9ff477..6196742e 100644 --- a/lib/philomena/tags.ex +++ b/lib/philomena/tags.ex @@ -8,17 +8,39 @@ defmodule Philomena.Tags do alias Philomena.Tags.Tag - @doc """ - Returns the list of tags. + @spec get_or_create_tags(String.t()) :: List.t() + def get_or_create_tags(tag_list) do + tag_names = Tag.parse_tag_list(tag_list) - ## Examples + existent_tags = + Tag + |> where([t], t.name in ^tag_names) + |> preload(:implied_tags) + |> Repo.all() - iex> list_tags() - [%Tag{}, ...] + existent_tag_names = + existent_tags + |> Map.new(&{&1.name, true}) - """ - def list_tags do - Repo.all(Tag |> order_by(desc: :images_count) |> limit(250)) + nonexistent_tag_names = + tag_names + |> Enum.reject(&existent_tag_names[&1]) + + new_tags = + nonexistent_tag_names + |> Enum.map(fn name -> + {:ok, tag} = + %Tag{} + |> Tag.creation_changeset(%{name: name}) + |> Repo.insert() + + %{tag | implied_tags: []} + end) + + new_tags + |> reindex_tags() + + existent_tags ++ new_tags end @doc """ @@ -51,7 +73,7 @@ defmodule Philomena.Tags do """ def create_tag(attrs \\ %{}) do %Tag{} - |> Tag.changeset(attrs) + |> Tag.creation_changeset(attrs) |> Repo.insert() end @@ -102,6 +124,29 @@ defmodule Philomena.Tags do Tag.changeset(tag, %{}) end + def reindex_tag(%Tag{} = tag) do + reindex_tags([%Tag{id: tag.id}]) + end + + def reindex_tags(tags) do + spawn fn -> + ids = + tags + |> Enum.map(& &1.id) + + Tag + |> preload(^indexing_preloads()) + |> where([t], t.id in ^ids) + |> Tag.reindex() + end + + tags + end + + def indexing_preloads do + [:aliased_tag, :aliases, :implied_tags, :implied_by_tags] + end + alias Philomena.Tags.Implication @doc """ diff --git a/lib/philomena/tags/tag.ex b/lib/philomena/tags/tag.ex index 8c691fb3..32fec576 100644 --- a/lib/philomena/tags/tag.ex +++ b/lib/philomena/tags/tag.ex @@ -8,6 +8,26 @@ defmodule Philomena.Tags.Tag do doc_type: "tag" alias Philomena.Tags.Tag + alias Philomena.Slug + + @namespaces [ + "artist", + "art pack", + "ask", + "blog", + "colorist", + "comic", + "editor", + "fanfic", + "oc", + "parent", + "parents", + "photographer", + "series", + "species", + "spoiler", + "video" + ] schema "tags" do belongs_to :aliased_tag, Tag, source: :aliased_tag_id @@ -37,4 +57,88 @@ defmodule Philomena.Tags.Tag do |> cast(attrs, []) |> validate_required([]) end + + @doc false + def creation_changeset(tag, attrs) do + tag + |> cast(attrs, [:name]) + |> validate_required([:name]) + |> put_slug() + |> put_name_and_namespace() + end + + def parse_tag_list(list) do + list + |> to_string() + |> String.split(",") + |> Enum.map(&clean_tag_name/1) + |> Enum.reject(&"" == &1) + end + + def clean_tag_name(name) do + # Downcase, replace extra runs of spaces, replace unicode quotes + # with ascii quotes, trim space from end + name + |> String.downcase() + |> String.replace(~r/[[:space:]]+/, " ") + |> String.replace(~r/[\x{00b4}\x{2018}\x{2019}\x{201a}\x{201b}\x{2032}]/u, "'") + |> String.replace(~r/[\x{201c}\x{201d}\x{201e}\x{201f}\x{2033}]/u, "\"") + |> String.trim() + |> clean_tag_namespace() + |> ununderscore() + end + + defp clean_tag_namespace(name) do + # Remove extra spaces after the colon in a namespace + # (artist:, oc:, etc.) + name + |> String.split(":", parts: 2) + |> Enum.map(&String.trim/1) + |> join_namespace_parts(name) + end + + defp join_namespace_parts([_name], original_name), + do: original_name + defp join_namespace_parts([namespace, name], _original_name) when namespace in @namespaces, + do: namespace <> ":" <> name + defp join_namespace_parts([_namespace, _name], original_name), + do: original_name + + defp ununderscore(<<"artist:", _rest::binary>> = name), + do: name + defp ununderscore(name), + do: String.replace(name, "_", " ") + + defp put_slug(changeset) do + slug = + changeset + |> get_field(:name) + |> to_string() + |> Slug.slug() + + changeset + |> change(slug: slug) + end + + defp put_name_and_namespace(changeset) do + {namespace, name_in_namespace} = + changeset + |> get_field(:name) + |> to_string() + |> extract_name_and_namespace() + + changeset + |> change(namespace: namespace) + |> change(name_in_namespace: name_in_namespace) + end + + defp extract_name_and_namespace(name) do + case String.split(name, ":", parts: 2) do + [namespace, name_in_namespace] when namespace in @namespaces -> + {namespace, name_in_namespace} + + _value -> + {nil, name} + end + end end diff --git a/lib/philomena_web/controllers/image/source_controller.ex b/lib/philomena_web/controllers/image/source_controller.ex new file mode 100644 index 00000000..9a0586cd --- /dev/null +++ b/lib/philomena_web/controllers/image/source_controller.ex @@ -0,0 +1,32 @@ +defmodule PhilomenaWeb.Image.SourceController do + use PhilomenaWeb, :controller + + alias Philomena.Images + alias Philomena.Images.Image + + plug PhilomenaWeb.FilterBannedUsersPlug + plug PhilomenaWeb.CaptchaPlug + plug PhilomenaWeb.UserAttributionPlug + plug PhilomenaWeb.CanaryMapPlug, update: :show + plug :load_and_authorize_resource, model: Image, id_name: "image_id" + + def update(conn, %{"image" => image_params}) do + attributes = conn.assigns.attributes + image = conn.assigns.image + + case Images.update_source(image, attributes, image_params) do + {:ok, %{image: image}} -> + changeset = + Images.change_image(image) + + conn + |> put_view(PhilomenaWeb.ImageView) + |> render("_source.html", image: image, changeset: changeset) + + {:error, :image, changeset, _} -> + conn + |> put_view(PhilomenaWeb.ImageView) + |> render("_source.html", image: image, changeset: changeset) + end + end +end \ No newline at end of file diff --git a/lib/philomena_web/controllers/image/tag_controller.ex b/lib/philomena_web/controllers/image/tag_controller.ex new file mode 100644 index 00000000..a9339d10 --- /dev/null +++ b/lib/philomena_web/controllers/image/tag_controller.ex @@ -0,0 +1,32 @@ +defmodule PhilomenaWeb.Image.TagController do + use PhilomenaWeb, :controller + + alias Philomena.Images + alias Philomena.Images.Image + + plug PhilomenaWeb.FilterBannedUsersPlug + plug PhilomenaWeb.CaptchaPlug + plug PhilomenaWeb.UserAttributionPlug + plug PhilomenaWeb.CanaryMapPlug, update: :show + plug :load_and_authorize_resource, model: Image, id_name: "image_id" + + def update(conn, %{"image" => image_params}) do + attributes = conn.assigns.attributes + image = conn.assigns.image + + case Images.update_tags(image, attributes, image_params) do + {:ok, %{image: image}} -> + changeset = + Images.change_image(image) + + conn + |> put_view(PhilomenaWeb.ImageView) + |> render("_tags.html", image: image, changeset: changeset) + + {:error, :image, changeset, _} -> + conn + |> put_view(PhilomenaWeb.ImageView) + |> render("_tags.html", image: image, changeset: changeset) + end + end +end \ No newline at end of file diff --git a/lib/philomena_web/plugs/captcha_plug.ex b/lib/philomena_web/plugs/captcha_plug.ex new file mode 100644 index 00000000..e2ef61d0 --- /dev/null +++ b/lib/philomena_web/plugs/captcha_plug.ex @@ -0,0 +1,26 @@ +defmodule PhilomenaWeb.CaptchaPlug do + alias Philomena.Captcha + alias Phoenix.Controller + alias Plug.Conn + + def init([]), do: false + + def call(conn, _opts) do + user = conn |> Pow.Plug.current_user() + + conn + |> maybe_check_captcha(user) + end + + defp maybe_check_captcha(conn, nil) do + case Captcha.valid_solution?(conn.params) do + true -> conn + false -> + conn + |> Controller.put_flash(:error, "There was an error verifying you're not a robot. Please try again.") + |> Controller.redirect(external: conn.assigns.referrer) + |> Conn.halt() + end + end + defp maybe_check_captcha(conn, _user), do: conn +end