source/tag change model setup

lib/philomena/images.ex

@@ -4,22 +4,14 @@ defmodule Philomena.Images do
   """
 
   import Ecto.Query, warn: false
+
+  alias Ecto.Multi
   alias Philomena.Repo
 
   alias Philomena.Images.Image
-
+  alias Philomena.SourceChanges.SourceChange
-  @doc """
+  alias Philomena.TagChanges.TagChange
-  Returns the list of images.
+  alias Philomena.Tags
-
-  ## Examples
-
-      iex> list_images()
-      [%Image{}, ...]
-
-  """
-  def list_images do
-    Repo.all(Image |> where(hidden_from_users: false) |> order_by(desc: :created_at) |> limit(25))
-  end
 
   @doc """
   Gets a single image.
@@ -75,6 +67,82 @@ defmodule Philomena.Images do
     |> Repo.update()
   end
 
+  def update_source(%Image{} = image, attribution, attrs) do
+    image_changes =
+      image
+      |> Image.source_changeset(attrs)
+
+    source_changes =
+      Ecto.build_assoc(image, :source_changes)
+      |> SourceChange.creation_changeset(attrs, attribution)
+
+    Multi.new
+    |> Multi.update(:image, image_changes)
+    |> Multi.insert(:source_change, source_changes)
+    |> Repo.isolated_transaction(:serializable)
+  end
+
+  def update_tags(%Image{} = image, attribution, attrs) do
+    old_tags = Tags.get_or_create_tags(attrs["old_tag_input"])
+    new_tags = Tags.get_or_create_tags(attrs["tag_input"])
+
+    Multi.new
+    |> Multi.run(:image, fn repo, _chg ->
+      image
+      |> repo.preload(:tags, force: true)
+      |> Image.tag_changeset(%{}, old_tags, new_tags)
+      |> repo.update()
+      |> case do
+        {:ok, image} ->
+          {:ok, {image, image.added_tags, image.removed_tags}}
+
+        error ->
+          error
+      end
+    end)
+    |> Multi.run(:added_tag_changes, fn repo, %{image: {image, added_tags, _removed}} ->
+      tag_changes =
+        added_tags
+        |> Enum.map(&tag_change_attributes(attribution, image, &1, true, attribution[:user]))
+      
+      {count, nil} = repo.insert_all(TagChange, tag_changes)
+
+      {:ok, count}
+    end)
+    |> Multi.run(:removed_tag_changes, fn repo, %{image: {image, _added, removed_tags}} ->
+      tag_changes =
+        removed_tags
+        |> Enum.map(&tag_change_attributes(attribution, image, &1, false, attribution[:user]))
+
+      {count, nil} = repo.insert_all(TagChange, tag_changes)
+
+      {:ok, count}
+    end)
+    |> Repo.isolated_transaction(:serializable)
+  end
+
+  defp tag_change_attributes(attribution, image, tag, added, user) do
+    now = DateTime.utc_now() |> DateTime.truncate(:second)
+    user_id =
+      case user do
+        nil  -> nil
+        user -> user.id
+      end
+
+    %{
+      image_id: image.id,
+      tag_id: tag.id,
+      user_id: user_id,
+      created_at: now,
+      tag_name_cache: tag.name,
+      ip: attribution[:ip],
+      fingerprint: attribution[:fingerprint],
+      user_agent: attribution[:user_agent],
+      referrer: attribution[:referrer],
+      added: added
+    }
+  end
+
   @doc """
   Deletes a Image.
 

lib/philomena/images/image.ex

@@ -13,9 +13,14 @@ defmodule Philomena.Images.Image do
   alias Philomena.ImageHides.ImageHide
   alias Philomena.Images.Subscription
   alias Philomena.Users.User
-  alias Philomena.Images.Tagging
+  alias Philomena.Tags.Tag
   alias Philomena.Galleries
   alias Philomena.Comments.Comment
+  alias Philomena.SourceChanges.SourceChange
+  alias Philomena.TagChanges.TagChange
+
+  alias Philomena.Images.TagDiffer
+  alias Philomena.Images.TagValidator
 
   schema "images" do
     belongs_to :user, User
@@ -25,14 +30,15 @@ defmodule Philomena.Images.Image do
     has_many :downvotes, ImageVote, where: [up: false]
     has_many :faves, ImageFave
     has_many :hides, ImageHide
-    has_many :taggings, Tagging
     has_many :gallery_interactions, Galleries.Interaction
     has_many :subscriptions, Subscription
-    has_many :tags, through: [:taggings, :tag]
+    has_many :source_changes, SourceChange
+    has_many :tag_changes, TagChange
     has_many :upvoters, through: [:upvotes, :user]
     has_many :downvoters, through: [:downvotes, :user]
     has_many :favers, through: [:faves, :user]
     has_many :hiders, through: [:hides, :user]
+    many_to_many :tags, Tag, join_through: "image_taggings", on_replace: :delete
 
     field :image, :string
     field :image_name, :string
@@ -78,6 +84,9 @@ defmodule Philomena.Images.Image do
     field :tag_list_plus_alias_cache, :string
     field :file_name_cache, :string
 
+    field :removed_tags, {:array, :any}, default: [], virtual: true
+    field :added_tags, {:array, :any}, default: [], virtual: true
+
     timestamps(inserted_at: :created_at)
   end
 
@@ -96,4 +105,18 @@ defmodule Philomena.Images.Image do
       downvotes: image.downvotes_count
     }
   end
+
+  def source_changeset(image, attrs) do
+    image
+    |> cast(attrs, [:source_url])
+    |> validate_required(:source_url)
+    |> validate_format(:source_url, ~r/\Ahttps?:\/\//)
+  end
+
+  def tag_changeset(image, attrs, old_tags, new_tags) do
+    image
+    |> cast(attrs, [])
+    |> TagDiffer.diff_input(old_tags, new_tags)
+    |> TagValidator.validate_tags()
+  end
 end

lib/philomena/images/tag_differ.ex

@@ -0,0 +1,91 @@
+defmodule Philomena.Images.TagDiffer do
+  import Ecto.Changeset
+  import Ecto.Query
+
+  alias Philomena.Tags.Tag
+  alias Philomena.Repo
+
+  def diff_input(changeset, old_tags, new_tags) do
+    old_set = to_set(old_tags)
+    new_set = to_set(new_tags)
+
+    tags         = changeset |> get_field(:tags)
+    added_tags   = added_set(old_set, new_set)
+    removed_tags = removed_set(old_set, new_set)
+
+    {tags, actually_added, actually_removed} =
+      apply_changes(tags, added_tags, removed_tags)
+
+    changeset
+    |> put_change(:added_tags, actually_added)
+    |> put_change(:removed_tags, actually_removed)
+    |> put_assoc(:tags, tags)
+  end
+
+  defp added_set(old_set, new_set) do
+    # new_tags - old_tags
+    added_set =
+      new_set
+      |> Map.drop(Map.keys(old_set))
+
+    implied_set =
+      added_set
+      |> Enum.flat_map(fn {_k, v} -> v.implied_tags end)
+      |> List.flatten()
+      |> to_set()
+
+    added_and_implied_set =
+      Map.merge(added_set, implied_set)
+
+    oc_set =
+      added_and_implied_set
+      |> Enum.filter(fn {_k, v} -> v.namespace == "oc" end)
+      |> get_oc_tag()
+
+    Map.merge(added_and_implied_set, oc_set)
+  end
+
+  defp removed_set(old_set, new_set) do
+    # old_tags - new_tags
+    old_set |> Map.drop(Map.keys(new_set))
+  end
+
+  defp get_oc_tag([]), do: Map.new()
+  defp get_oc_tag(_any_oc_tag) do
+    Tag
+    |> where(name: "oc")
+    |> Repo.all()
+    |> to_set()
+  end
+
+  defp to_set(tags) do
+    tags |> Map.new(&{&1.id, &1})
+  end
+
+  defp to_tag_list(set) do
+    set |> Enum.map(fn {_k, v} -> v end)
+  end
+
+  defp apply_changes(tags, added_set, removed_set) do
+    tag_set = tags |> to_set()
+
+    desired_tags =
+      tag_set
+      |> Map.drop(Map.keys(removed_set))
+      |> Map.merge(added_set)
+
+    actually_added =
+      desired_tags
+      |> Map.drop(Map.keys(tag_set))
+
+    actually_removed =
+      tag_set
+      |> Map.drop(Map.keys(desired_tags))
+
+    tags             = desired_tags |> to_tag_list()
+    actually_added   = actually_added |> to_tag_list()
+    actually_removed = actually_removed |> to_tag_list()
+
+    {tags, actually_added, actually_removed}
+  end
+end

lib/philomena/images/tag_validator.ex

@@ -0,0 +1,92 @@
+defmodule Philomena.Images.TagValidator do
+  import Ecto.Changeset
+
+  @safe_rating    MapSet.new(["safe"])
+  @sexual_ratings MapSet.new(["suggestive", "questionable", "explicit"])
+  @horror_ratings MapSet.new(["semi-grimdark", "grimdark"])
+  @gross_rating   MapSet.new(["grotesque"])
+  @empty          MapSet.new()
+
+  def validate_tags(changeset) do
+    tags = changeset |> get_field(:tags)
+
+    validate_tag_input(changeset, tags)
+  end
+
+  defp validate_tag_input(changeset, tags) do
+    tag_set    = extract_names(tags)
+    rating_set = ratings(tag_set)
+
+    changeset
+    |> validate_number_of_tags(tag_set, 3)
+    |> validate_has_rating(rating_set)
+    |> validate_safe(rating_set)
+    |> validate_sexual_exclusion(rating_set)
+    |> validate_horror_exclusion(rating_set)
+  end
+
+  defp ratings(%MapSet{} = tag_set) do
+    safe   = MapSet.intersection(tag_set, @safe_rating)
+    sexual = MapSet.intersection(tag_set, @sexual_ratings)
+    horror = MapSet.intersection(tag_set, @horror_ratings)
+    gross  = MapSet.intersection(tag_set, @gross_rating)
+
+    %{
+      safe: safe,
+      sexual: sexual,
+      horror: horror,
+      gross: gross
+    }
+  end
+
+  defp validate_number_of_tags(changeset, tag_set, num) do
+    cond do
+      MapSet.size(tag_set) < num ->
+        changeset
+        |> add_error(:tag_input, "must contain at least #{num} tags")
+
+      true ->
+        changeset
+    end
+  end
+
+  defp validate_has_rating(changeset, %{safe: s, sexual: x, horror: h, gross: g}) when s == @empty and x == @empty and h == @empty and g == @empty do
+    changeset
+    |> add_error(:tag_input, "must contain at least one rating tag")
+  end
+  defp validate_has_rating(changeset, _ratings), do: changeset
+
+  defp validate_safe(changeset, %{safe: s, sexual: x, horror: h, gross: g}) when s != @empty and (x != @empty or h != @empty or g != @empty) do
+    changeset
+    |> add_error(:tag_input, "may not contain any other rating if safe")
+  end
+  defp validate_safe(changeset, _ratings), do: changeset
+
+  defp validate_sexual_exclusion(changeset, %{sexual: x}) do
+    cond do
+      MapSet.size(x) > 1 ->
+        changeset
+        |> add_error(:tag_input, "may contain at most one sexual rating")
+
+      true ->
+        changeset
+    end
+  end
+
+  defp validate_horror_exclusion(changeset, %{horror: h}) do
+    cond do
+      MapSet.size(h) > 1 ->
+        changeset
+        |> add_error(:tag_input, "may contain at most one grim rating")
+
+      true ->
+        changeset
+    end
+  end
+
+  defp extract_names(tags) do
+    tags
+    |> Enum.map(& &1.name)
+    |> MapSet.new()
+  end
+end

lib/philomena/source_changes.ex

@@ -8,19 +8,6 @@ defmodule Philomena.SourceChanges do
 
   alias Philomena.SourceChanges.SourceChange
 
-  @doc """
-  Returns the list of source_changes.
-
-  ## Examples
-
-      iex> list_source_changes()
-      [%SourceChange{}, ...]
-
-  """
-  def list_source_changes do
-    Repo.all(SourceChange)
-  end
-
   @doc """
   Gets a single source_change.
 

lib/philomena/source_changes/source_change.ex

@@ -13,6 +13,8 @@ defmodule Philomena.SourceChanges.SourceChange do
     field :new_value, :string
     field :initial, :boolean, default: false
 
+    field :source_url, :string, source: :new_value
+
     timestamps(inserted_at: :created_at)
   end
 
@@ -22,4 +24,11 @@ defmodule Philomena.SourceChanges.SourceChange do
     |> cast(attrs, [])
     |> validate_required([])
   end
+
+  @doc false
+  def creation_changeset(source_change, attrs, attribution) do
+    source_change
+    |> cast(attrs, [:source_url])
+    |> change(attribution)
+  end
 end

lib/philomena/tags.ex

@@ -8,17 +8,39 @@ defmodule Philomena.Tags do
 
   alias Philomena.Tags.Tag
 
-  @doc """
+  @spec get_or_create_tags(String.t()) :: List.t()
-  Returns the list of tags.
+  def get_or_create_tags(tag_list) do
+    tag_names = Tag.parse_tag_list(tag_list)
 
-  ## Examples
+    existent_tags =
+      Tag
+      |> where([t], t.name in ^tag_names)
+      |> preload(:implied_tags)
+      |> Repo.all()
 
-      iex> list_tags()
+    existent_tag_names =
-      [%Tag{}, ...]
+      existent_tags
+      |> Map.new(&{&1.name, true})
 
-  """
+    nonexistent_tag_names =
-  def list_tags do
+      tag_names
-    Repo.all(Tag |> order_by(desc: :images_count) |> limit(250))
+      |> Enum.reject(&existent_tag_names[&1])
+
+    new_tags =
+      nonexistent_tag_names
+      |> Enum.map(fn name ->
+        {:ok, tag} =
+          %Tag{}
+          |> Tag.creation_changeset(%{name: name})
+          |> Repo.insert()
+
+        %{tag | implied_tags: []}
+      end)
+
+    new_tags
+    |> reindex_tags()
+
+    existent_tags ++ new_tags
   end
 
   @doc """
@@ -51,7 +73,7 @@ defmodule Philomena.Tags do
   """
   def create_tag(attrs \\ %{}) do
     %Tag{}
-    |> Tag.changeset(attrs)
+    |> Tag.creation_changeset(attrs)
     |> Repo.insert()
   end
 
@@ -102,6 +124,29 @@ defmodule Philomena.Tags do
     Tag.changeset(tag, %{})
   end
 
+  def reindex_tag(%Tag{} = tag) do
+    reindex_tags([%Tag{id: tag.id}])
+  end
+
+  def reindex_tags(tags) do
+    spawn fn ->
+      ids =
+        tags
+        |> Enum.map(& &1.id)
+
+      Tag
+      |> preload(^indexing_preloads())
+      |> where([t], t.id in ^ids)
+      |> Tag.reindex()
+    end
+
+    tags
+  end
+
+  def indexing_preloads do
+    [:aliased_tag, :aliases, :implied_tags, :implied_by_tags]
+  end
+
   alias Philomena.Tags.Implication
 
   @doc """

lib/philomena/tags/tag.ex

@@ -8,6 +8,26 @@ defmodule Philomena.Tags.Tag do
     doc_type: "tag"
 
   alias Philomena.Tags.Tag
+  alias Philomena.Slug
+
+  @namespaces [
+    "artist",
+    "art pack",
+    "ask",
+    "blog",
+    "colorist",
+    "comic",
+    "editor",
+    "fanfic",
+    "oc",
+    "parent",
+    "parents",
+    "photographer",
+    "series",
+    "species",
+    "spoiler",
+    "video"
+  ]
 
   schema "tags" do
     belongs_to :aliased_tag, Tag, source: :aliased_tag_id
@@ -37,4 +57,88 @@ defmodule Philomena.Tags.Tag do
     |> cast(attrs, [])
     |> validate_required([])
   end
+
+  @doc false
+  def creation_changeset(tag, attrs) do
+    tag
+    |> cast(attrs, [:name])
+    |> validate_required([:name])
+    |> put_slug()
+    |> put_name_and_namespace()
+  end
+
+  def parse_tag_list(list) do
+    list
+    |> to_string()
+    |> String.split(",")
+    |> Enum.map(&clean_tag_name/1)
+    |> Enum.reject(&"" == &1)
+  end
+
+  def clean_tag_name(name) do
+    # Downcase, replace extra runs of spaces, replace unicode quotes
+    # with ascii quotes, trim space from end
+    name
+    |> String.downcase()
+    |> String.replace(~r/[[:space:]]+/, " ")
+    |> String.replace(~r/[\x{00b4}\x{2018}\x{2019}\x{201a}\x{201b}\x{2032}]/u, "'")
+    |> String.replace(~r/[\x{201c}\x{201d}\x{201e}\x{201f}\x{2033}]/u, "\"")
+    |> String.trim()
+    |> clean_tag_namespace()
+    |> ununderscore()
+  end
+
+  defp clean_tag_namespace(name) do
+    # Remove extra spaces after the colon in a namespace
+    # (artist:, oc:, etc.)
+    name
+    |> String.split(":", parts: 2)
+    |> Enum.map(&String.trim/1)
+    |> join_namespace_parts(name)
+  end
+
+  defp join_namespace_parts([_name], original_name),
+    do: original_name
+  defp join_namespace_parts([namespace, name], _original_name) when namespace in @namespaces,
+    do: namespace <> ":" <> name
+  defp join_namespace_parts([_namespace, _name], original_name),
+    do: original_name
+
+  defp ununderscore(<<"artist:", _rest::binary>> = name),
+    do: name
+  defp ununderscore(name),
+    do: String.replace(name, "_", " ")
+
+  defp put_slug(changeset) do
+    slug =
+      changeset
+      |> get_field(:name)
+      |> to_string()
+      |> Slug.slug()
+
+    changeset
+    |> change(slug: slug)
+  end
+
+  defp put_name_and_namespace(changeset) do
+    {namespace, name_in_namespace} =
+      changeset
+      |> get_field(:name)
+      |> to_string()
+      |> extract_name_and_namespace()
+
+    changeset
+    |> change(namespace: namespace)
+    |> change(name_in_namespace: name_in_namespace)
+  end
+
+  defp extract_name_and_namespace(name) do
+    case String.split(name, ":", parts: 2) do
+      [namespace, name_in_namespace] when namespace in @namespaces ->
+        {namespace, name_in_namespace}
+
+      _value ->
+        {nil, name}
+    end
+  end
 end

lib/philomena_web/controllers/image/source_controller.ex

@@ -0,0 +1,32 @@
+defmodule PhilomenaWeb.Image.SourceController do
+  use PhilomenaWeb, :controller
+
+  alias Philomena.Images
+  alias Philomena.Images.Image
+
+  plug PhilomenaWeb.FilterBannedUsersPlug
+  plug PhilomenaWeb.CaptchaPlug
+  plug PhilomenaWeb.UserAttributionPlug
+  plug PhilomenaWeb.CanaryMapPlug, update: :show
+  plug :load_and_authorize_resource, model: Image, id_name: "image_id"
+
+  def update(conn, %{"image" => image_params}) do
+    attributes = conn.assigns.attributes
+    image = conn.assigns.image
+
+    case Images.update_source(image, attributes, image_params) do
+      {:ok, %{image: image}} ->
+        changeset =
+          Images.change_image(image)
+
+        conn
+        |> put_view(PhilomenaWeb.ImageView)
+        |> render("_source.html", image: image, changeset: changeset)
+
+      {:error, :image, changeset, _} ->
+        conn
+        |> put_view(PhilomenaWeb.ImageView)
+        |> render("_source.html", image: image, changeset: changeset)
+    end
+  end
+end

lib/philomena_web/controllers/image/tag_controller.ex

@@ -0,0 +1,32 @@
+defmodule PhilomenaWeb.Image.TagController do
+  use PhilomenaWeb, :controller
+
+  alias Philomena.Images
+  alias Philomena.Images.Image
+
+  plug PhilomenaWeb.FilterBannedUsersPlug
+  plug PhilomenaWeb.CaptchaPlug
+  plug PhilomenaWeb.UserAttributionPlug
+  plug PhilomenaWeb.CanaryMapPlug, update: :show
+  plug :load_and_authorize_resource, model: Image, id_name: "image_id"
+
+  def update(conn, %{"image" => image_params}) do
+    attributes = conn.assigns.attributes
+    image = conn.assigns.image
+
+    case Images.update_tags(image, attributes, image_params) do
+      {:ok, %{image: image}} ->
+        changeset =
+          Images.change_image(image)
+
+        conn
+        |> put_view(PhilomenaWeb.ImageView)
+        |> render("_tags.html", image: image, changeset: changeset)
+
+      {:error, :image, changeset, _} ->
+        conn
+        |> put_view(PhilomenaWeb.ImageView)
+        |> render("_tags.html", image: image, changeset: changeset)
+    end
+  end
+end

lib/philomena_web/plugs/captcha_plug.ex

@@ -0,0 +1,26 @@
+defmodule PhilomenaWeb.CaptchaPlug do
+  alias Philomena.Captcha
+  alias Phoenix.Controller
+  alias Plug.Conn
+
+  def init([]), do: false
+
+  def call(conn, _opts) do
+    user = conn |> Pow.Plug.current_user()
+
+    conn
+    |> maybe_check_captcha(user)
+  end
+
+  defp maybe_check_captcha(conn, nil) do
+    case Captcha.valid_solution?(conn.params) do
+      true  -> conn
+      false ->
+        conn
+        |> Controller.put_flash(:error, "There was an error verifying you're not a robot. Please try again.")
+        |> Controller.redirect(external: conn.assigns.referrer)
+        |> Conn.halt()
+    end
+  end
+  defp maybe_check_captcha(conn, _user), do: conn
+end