profile emojis in User.emoji instead of source_data

lib/pleroma/emoji/formatter.ex

@@ -38,22 +38,14 @@ defmodule Pleroma.Emoji.Formatter do
 
   def demojify(text, nil), do: text
 
-  @doc "Outputs a list of the emoji-shortcodes in a text"
-  def get_emoji(text) when is_binary(text) do
-    Enum.filter(Emoji.get_all(), fn {emoji, %Emoji{}} ->
-      String.contains?(text, ":#{emoji}:")
-    end)
-  end
-
-  def get_emoji(_), do: []
-
   @doc "Outputs a list of the emoji-Maps in a text"
   def get_emoji_map(text) when is_binary(text) do
-    get_emoji(text)
+    Emoji.get_all()
+    |> Enum.filter(fn {emoji, %Emoji{}} -> String.contains?(text, ":#{emoji}:") end)
     |> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc ->
       Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
     end)
   end
 
-  def get_emoji_map(_), do: []
+  def get_emoji_map(_), do: %{}
 end

lib/pleroma/user.ex

@@ -15,6 +15,7 @@ defmodule Pleroma.User do
   alias Pleroma.Config
   alias Pleroma.Conversation.Participation
   alias Pleroma.Delivery
+  alias Pleroma.Emoji
   alias Pleroma.FollowingRelationship
   alias Pleroma.Formatter
   alias Pleroma.HTML
@@ -124,7 +125,7 @@ defmodule Pleroma.User do
     field(:pinned_activities, {:array, :string}, default: [])
     field(:email_notifications, :map, default: %{"digest" => false})
     field(:mascot, :map, default: nil)
-    field(:emoji, {:array, :map}, default: [])
+    field(:emoji, :map, default: %{})
     field(:pleroma_settings_store, :map, default: %{})
     field(:fields, {:array, :map}, default: [])
     field(:raw_fields, {:array, :map}, default: [])
@@ -368,6 +369,7 @@ defmodule Pleroma.User do
       [
         :bio,
         :name,
+        :emoji,
         :ap_id,
         :inbox,
         :shared_inbox,
@@ -413,6 +415,7 @@ defmodule Pleroma.User do
       [
         :bio,
         :name,
+        :emoji,
         :avatar,
         :public_key,
         :inbox,
@@ -443,6 +446,7 @@ defmodule Pleroma.User do
     |> validate_length(:bio, max: bio_limit)
     |> validate_length(:name, min: 1, max: name_limit)
     |> put_fields()
+    |> put_emoji()
     |> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)})
     |> put_change_if_present(:avatar, &put_upload(&1, :avatar))
     |> put_change_if_present(:banner, &put_upload(&1, :banner))
@@ -478,6 +482,18 @@ defmodule Pleroma.User do
     |> elem(0)
   end
 
+  defp put_emoji(changeset) do
+    bio = get_change(changeset, :bio)
+    name = get_change(changeset, :name)
+
+    if bio || name do
+      emoji = Map.merge(Emoji.Formatter.get_emoji_map(bio), Emoji.Formatter.get_emoji_map(name))
+      put_change(changeset, :emoji, emoji)
+    else
+      changeset
+    end
+  end
+
   defp put_change_if_present(changeset, map_field, value_function) do
     if value = get_change(changeset, map_field) do
       with {:ok, new_value} <- value_function.(value) do
@@ -511,6 +527,7 @@ defmodule Pleroma.User do
       [
         :bio,
         :name,
+        :emoji,
         :follower_address,
         :following_address,
         :public_key,
@@ -618,7 +635,7 @@ defmodule Pleroma.User do
 
     struct
     |> confirmation_changeset(need_confirmation: need_confirmation?)
-    |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation])
+    |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation, :emoji])
     |> validate_required([:name, :nickname, :password, :password_confirmation])
     |> validate_confirmation(:password)
     |> unique_constraint(:email)
@@ -1969,12 +1986,6 @@ defmodule Pleroma.User do
     |> update_and_set_cache()
   end
 
-  def update_source_data(user, source_data) do
-    user
-    |> cast(%{source_data: source_data}, [:source_data])
-    |> update_and_set_cache()
-  end
-
   def roles(%{is_moderator: is_moderator, is_admin: is_admin}) do
     %{
       admin: is_admin,

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -1427,6 +1427,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end)
       |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end)
 
+    emojis =
+      data
+      |> Map.get("tag", [])
+      |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
+      |> Enum.reduce(%{}, fn %{"icon" => %{"url" => url}, "name" => name}, acc ->
+        Map.put(acc, String.trim(name, ":"), url)
+      end)
+
     locked = data["manuallyApprovesFollowers"] || false
     data = Transmogrifier.maybe_fix_user_object(data)
     discoverable = data["discoverable"] || false
@@ -1454,6 +1462,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       source_data: data,
       banner: banner,
       fields: fields,
+      emoji: emojis,
       locked: locked,
       discoverable: discoverable,
       invisible: invisible,

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -1129,7 +1129,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def take_emoji_tags(%User{emoji: emoji}) do
     emoji
-    |> Enum.flat_map(&Map.to_list/1)
+    |> Map.to_list()
     |> Enum.map(&build_emoji_tag/1)
   end
 

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -103,7 +103,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
       },
       "endpoints" => endpoints,
       "attachment" => fields,
-      "tag" => (user.source_data["tag"] || []) ++ emoji_tags,
+      "tag" => emoji_tags,
       "discoverable" => user.discoverable
     }
     |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))

lib/pleroma/web/common_api/common_api.ex

@@ -332,26 +332,6 @@ defmodule Pleroma.Web.CommonAPI do
 
   defp maybe_create_activity_expiration(result, _), do: result
 
-  # Updates the emojis for a user based on their profile
-  def update(user) do
-    emoji = emoji_from_profile(user)
-    source_data = Map.put(user.source_data, "tag", emoji)
-
-    user =
-      case User.update_source_data(user, source_data) do
-        {:ok, user} -> user
-        _ -> user
-      end
-
-    ActivityPub.update(%{
-      local: true,
-      to: [Pleroma.Constants.as_public(), user.follower_address],
-      cc: [],
-      actor: user.ap_id,
-      object: Pleroma.Web.ActivityPub.UserView.render("user.json", %{user: user})
-    })
-  end
-
   def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do
     with %Activity{
            actor: ^user_ap_id,

lib/pleroma/web/common_api/utils.ex

@@ -10,7 +10,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   alias Pleroma.Activity
   alias Pleroma.Config
   alias Pleroma.Conversation.Participation
-  alias Pleroma.Emoji
   alias Pleroma.Formatter
   alias Pleroma.Object
   alias Pleroma.Plugs.AuthenticationPlug
@@ -18,7 +17,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
-  alias Pleroma.Web.Endpoint
   alias Pleroma.Web.MediaProxy
 
   require Logger
@@ -175,7 +173,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
             "replies" => %{"type" => "Collection", "totalItems" => 0}
           }
 
-          {note, Map.merge(emoji, Emoji.Formatter.get_emoji_map(option))}
+          {note, Map.merge(emoji, Pleroma.Emoji.Formatter.get_emoji_map(option))}
         end)
 
       end_time =
@@ -431,19 +429,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     end
   end
 
-  def emoji_from_profile(%User{bio: bio, name: name}) do
-    [bio, name]
-    |> Enum.map(&Emoji.Formatter.get_emoji/1)
-    |> Enum.concat()
-    |> Enum.map(fn {shortcode, %Emoji{file: path}} ->
-      %{
-        "type" => "Emoji",
-        "icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}#{path}"},
-        "name" => ":#{shortcode}:"
-      }
-    end)
-  end
-
   def maybe_notify_to_recipients(
         recipients,
         %Activity{data: %{"to" => to, "type" => _type}} = _activity

lib/pleroma/web/mastodon_api/controllers/account_controller.ex

@@ -140,9 +140,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
   end
 
   @doc "PATCH /api/v1/accounts/update_credentials"
-  def update_credentials(%{assigns: %{user: original_user}} = conn, params) do
+  def update_credentials(%{assigns: %{user: user}} = conn, params) do
-    user = original_user
-
     user_params =
       [
         :no_rich_text,
@@ -178,8 +176,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
     changeset = User.update_changeset(user, user_params)
 
     with {:ok, user} <- User.update_and_set_cache(changeset) do
-      if original_user != user, do: CommonAPI.update(user)
-
       render(conn, "show.json", user: user, for: user, with_pleroma_settings: true)
     else
       _e -> render_error(conn, :forbidden, "Invalid request")

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -180,13 +180,11 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
     bot = user.actor_type in ["Application", "Service"]
 
     emojis =
-      (user.source_data["tag"] || [])
+      Enum.map(user.emoji, fn {shortcode, url} ->
-      |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
-      |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} ->
         %{
-          "shortcode" => String.trim(name, ":"),
+          "shortcode" => shortcode,
-          "url" => MediaProxy.url(url),
+          "url" => url,
-          "static_url" => MediaProxy.url(url),
+          "static_url" => url,
           "visible_in_picker" => false
         }
       end)

lib/pleroma/web/pleroma_api/controllers/account_controller.ex

@@ -13,7 +13,6 @@ defmodule Pleroma.Web.PleromaAPI.AccountController do
   alias Pleroma.Plugs.RateLimiter
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.StatusView
 
   require Pleroma.Constants
@@ -58,38 +57,32 @@ defmodule Pleroma.Web.PleromaAPI.AccountController do
 
   @doc "PATCH /api/v1/pleroma/accounts/update_avatar"
   def update_avatar(%{assigns: %{user: user}} = conn, %{"img" => ""}) do
-    {:ok, user} =
+    {:ok, _user} =
       user
       |> Changeset.change(%{avatar: nil})
       |> User.update_and_set_cache()
 
-    CommonAPI.update(user)
-
     json(conn, %{url: nil})
   end
 
   def update_avatar(%{assigns: %{user: user}} = conn, params) do
     {:ok, %{data: data}} = ActivityPub.upload(params, type: :avatar)
-    {:ok, user} = user |> Changeset.change(%{avatar: data}) |> User.update_and_set_cache()
+    {:ok, _user} = user |> Changeset.change(%{avatar: data}) |> User.update_and_set_cache()
     %{"url" => [%{"href" => href} | _]} = data
 
-    CommonAPI.update(user)
-
     json(conn, %{url: href})
   end
 
   @doc "PATCH /api/v1/pleroma/accounts/update_banner"
   def update_banner(%{assigns: %{user: user}} = conn, %{"banner" => ""}) do
-    with {:ok, user} <- User.update_banner(user, %{}) do
+    with {:ok, _user} <- User.update_banner(user, %{}) do
-      CommonAPI.update(user)
       json(conn, %{url: nil})
     end
   end
 
   def update_banner(%{assigns: %{user: user}} = conn, params) do
     with {:ok, object} <- ActivityPub.upload(%{"img" => params["banner"]}, type: :banner),
-         {:ok, user} <- User.update_banner(user, object.data) do
+         {:ok, _user} <- User.update_banner(user, object.data) do
-      CommonAPI.update(user)
       %{"url" => [%{"href" => href} | _]} = object.data
 
       json(conn, %{url: href})

lib/pleroma/web/static_fe/static_fe_view.ex

@@ -18,15 +18,6 @@ defmodule Pleroma.Web.StaticFE.StaticFEView do
 
   @media_types ["image", "audio", "video"]
 
-  def emoji_for_user(%User{} = user) do
-    user.source_data
-    |> Map.get("tag", [])
-    |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
-    |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} ->
-      {String.trim(name, ":"), url}
-    end)
-  end
-
   def fetch_media_type(%{"mediaType" => mediaType}) do
     Utils.fetch_media_type(@media_types, mediaType)
   end

lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex

@@ -4,7 +4,7 @@
       <img src="<%= User.avatar_url(@user) |> MediaProxy.url %>" width="48" height="48" alt="">
     </div>
     <span class="display-name">
-      <bdi><%= raw (@user.name |> Formatter.emojify(emoji_for_user(@user))) %></bdi>
+      <bdi><%= raw Formatter.emojify(@user.name, @user.emoji) %></bdi>
       <span class="nickname"><%= @user.nickname %></span>
     </span>
   </a>

lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex

@@ -7,7 +7,7 @@
       <input type="hidden" name="profile" value="">
       <button type="submit" class="collapse">Remote follow</button>
     </form>
-    <%= raw Formatter.emojify(@user.name, emoji_for_user(@user)) %> |
+    <%= raw Formatter.emojify(@user.name, @user.emoji) %> |
     <%= link "@#{@user.nickname}@#{Endpoint.host()}", to: (@user.uri || @user.ap_id) %>
   </h3>
   <p><%= raw @user.bio %></p>

priv/repo/migrations/20200406100225_users_add_emoji.exs

@@ -0,0 +1,35 @@
+defmodule Pleroma.Repo.Migrations.UsersPopulateEmoji do
+  use Ecto.Migration
+
+  import Ecto.Query
+
+  alias Pleroma.User
+  alias Pleroma.Repo
+
+  def up do
+    execute("ALTER TABLE users ALTER COLUMN emoji SET DEFAULT '{}'::jsonb")
+    execute("UPDATE users SET emoji = DEFAULT WHERE emoji = '[]'::jsonb")
+
+    from(u in User)
+    |> select([u], struct(u, [:id, :ap_id, :source_data]))
+    |> Repo.stream()
+    |> Enum.each(fn user ->
+      emoji =
+        user.source_data
+        |> Map.get("tag", [])
+        |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
+        |> Enum.reduce(%{}, fn %{"icon" => %{"url" => url}, "name" => name}, acc ->
+          Map.put(acc, String.trim(name, ":"), url)
+        end)
+
+      user
+      |> Ecto.Changeset.cast(%{emoji: emoji}, [:emoji])
+      |> Repo.update()
+    end)
+  end
+
+  def down do
+    execute("ALTER TABLE users ALTER COLUMN emoji SET DEFAULT '[]'::jsonb")
+    execute("UPDATE users SET emoji = DEFAULT WHERE emoji = '{}'::jsonb")
+  end
+end

test/emoji/formatter_test.exs

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Emoji.FormatterTest do
-  alias Pleroma.Emoji
   alias Pleroma.Emoji.Formatter
   use Pleroma.DataCase
 
@@ -32,30 +31,19 @@ defmodule Pleroma.Emoji.FormatterTest do
     end
   end
 
-  describe "get_emoji" do
+  describe "get_emoji_map" do
     test "it returns the emoji used in the text" do
-      text = "I love :firefox:"
+      assert Formatter.get_emoji_map("I love :firefox:") == %{
-
+               "firefox" => "http://localhost:4001/emoji/Firefox.gif"
-      assert Formatter.get_emoji(text) == [
+             }
-               {"firefox",
-                %Emoji{
-                  code: "firefox",
-                  file: "/emoji/Firefox.gif",
-                  tags: ["Gif", "Fun"],
-                  safe_code: "firefox",
-                  safe_file: "/emoji/Firefox.gif"
-                }}
-             ]
     end
 
     test "it returns a nice empty result when no emojis are present" do
-      text = "I love moominamma"
+      assert Formatter.get_emoji_map("I love moominamma") == %{}
-      assert Formatter.get_emoji(text) == []
     end
 
     test "it doesn't die when text is absent" do
-      text = nil
+      assert Formatter.get_emoji_map(nil) == %{}
-      assert Formatter.get_emoji(text) == []
     end
   end
 end

test/web/activity_pub/transmogrifier_test.exs

@@ -2182,4 +2182,18 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
                Transmogrifier.set_replies(object.data)["replies"]
     end
   end
+
+  test "take_emoji_tags/1" do
+    user = insert(:user, %{emoji: %{"firefox" => "https://example.org/firefox.png"}})
+
+    assert Transmogrifier.take_emoji_tags(user) == [
+             %{
+               "icon" => %{"type" => "Image", "url" => "https://example.org/firefox.png"},
+               "id" => "https://example.org/firefox.png",
+               "name" => ":firefox:",
+               "type" => "Emoji",
+               "updated" => "1970-01-01T00:00:00Z"
+             }
+           ]
+  end
 end

test/web/activity_pub/views/user_view_test.exs

@@ -38,7 +38,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do
   end
 
   test "Renders with emoji tags" do
-    user = insert(:user, emoji: [%{"bib" => "/test"}])
+    user = insert(:user, emoji: %{"bib" => "/test"})
 
     assert %{
              "tag" => [

test/web/common_api/common_api_test.exs

@@ -97,18 +97,6 @@ defmodule Pleroma.Web.CommonAPITest do
     assert Object.normalize(activity).data["emoji"]["firefox"]
   end
 
-  test "it adds emoji when updating profiles" do
-    user = insert(:user, %{name: ":firefox:"})
-
-    {:ok, activity} = CommonAPI.update(user)
-    user = User.get_cached_by_ap_id(user.ap_id)
-    [firefox] = user.source_data["tag"]
-
-    assert firefox["name"] == ":firefox:"
-
-    assert Pleroma.Constants.as_public() in activity.recipients
-  end
-
   describe "posting" do
     test "it supports explicit addressing" do
       user = insert(:user)

test/web/common_api/common_api_utils_test.exs

@@ -7,7 +7,6 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
   alias Pleroma.Object
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
-  alias Pleroma.Web.Endpoint
   use Pleroma.DataCase
 
   import ExUnit.CaptureLog
@@ -42,28 +41,6 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
     end
   end
 
-  test "parses emoji from name and bio" do
-    {:ok, user} = UserBuilder.insert(%{name: ":blank:", bio: ":firefox:"})
-
-    expected = [
-      %{
-        "type" => "Emoji",
-        "icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}/emoji/Firefox.gif"},
-        "name" => ":firefox:"
-      },
-      %{
-        "type" => "Emoji",
-        "icon" => %{
-          "type" => "Image",
-          "url" => "#{Endpoint.url()}/emoji/blank.png"
-        },
-        "name" => ":blank:"
-      }
-    ]
-
-    assert expected == Utils.emoji_from_profile(user)
-  end
-
   describe "format_input/3" do
     test "works for bare text/plain" do
       text = "hello world!"

test/web/mastodon_api/views/account_view_test.exs

@@ -19,16 +19,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
   end
 
   test "Represent a user account" do
-    source_data = %{
-      "tag" => [
-        %{
-          "type" => "Emoji",
-          "icon" => %{"url" => "/file.png"},
-          "name" => ":karjalanpiirakka:"
-        }
-      ]
-    }
-
     background_image = %{
       "url" => [%{"href" => "https://example.com/images/asuka_hospital.png"}]
     }
@@ -37,13 +27,13 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
       insert(:user, %{
         follower_count: 3,
         note_count: 5,
-        source_data: source_data,
         background: background_image,
         nickname: "shp@shitposter.club",
         name: ":karjalanpiirakka: shp",
         bio:
           "<script src=\"invalid-html\"></script><span>valid html</span>. a<br>b<br/>c<br >d<br />f",
-        inserted_at: ~N[2017-08-15 15:47:06.597036]
+        inserted_at: ~N[2017-08-15 15:47:06.597036],
+        emoji: %{"karjalanpiirakka" => "/file.png"}
       })
 
     expected = %{
@@ -117,7 +107,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
       insert(:user, %{
         follower_count: 3,
         note_count: 5,
-        source_data: %{},
         actor_type: "Service",
         nickname: "shp@shitposter.club",
         inserted_at: ~N[2017-08-15 15:47:06.597036]
@@ -311,7 +300,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
       insert(:user, %{
         follower_count: 0,
         note_count: 5,
-        source_data: %{},
         actor_type: "Service",
         nickname: "shp@shitposter.club",
         inserted_at: ~N[2017-08-15 15:47:06.597036]