User: Add raw_bio, storing unformatted bio

Related: https://git.pleroma.social/pleroma/pleroma/issues/1643

lib/pleroma/user.ex

@@ -79,6 +79,7 @@ defmodule Pleroma.User do
 
   schema "users" do
     field(:bio, :string)
+    field(:raw_bio, :string)
     field(:email, :string)
     field(:name, :string)
     field(:nickname, :string)
@@ -432,6 +433,7 @@ defmodule Pleroma.User do
       params,
       [
         :bio,
+        :raw_bio,
         :name,
         :emoji,
         :avatar,
@@ -607,7 +609,16 @@ defmodule Pleroma.User do
 
     struct
     |> confirmation_changeset(need_confirmation: need_confirmation?)
-    |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation, :emoji])
+    |> cast(params, [
+      :bio,
+      :raw_bio,
+      :email,
+      :name,
+      :nickname,
+      :password,
+      :password_confirmation,
+      :emoji
+    ])
     |> validate_required([:name, :nickname, :password, :password_confirmation])
     |> validate_confirmation(:password)
     |> unique_constraint(:email)

lib/pleroma/web/mastodon_api/controllers/account_controller.ex

@@ -165,6 +165,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
       end)
       |> Maps.put_if_present(:name, params[:display_name])
       |> Maps.put_if_present(:bio, params[:note])
+      |> Maps.put_if_present(:raw_bio, params[:note])
       |> Maps.put_if_present(:avatar, params[:avatar])
       |> Maps.put_if_present(:banner, params[:header])
       |> Maps.put_if_present(:background, params[:pleroma_background_image])

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -224,7 +224,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       fields: user.fields,
       bot: bot,
       source: %{
-        note: prepare_user_bio(user),
+        note: user.raw_bio || "",
         sensitive: false,
         fields: user.raw_fields,
         pleroma: %{
@@ -259,17 +259,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
     |> maybe_put_unread_notification_count(user, opts[:for])
   end
 
-  defp prepare_user_bio(%User{bio: ""}), do: ""
-
-  defp prepare_user_bio(%User{bio: bio}) when is_binary(bio) do
-    bio
-    |> String.replace(~r(<br */?>), "\n")
-    |> Pleroma.HTML.strip_tags()
-    |> HtmlEntities.decode()
-  end
-
-  defp prepare_user_bio(_), do: ""
-
   defp username_from_nickname(string) when is_binary(string) do
     hd(String.split(string, "@"))
   end

priv/repo/migrations/20200322174133_user_raw_bio.exs

@@ -0,0 +1,9 @@
+defmodule Pleroma.Repo.Migrations.UserRawBio do
+  use Ecto.Migration
+
+  def change do
+    alter table(:users) do
+      add_if_not_exists(:raw_bio, :text)
+    end
+  end
+end

priv/repo/migrations/20200328193433_populate_user_raw_bio.exs

@@ -0,0 +1,25 @@
+defmodule Pleroma.Repo.Migrations.PopulateUserRawBio do
+  use Ecto.Migration
+  import Ecto.Query
+  alias Pleroma.User
+  alias Pleroma.Repo
+
+  def change do
+    {:ok, _} = Application.ensure_all_started(:fast_sanitize)
+
+    User.Query.build(%{local: true})
+    |> select([u], struct(u, [:id, :ap_id, :bio]))
+    |> Repo.stream()
+    |> Enum.each(fn %{bio: bio} = user ->
+      if bio do
+        raw_bio =
+          bio
+          |> String.replace(~r(<br */?>), "\n")
+          |> Pleroma.HTML.strip_tags()
+
+        Ecto.Changeset.cast(user, %{raw_bio: raw_bio}, [:raw_bio])
+        |> Repo.update()
+      end
+    end)
+  end
+end

test/support/factory.ex

@@ -42,7 +42,8 @@ defmodule Pleroma.Factory do
       user
       | ap_id: User.ap_id(user),
         follower_address: User.ap_followers(user),
-        following_address: User.ap_following(user)
+        following_address: User.ap_following(user),
+        raw_bio: user.bio
     }
   end
 

test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs

@@ -83,10 +83,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do
     test "updates the user's bio", %{conn: conn} do
       user2 = insert(:user)
 
-      conn =
+      raw_bio = "I drink #cofe with @#{user2.nickname}\n\nsuya.."
-        patch(conn, "/api/v1/accounts/update_credentials", %{
+
-          "note" => "I drink #cofe with @#{user2.nickname}\n\nsuya.."
+      conn = patch(conn, "/api/v1/accounts/update_credentials", %{"note" => raw_bio})
-        })
 
       assert user_data = json_response_and_validate_schema(conn, 200)
 
@@ -94,6 +93,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do
                ~s(I drink <a class="hashtag" data-tag="cofe" href="http://localhost:4001/tag/cofe">#cofe</a> with <span class="h-card"><a class="u-url mention" data-user="#{
                  user2.id
                }" href="#{user2.ap_id}" rel="ugc">@<span>#{user2.nickname}</span></a></span><br/><br/>suya..)
+
+      assert user_data["source"]["note"] == raw_bio
+
+      user = Repo.get(User, user_data["id"])
+
+      assert user.raw_bio == raw_bio
     end
 
     test "updates the user's locking status", %{conn: conn} do

test/web/mastodon_api/views/account_view_test.exs

@@ -33,7 +33,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
         bio:
           "<script src=\"invalid-html\"></script><span>valid html</span>. a<br>b<br/>c<br >d<br />f '&<>\"",
         inserted_at: ~N[2017-08-15 15:47:06.597036],
-        emoji: %{"karjalanpiirakka" => "/file.png"}
+        emoji: %{"karjalanpiirakka" => "/file.png"},
+        raw_bio: "valid html. a\nb\nc\nd\nf '&<>\""
       })
 
     expected = %{