Merge branch 'develop' into conversations-import

CHANGELOG.md

@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - [Prometheus](https://prometheus.io/) metrics
 - Support for Mastodon's remote interaction
 - Mix Tasks: `mix pleroma.database remove_embedded_objects`
+- Mix Tasks: `mix pleroma.user toggle_confirmed`
 - Federation: Support for reports
 - Configuration: `safe_dm_mentions` option
 - Configuration: `link_name` option

docs/api/admin_api.md

@@ -106,15 +106,15 @@ Authentication is required and the user must be an admin.
 
 - Method: `PUT`
 - Params:
-  - `nickname`
-  - `tags`
+  - `nicknames` (array)
+  - `tags` (array)
 
 ### Untag a list of users
 
 - Method: `DELETE`
 - Params:
-  - `nickname`
-  - `tags`
+  - `nicknames` (array)
+  - `tags` (array)
 
 ## `/api/pleroma/admin/users/:nickname/permission_group`
 

docs/installation/debian_based_en.md

@@ -12,6 +12,7 @@ This guide will assume you are on Debian Stretch. This guide should also work wi
 * `erlang-tools`
 * `erlang-parsetools`
 * `erlang-eldap`, if you want to enable ldap authenticator
+* `erlang-ssh`
 * `erlang-xmerl`
 * `git`
 * `build-essential`
@@ -49,7 +50,7 @@ sudo dpkg -i /tmp/erlang-solutions_1.0_all.deb
 
 ```shell
 sudo apt update
-sudo apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools
+sudo apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools erlang-ssh
 ```
 
 ### Install PleromaBE

docs/installation/debian_based_jp.md

@@ -14,6 +14,7 @@
 - erlang-dev
 - erlang-tools
 - erlang-parsetools
+- erlang-ssh
 - erlang-xmerl (Jessieではバックポートからインストールすること！)
 - git
 - build-essential
@@ -44,7 +45,7 @@ wget -P /tmp/ https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb
 
 * ElixirとErlangをインストールします、
 ```
-apt update && apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools
+apt update && apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools erlang-ssh
 ```
 
 ### Pleroma BE (バックエンド) をインストールします

lib/mix/tasks/pleroma/user.ex

@@ -77,6 +77,10 @@ defmodule Mix.Tasks.Pleroma.User do
   ## Delete tags from a user.
 
       mix pleroma.user untag NICKNAME TAGS
+
+  ## Toggle confirmation of the user's account.
+
+      mix pleroma.user toggle_confirmed NICKNAME
   """
   def run(["new", nickname, email | rest]) do
     {options, [], []} =
@@ -388,6 +392,21 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
+  def run(["toggle_confirmed", nickname]) do
+    Common.start_pleroma()
+
+    with %User{} = user <- User.get_cached_by_nickname(nickname) do
+      {:ok, user} = User.toggle_confirmation(user)
+
+      message = if user.info.confirmation_pending, do: "needs", else: "doesn't need"
+
+      Mix.shell().info("#{nickname} #{message} confirmation.")
+    else
+      _ ->
+        Mix.shell().error("No local user #{nickname}")
+    end
+  end
+
   defp set_moderator(user, value) do
     info_cng = User.Info.admin_api_update(user.info, %{is_moderator: value})
 

lib/pleroma/bbs/handler.ex

@@ -95,7 +95,6 @@ defmodule Pleroma.BBS.Handler do
     activities =
       [user.ap_id | user.following]
       |> ActivityPub.fetch_activities(params)
-      |> ActivityPub.contain_timeline(user)
 
     Enum.each(activities, fn activity ->
       puts_activity(activity)

lib/pleroma/filter.ex

@@ -38,7 +38,8 @@ defmodule Pleroma.Filter do
     query =
       from(
         f in Pleroma.Filter,
-        where: f.user_id == ^user_id
+        where: f.user_id == ^user_id,
+        order_by: [desc: :id]
       )
 
     Repo.all(query)

lib/pleroma/user.ex

@@ -1378,4 +1378,17 @@ defmodule Pleroma.User do
   def showing_reblogs?(%User{} = user, %User{} = target) do
     target.ap_id not in user.info.muted_reblogs
   end
+
+  @spec toggle_confirmation(User.t()) :: {:ok, User.t()} | {:error, Changeset.t()}
+  def toggle_confirmation(%User{} = user) do
+    need_confirmation? = !user.info.confirmation_pending
+
+    info_changeset =
+      User.Info.confirmation_changeset(user.info, need_confirmation: need_confirmation?)
+
+    user
+    |> change()
+    |> put_embed(:info, info_changeset)
+    |> update_and_set_cache()
+  end
 end

lib/pleroma/user/info.ex

@@ -212,7 +212,7 @@ defmodule Pleroma.User.Info do
     ])
   end
 
-  @spec confirmation_changeset(Info.t(), keyword()) :: Ecto.Changerset.t()
+  @spec confirmation_changeset(Info.t(), keyword()) :: Changeset.t()
   def confirmation_changeset(info, opts) do
     need_confirmation? = Keyword.get(opts, :need_confirmation)
 

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -527,6 +527,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp restrict_visibility(query, %{visibility: visibility})
        when is_list(visibility) do
     if Enum.all?(visibility, &(&1 in @valid_visibilities)) do
+      query =
         from(
           a in query,
           where:
@@ -538,6 +539,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
               ^visibility
             )
         )
+
+      query
     else
       Logger.error("Could not restrict visibility to #{visibility}")
     end
@@ -545,11 +548,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_visibility(query, %{visibility: visibility})
        when visibility in @valid_visibilities do
+    query =
       from(
         a in query,
         where:
           fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility)
       )
+
+    query
   end
 
   defp restrict_visibility(_query, %{visibility: visibility})
@@ -559,6 +565,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_visibility(query, _visibility), do: query
 
+  defp restrict_thread_visibility(query, %{"user" => %User{ap_id: ap_id}}) do
+    query =
+      from(
+        a in query,
+        where: fragment("thread_visibility(?, (?)->>'id') = true", ^ap_id, a.data)
+      )
+
+    query
+  end
+
+  defp restrict_thread_visibility(query, _), do: query
+
   def fetch_user_activities(user, reading_user, params \\ %{}) do
     params =
       params
@@ -838,6 +856,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> restrict_muted(opts)
     |> restrict_media(opts)
     |> restrict_visibility(opts)
+    |> restrict_thread_visibility(opts)
     |> restrict_replies(opts)
     |> restrict_reblogs(opts)
     |> restrict_pinned(opts)
@@ -956,14 +975,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     contain_broken_threads(activity, user)
   end
 
-  # do post-processing on a timeline
-  def contain_timeline(timeline, user) do
-    timeline
-    |> Enum.filter(fn activity ->
-      contain_activity(activity, user)
-    end)
-  end
-
   def fetch_direct_messages_query do
     Activity
     |> restrict_type(%{"type" => "Create"})

lib/pleroma/web/activity_pub/visibility.ex

@@ -1,6 +1,7 @@
 defmodule Pleroma.Web.ActivityPub.Visibility do
   alias Pleroma.Activity
   alias Pleroma.Object
+  alias Pleroma.Repo
   alias Pleroma.User
 
   def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false
@@ -39,25 +40,14 @@ defmodule Pleroma.Web.ActivityPub.Visibility do
     visible_for_user?(activity, nil) || Enum.any?(x, &(&1 in y))
   end
 
-  # guard
-  def entire_thread_visible_for_user?(nil, _user), do: false
+  def entire_thread_visible_for_user?(%Activity{} = activity, %User{} = user) do
+    {:ok, %{rows: [[result]]}} =
+      Ecto.Adapters.SQL.query(Repo, "SELECT thread_visibility($1, $2)", [
+        user.ap_id,
+        activity.data["id"]
+      ])
 
-  # XXX: Probably even more inefficient than the previous implementation intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop
-  # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
-
-  def entire_thread_visible_for_user?(
-        %Activity{} = tail,
-        # %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail,
-        user
-      ) do
-    case Object.normalize(tail) do
-      %{data: %{"inReplyTo" => parent_id}} when is_binary(parent_id) ->
-        parent = Activity.get_in_reply_to_activity(tail)
-        visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user)
-
-      _ ->
-        visible_for_user?(tail, user)
-    end
+    result
   end
 
   def get_visibility(object) do

lib/pleroma/web/federator/publisher.ex

@@ -31,7 +31,7 @@ defmodule Pleroma.Web.Federator.Publisher do
   """
   @spec enqueue_one(module(), Map.t()) :: :ok
   def enqueue_one(module, %{} = params),
-    do: PleromaJobQueue.enqueue(:federation_outgoing, __MODULE__, [:publish_one, module, params])
+    do: PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_one, module, params])
 
   @spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()}
   def perform(:publish_one, module, params) do

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -303,7 +303,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     activities =
       [user.ap_id | user.following]
       |> ActivityPub.fetch_activities(params)
-      |> ActivityPub.contain_timeline(user)
       |> Enum.reverse()
 
     conn

lib/pleroma/web/twitter_api/twitter_api_controller.ex

@@ -101,9 +101,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
       |> Map.put("blocking_user", user)
       |> Map.put("user", user)
 
-    activities =
-      ActivityPub.fetch_activities([user.ap_id | user.following], params)
-      |> ActivityPub.contain_timeline(user)
+    activities = ActivityPub.fetch_activities([user.ap_id | user.following], params)
 
     conn
     |> put_view(ActivityView)

priv/repo/migrations/20190515222404_add_thread_visibility_function.exs

@@ -0,0 +1,73 @@
+defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do
+  use Ecto.Migration
+  @disable_ddl_transaction true
+
+  def up do
+    statement = """
+    CREATE OR REPLACE FUNCTION thread_visibility(actor varchar, activity_id varchar) RETURNS boolean AS $$
+    DECLARE
+      public varchar := 'https://www.w3.org/ns/activitystreams#Public';
+      child objects%ROWTYPE;
+      activity activities%ROWTYPE;
+      actor_user users%ROWTYPE;
+      author_fa varchar;
+      valid_recipients varchar[];
+    BEGIN
+      --- Fetch our actor.
+      SELECT * INTO actor_user FROM users WHERE users.ap_id = actor;
+
+      --- Fetch our initial activity.
+      SELECT * INTO activity FROM activities WHERE activities.data->>'id' = activity_id;
+
+      LOOP
+        --- Ensure that we have an activity before continuing.
+        --- If we don't, the thread is not satisfiable.
+        IF activity IS NULL THEN
+          RETURN false;
+        END IF;
+
+        --- We only care about Create activities.
+        IF activity.data->>'type' != 'Create' THEN
+          RETURN true;
+        END IF;
+
+        --- Normalize the child object into child.
+        SELECT * INTO child FROM objects
+        INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id'
+        WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id';
+
+        --- Fetch the author's AS2 following collection.
+        SELECT COALESCE(users.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor;
+
+        --- Prepare valid recipients array.
+        valid_recipients := ARRAY[actor, public];
+        IF ARRAY[author_fa] && actor_user.following THEN
+          valid_recipients := valid_recipients || author_fa;
+        END IF;
+
+        --- Check visibility.
+        IF NOT valid_recipients && activity.recipients THEN
+          --- activity not visible, break out of the loop
+          RETURN false;
+        END IF;
+
+        --- If there's a parent, load it and do this all over again.
+        IF (child.data->'inReplyTo' IS NOT NULL) AND (child.data->'inReplyTo' != 'null'::jsonb) THEN
+          SELECT * INTO activity FROM activities
+          INNER JOIN objects ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id'
+          WHERE child.data->>'inReplyTo' = objects.data->>'id';
+        ELSE
+          RETURN true;
+        END IF;
+      END LOOP;
+    END;
+    $$ LANGUAGE plpgsql IMMUTABLE;
+    """
+
+    execute(statement)
+  end
+
+  def down do
+    execute("drop function thread_visibility(actor varchar, activity_id varchar)")
+  end
+end

test/tasks/user_test.exs

@@ -338,4 +338,31 @@ defmodule Mix.Tasks.Pleroma.UserTest do
       assert message == "User #{nickname} statuses deleted."
     end
   end
+
+  describe "running toggle_confirmed" do
+    test "user is confirmed" do
+      %{id: id, nickname: nickname} = insert(:user, info: %{confirmation_pending: false})
+
+      assert :ok = Mix.Tasks.Pleroma.User.run(["toggle_confirmed", nickname])
+      assert_received {:mix_shell, :info, [message]}
+      assert message == "#{nickname} needs confirmation."
+
+      user = Repo.get(User, id)
+      assert user.info.confirmation_pending
+      assert user.info.confirmation_token
+    end
+
+    test "user is not confirmed" do
+      %{id: id, nickname: nickname} =
+        insert(:user, info: %{confirmation_pending: true, confirmation_token: "some token"})
+
+      assert :ok = Mix.Tasks.Pleroma.User.run(["toggle_confirmed", nickname])
+      assert_received {:mix_shell, :info, [message]}
+      assert message == "#{nickname} doesn't need confirmation."
+
+      user = Repo.get(User, id)
+      refute user.info.confirmation_pending
+      refute user.info.confirmation_token
+    end
+  end
 end

test/user_test.exs

@@ -873,7 +873,6 @@ defmodule Pleroma.UserTest do
 
       assert [activity] ==
                ActivityPub.fetch_activities([user2.ap_id | user2.following], %{"user" => user2})
-               |> ActivityPub.contain_timeline(user2)
 
       {:ok, _user} = User.deactivate(user)
 
@@ -882,7 +881,6 @@ defmodule Pleroma.UserTest do
 
       assert [] ==
                ActivityPub.fetch_activities([user2.ap_id | user2.following], %{"user" => user2})
-               |> ActivityPub.contain_timeline(user2)
     end
   end
 
@@ -1204,4 +1202,22 @@ defmodule Pleroma.UserTest do
 
     assert Map.get(user_show, "followers_count") == 2
   end
+
+  describe "toggle_confirmation/1" do
+    test "if user is confirmed" do
+      user = insert(:user, info: %{confirmation_pending: false})
+      {:ok, user} = User.toggle_confirmation(user)
+
+      assert user.info.confirmation_pending
+      assert user.info.confirmation_token
+    end
+
+    test "if user is unconfirmed" do
+      user = insert(:user, info: %{confirmation_pending: true, confirmation_token: "some token"})
+      {:ok, user} = User.toggle_confirmation(user)
+
+      refute user.info.confirmation_pending
+      refute user.info.confirmation_token
+    end
+  end
 end

test/web/activity_pub/activity_pub_test.exs

@@ -960,17 +960,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
           "in_reply_to_status_id" => private_activity_2.id
         })
 
-      activities = ActivityPub.fetch_activities([user1.ap_id | user1.following])
+      activities =
+        ActivityPub.fetch_activities([user1.ap_id | user1.following])
+        |> Enum.map(fn a -> a.id end)
 
       private_activity_1 = Activity.get_by_ap_id_with_object(private_activity_1.data["id"])
 
-      assert [public_activity, private_activity_1, private_activity_3] == activities
+      assert [public_activity.id, private_activity_1.id, private_activity_3.id] == activities
 
       assert length(activities) == 3
 
-      activities = ActivityPub.contain_timeline(activities, user1)
+      activities =
+        ActivityPub.fetch_activities([user1.ap_id | user1.following], %{"user" => user1})
+        |> Enum.map(fn a -> a.id end)
 
-      assert [public_activity, private_activity_1] == activities
+      assert [public_activity.id, private_activity_1.id] == activities
       assert length(activities) == 2
     end
   end