From 23ca5f75afa7369ff52772c39dc3324e9402b230 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 8 Sep 2020 16:39:08 -0500 Subject: [PATCH 01/49] Make it possible to bulk send confirmation emails to all unconfirmed users --- docs/administration/CLI_tasks/email.md | 16 +++++++++++++++- lib/mix/tasks/pleroma/email.ex | 19 ++++++++++++++++++- lib/pleroma/user/query.ex | 4 ++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/docs/administration/CLI_tasks/email.md b/docs/administration/CLI_tasks/email.md index 00d2e74f8..439ea877a 100644 --- a/docs/administration/CLI_tasks/email.md +++ b/docs/administration/CLI_tasks/email.md @@ -1,4 +1,4 @@ -# Managing emails +# E-Mail administration tasks {! backend/administration/CLI_tasks/general_cli_task_info.include !} @@ -30,3 +30,17 @@ Example: ```sh mix pleroma.email test --to root@example.org ``` + +## Send confirmation emails to all unconfirmed user accounts + +=== "OTP" + + ```sh + ./bin/pleroma_ctl email send_confirmation_mails + ``` + +=== "From Source" + + ```sh + mix pleroma.email send_confirmation_mails + ``` diff --git a/lib/mix/tasks/pleroma/email.ex b/lib/mix/tasks/pleroma/email.ex index d3fac6ec8..61d431971 100644 --- a/lib/mix/tasks/pleroma/email.ex +++ b/lib/mix/tasks/pleroma/email.ex @@ -2,7 +2,7 @@ defmodule Mix.Tasks.Pleroma.Email do use Mix.Task import Mix.Pleroma - @shortdoc "Simple Email test" + @shortdoc "Email administrative tasks" @moduledoc File.read!("docs/administration/CLI_tasks/email.md") def run(["test" | args]) do @@ -21,4 +21,21 @@ defmodule Mix.Tasks.Pleroma.Email do shell_info("Test email has been sent to #{inspect(email.to)} from #{inspect(email.from)}") end + + def run(["resend_confirmation_emails"]) do + start_pleroma() + + Pleroma.User.Query.build(%{ + local: true, + deactivated: false, + confirmation_pending: true, + invisible: false + }) + |> Pleroma.RepoStreamer.chunk_stream(500) + |> Stream.each(fn users -> + users + |> Enum.each(fn user -> Pleroma.User.send_confirmation_email(user) end) + end) + |> Stream.run() + end end diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index d618432ff..f59ca6f9c 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -148,6 +148,10 @@ defmodule Pleroma.User.Query do |> where([u], not is_nil(u.nickname)) end + defp compose_query({:confirmation_pending, bool}, query) do + where(query, [u], u.confirmation_pending == ^bool) + end + defp compose_query({:need_approval, _}, query) do where(query, [u], u.approval_pending) end From 75b6fef25dd81c81cd5709739f97182045eec5b8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 8 Sep 2020 16:39:41 -0500 Subject: [PATCH 02/49] Add mix task for bulk [un]confirming the local instance users --- docs/administration/CLI_tasks/user.md | 37 +++++++++++++-- lib/mix/tasks/pleroma/user.ex | 66 ++++++++++++++++++++++++--- lib/pleroma/user.ex | 7 +++ lib/pleroma/user/query.ex | 8 ++-- 4 files changed, 105 insertions(+), 13 deletions(-) diff --git a/docs/administration/CLI_tasks/user.md b/docs/administration/CLI_tasks/user.md index 3e7f028ba..0fcc8cfb9 100644 --- a/docs/administration/CLI_tasks/user.md +++ b/docs/administration/CLI_tasks/user.md @@ -224,9 +224,10 @@ ``` ### Options -- `--locked`/`--no-locked` - whether the user should be locked -- `--moderator`/`--no-moderator` - whether the user should be a moderator -- `--admin`/`--no-admin` - whether the user should be an admin +- `--admin`/`--no-admin` - the user account admin status +- `--confirmed`/`--no-confirmed` - the user account confirmation status +- `--locked`/`--no-locked` - the user account locked status +- `--moderator`/`--no-moderator` - the user account moderator status ## Add tags to a user @@ -271,3 +272,33 @@ ```sh mix pleroma.user toggle_confirmed ``` + +## Set confirmation status for all regular active users +*Admins and moderators are excluded* + +=== "OTP" + + ```sh + ./bin/pleroma_ctl user confirm_all + ``` + +=== "From Source" + + ```sh + mix pleroma.user confirm_all + ``` + +## Revoke confirmation status for all regular active users +*Admins and moderators are excluded* + +=== "OTP" + + ```sh + ./bin/pleroma_ctl user unconfirm_all + ``` + +=== "From Source" + + ```sh + mix pleroma.user unconfirm_all + ``` diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 01824aa18..4073fe5a4 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -196,17 +196,24 @@ defmodule Mix.Tasks.Pleroma.User do OptionParser.parse( rest, strict: [ - moderator: :boolean, admin: :boolean, - locked: :boolean + confirmed: :boolean, + locked: :boolean, + moderator: :boolean ] ) with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do user = - case Keyword.get(options, :moderator) do + case Keyword.get(options, :admin) do nil -> user - value -> set_moderator(user, value) + value -> set_admin(user, value) + end + + user = + case Keyword.get(options, :confirmed) do + nil -> user + value -> set_confirmed(user, value) end user = @@ -216,9 +223,9 @@ defmodule Mix.Tasks.Pleroma.User do end _user = - case Keyword.get(options, :admin) do + case Keyword.get(options, :moderator) do nil -> user - value -> set_admin(user, value) + value -> set_moderator(user, value) end else _ -> @@ -353,6 +360,42 @@ defmodule Mix.Tasks.Pleroma.User do end end + def run(["confirm_all"]) do + start_pleroma() + + Pleroma.User.Query.build(%{ + local: true, + deactivated: false, + is_moderator: false, + is_admin: false, + invisible: false + }) + |> Pleroma.RepoStreamer.chunk_stream(500) + |> Stream.each(fn users -> + users + |> Enum.each(fn user -> User.need_confirmation(user, false) end) + end) + |> Stream.run() + end + + def run(["unconfirm_all"]) do + start_pleroma() + + Pleroma.User.Query.build(%{ + local: true, + deactivated: false, + is_moderator: false, + is_admin: false, + invisible: false + }) + |> Pleroma.RepoStreamer.chunk_stream(500) + |> Stream.each(fn users -> + users + |> Enum.each(fn user -> User.need_confirmation(user, true) end) + end) + |> Stream.run() + end + def run(["sign_out", nickname]) do start_pleroma() @@ -410,4 +453,15 @@ defmodule Mix.Tasks.Pleroma.User do shell_info("Locked status of #{user.nickname}: #{user.locked}") user end + + defp set_confirmed(user, value) do + {:ok, user} = + case value do + true -> User.need_confirmation(user, false) + false -> User.need_confirmation(user, true) + end + + shell_info("Confirmation pending status of #{user.nickname}: #{user.confirmation_pending}") + user + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f323fc6ed..603fc3b44 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2123,6 +2123,13 @@ defmodule Pleroma.User do Enum.map(users, &toggle_confirmation/1) end + @spec need_confirmation(User.t(), boolean()) :: {:ok, User.t()} | {:error, Changeset.t()} + def need_confirmation(%User{} = user, bool) do + user + |> confirmation_changeset(need_confirmation: bool) + |> update_and_set_cache() + end + def get_mascot(%{mascot: %{} = mascot}) when not is_nil(mascot) do mascot end diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index f59ca6f9c..64bb24c0e 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -107,12 +107,12 @@ defmodule Pleroma.User.Query do where(query, [u], fragment("? && ?", u.tags, ^tags)) end - defp compose_query({:is_admin, _}, query) do - where(query, [u], u.is_admin) + defp compose_query({:is_admin, bool}, query) do + where(query, [u], u.is_admin == ^bool) end - defp compose_query({:is_moderator, _}, query) do - where(query, [u], u.is_moderator) + defp compose_query({:is_moderator, bool}, query) do + where(query, [u], u.is_moderator == ^bool) end defp compose_query({:super_users, _}, query) do From d23d0c27c25e3ce7b39cae6e504062b4cb389ea4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 8 Sep 2020 16:48:54 -0500 Subject: [PATCH 03/49] Handle possibility of user account in a bulk operation not having an email address --- lib/mix/tasks/pleroma/email.ex | 2 +- lib/pleroma/user.ex | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/pleroma/email.ex b/lib/mix/tasks/pleroma/email.ex index 61d431971..c0bef0386 100644 --- a/lib/mix/tasks/pleroma/email.ex +++ b/lib/mix/tasks/pleroma/email.ex @@ -34,7 +34,7 @@ defmodule Mix.Tasks.Pleroma.Email do |> Pleroma.RepoStreamer.chunk_stream(500) |> Stream.each(fn users -> users - |> Enum.each(fn user -> Pleroma.User.send_confirmation_email(user) end) + |> Enum.each(fn user -> Pleroma.User.try_send_confirmation_email(user) end) end) |> Stream.run() end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 603fc3b44..9dea39619 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -814,7 +814,8 @@ defmodule Pleroma.User do def send_welcome_email(_), do: {:ok, :noop} @spec try_send_confirmation_email(User.t()) :: {:ok, :enqueued | :noop} - def try_send_confirmation_email(%User{confirmation_pending: true} = user) do + def try_send_confirmation_email(%User{confirmation_pending: true, email: email} = user) + when is_binary(email) do if Config.get([:instance, :account_activation_required]) do send_confirmation_email(user) {:ok, :enqueued} From ed847474c305386adbcb97c569a0861d6fad5284 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 8 Sep 2020 16:59:53 -0500 Subject: [PATCH 04/49] Fix descriptions --- docs/administration/CLI_tasks/user.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/administration/CLI_tasks/user.md b/docs/administration/CLI_tasks/user.md index 0fcc8cfb9..c64ed4f22 100644 --- a/docs/administration/CLI_tasks/user.md +++ b/docs/administration/CLI_tasks/user.md @@ -224,10 +224,10 @@ ``` ### Options -- `--admin`/`--no-admin` - the user account admin status -- `--confirmed`/`--no-confirmed` - the user account confirmation status -- `--locked`/`--no-locked` - the user account locked status -- `--moderator`/`--no-moderator` - the user account moderator status +- `--admin`/`--no-admin` - whether the user should be an admin +- `--confirmed`/`--no-confirmed` - whether the user account is confirmed +- `--locked`/`--no-locked` - whether the user should be locked +- `--moderator`/`--no-moderator` - whether the user should be a moderator ## Add tags to a user From bccef4b9973e6fff898e819ff9f0234ad0cbf190 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 8 Sep 2020 17:01:55 -0500 Subject: [PATCH 05/49] Pedantry --- docs/administration/CLI_tasks/email.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration/CLI_tasks/email.md b/docs/administration/CLI_tasks/email.md index 439ea877a..d9aa0e71b 100644 --- a/docs/administration/CLI_tasks/email.md +++ b/docs/administration/CLI_tasks/email.md @@ -1,4 +1,4 @@ -# E-Mail administration tasks +# EMail administration tasks {! backend/administration/CLI_tasks/general_cli_task_info.include !} From 6c79a60649c8d6b3ef9ce0fbbb4792410fe585bd Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 8 Sep 2020 17:59:25 -0500 Subject: [PATCH 06/49] Add test for pleroma.user set --confirmed Order now matters because of testing shell_info --- test/tasks/user_test.exs | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/test/tasks/user_test.exs b/test/tasks/user_test.exs index ce43a9cc7..ef77fdc9c 100644 --- a/test/tasks/user_test.exs +++ b/test/tasks/user_test.exs @@ -225,47 +225,64 @@ defmodule Mix.Tasks.Pleroma.UserTest do test "All statuses set" do user = insert(:user) - Mix.Tasks.Pleroma.User.run(["set", user.nickname, "--moderator", "--admin", "--locked"]) + Mix.Tasks.Pleroma.User.run([ + "set", + user.nickname, + "--admin", + "--confirmed", + "--locked", + "--moderator" + ]) assert_received {:mix_shell, :info, [message]} - assert message =~ ~r/Moderator status .* true/ + assert message =~ ~r/Admin status .* true/ + + assert_received {:mix_shell, :info, [message]} + assert message =~ ~r/Confirmation pending .* false/ assert_received {:mix_shell, :info, [message]} assert message =~ ~r/Locked status .* true/ assert_received {:mix_shell, :info, [message]} - assert message =~ ~r/Admin status .* true/ + assert message =~ ~r/Moderator status .* true/ user = User.get_cached_by_nickname(user.nickname) assert user.is_moderator assert user.locked assert user.is_admin + refute user.confirmation_pending end test "All statuses unset" do - user = insert(:user, locked: true, is_moderator: true, is_admin: true) + user = + insert(:user, locked: true, is_moderator: true, is_admin: true, confirmation_pending: true) Mix.Tasks.Pleroma.User.run([ "set", user.nickname, - "--no-moderator", "--no-admin", - "--no-locked" + "--no-confirmed", + "--no-locked", + "--no-moderator" ]) assert_received {:mix_shell, :info, [message]} - assert message =~ ~r/Moderator status .* false/ + assert message =~ ~r/Admin status .* false/ + + assert_received {:mix_shell, :info, [message]} + assert message =~ ~r/Confirmation pending .* true/ assert_received {:mix_shell, :info, [message]} assert message =~ ~r/Locked status .* false/ assert_received {:mix_shell, :info, [message]} - assert message =~ ~r/Admin status .* false/ + assert message =~ ~r/Moderator status .* false/ user = User.get_cached_by_nickname(user.nickname) refute user.is_moderator refute user.locked refute user.is_admin + assert user.confirmation_pending end test "no user to set status" do From b900c06d4e2bc5d607af542e2c9cf9eacade376b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 9 Sep 2020 09:02:07 -0500 Subject: [PATCH 07/49] Add tests for the bulk confirm/unconfirm tasks --- test/tasks/user_test.exs | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/tasks/user_test.exs b/test/tasks/user_test.exs index ef77fdc9c..b8c423c48 100644 --- a/test/tasks/user_test.exs +++ b/test/tasks/user_test.exs @@ -571,4 +571,44 @@ defmodule Mix.Tasks.Pleroma.UserTest do assert message =~ "Could not change user tags" end end + + describe "bulk confirm and unconfirm" do + test "confirm all" do + user1 = insert(:user, confirmation_pending: true) + user2 = insert(:user, confirmation_pending: true) + + assert user1.confirmation_pending + assert user2.confirmation_pending + + Mix.Tasks.Pleroma.User.run(["confirm_all"]) + + user1 = User.get_cached_by_nickname(user1.nickname) + user2 = User.get_cached_by_nickname(user2.nickname) + + refute user1.confirmation_pending + refute user2.confirmation_pending + end + + test "unconfirm all" do + user1 = insert(:user, confirmation_pending: false) + user2 = insert(:user, confirmation_pending: false) + admin = insert(:user, is_admin: true, confirmation_pending: false) + mod = insert(:user, is_moderator: true, confirmation_pending: false) + + refute user1.confirmation_pending + refute user2.confirmation_pending + + Mix.Tasks.Pleroma.User.run(["unconfirm_all"]) + + user1 = User.get_cached_by_nickname(user1.nickname) + user2 = User.get_cached_by_nickname(user2.nickname) + admin = User.get_cached_by_nickname(admin.nickname) + mod = User.get_cached_by_nickname(mod.nickname) + + assert user1.confirmation_pending + assert user2.confirmation_pending + refute admin.confirmation_pending + refute mod.confirmation_pending + end + end end From 34d7e864db8f9cc7fb73ce2fef8466ce8e09ed85 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 9 Sep 2020 09:10:44 -0500 Subject: [PATCH 08/49] New mix tasks for controlling user confirmation status and sending confirmation mails --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19b2596cc..47cab144b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +### Added +- Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`) +- Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`) + ### Changed - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated. From e5927e92a641a799f9b4e89466374be3c76c4ef2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 21 Sep 2020 17:08:49 -0500 Subject: [PATCH 09/49] Fix deprecation warning for welcome message --- lib/pleroma/config/deprecation_warnings.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 98c4dc9c8..51ddb2889 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -83,9 +83,9 @@ defmodule Pleroma.Config.DeprecationWarnings do if use_old_config do Logger.error(""" !!!DEPRECATION WARNING!!! - Your config is using the old namespace for Welcome messages configuration. You need to change to the new namespace: - \n* `config :pleroma, :instance, welcome_user_nickname` is now `config :pleroma, :welcome, :direct_message, :sender_nickname` - \n* `config :pleroma, :instance, welcome_message` is now `config :pleroma, :welcome, :direct_message, :message` + Your config is using the old namespace for Welcome messages configuration. You need to convert to the new namespace. e.g., + \n* `config :pleroma, :instance, welcome_user_nickname` and `config :pleroma, :instance, welcome_message` are now equal to: + \n* `config :pleroma, :welcome, direct_message: [enabled: true, sender_nickname: "NICKNAME", message: "Your welcome message"]`" """) :error From 2fc3b46ee7e8ee8c22d4cf0039bbe9755a0e3beb Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Sep 2020 11:13:36 -0500 Subject: [PATCH 10/49] Remove deprecation warning. We were supposed to remove backwards compat for this in 2.1. --- lib/pleroma/config/deprecation_warnings.ex | 26 ---------------------- 1 file changed, 26 deletions(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 51ddb2889..091d02366 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -33,34 +33,8 @@ defmodule Pleroma.Config.DeprecationWarnings do end end - def mrf_user_allowlist do - config = Config.get(:mrf_user_allowlist) - - if config && Enum.any?(config, fn {k, _} -> is_atom(k) end) do - rewritten = - Enum.reduce(Config.get(:mrf_user_allowlist), Map.new(), fn {k, v}, acc -> - Map.put(acc, to_string(k), v) - end) - - Config.put(:mrf_user_allowlist, rewritten) - - Logger.error(""" - !!!DEPRECATION WARNING!!! - As of Pleroma 2.0.7, the `mrf_user_allowlist` setting changed of format. - Pleroma 2.1 will remove support for the old format. Please change your configuration to match this: - - config :pleroma, :mrf_user_allowlist, #{inspect(rewritten, pretty: true)} - """) - - :error - else - :ok - end - end - def warn do with :ok <- check_hellthread_threshold(), - :ok <- mrf_user_allowlist(), :ok <- check_old_mrf_config(), :ok <- check_media_proxy_whitelist_config(), :ok <- check_welcome_message_config(), From e2dcf039d24b1606c90cea75ef11c79b7677c209 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Sep 2020 11:15:40 -0500 Subject: [PATCH 11/49] Fix gun_pool_options deprecation warning message --- lib/pleroma/config/deprecation_warnings.ex | 2 +- test/config/deprecation_warnings_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 091d02366..4ba6eaa77 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -122,7 +122,7 @@ defmodule Pleroma.Config.DeprecationWarnings do if timeout = pool_config[:await_up_timeout] do Logger.warn(""" !!!DEPRECATION WARNING!!! - Your config is using old setting name `await_up_timeout` instead of `connect_timeout`. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later. + Your config is using old setting `config :pleroma, :connections_pool, await_up_timeout`. Please change to `config :pleroma, :connections_pool, connect_timeout` to ensure compatibility with future releases. """) Config.put(:connections_pool, Keyword.put_new(pool_config, :connect_timeout, timeout)) diff --git a/test/config/deprecation_warnings_test.exs b/test/config/deprecation_warnings_test.exs index e22052404..7f0d2a298 100644 --- a/test/config/deprecation_warnings_test.exs +++ b/test/config/deprecation_warnings_test.exs @@ -74,7 +74,7 @@ defmodule Pleroma.Config.DeprecationWarningsTest do assert capture_log(fn -> DeprecationWarnings.check_gun_pool_options() end) =~ - "Your config is using old setting name `await_up_timeout` instead of `connect_timeout`" + "Your config is using old setting `config :pleroma, :connections_pool, await_up_timeout`." end test "pool timeout" do From 25bdf0d0d95bf748f10c43d569640794a8a0d4c7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Sep 2020 11:19:29 -0500 Subject: [PATCH 12/49] Add test for welcome message format --- test/config/deprecation_warnings_test.exs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/config/deprecation_warnings_test.exs b/test/config/deprecation_warnings_test.exs index 7f0d2a298..e8140f58e 100644 --- a/test/config/deprecation_warnings_test.exs +++ b/test/config/deprecation_warnings_test.exs @@ -66,6 +66,14 @@ defmodule Pleroma.Config.DeprecationWarningsTest do end) =~ "Your config is using old format (only domain) for MediaProxy whitelist option" end + test "check_welcome_message_config/0" do + clear_config([:instance, :welcome_user_nickname], "LainChan") + + assert capture_log(fn -> + DeprecationWarnings.check_welcome_message_config() + end) =~ "Your config is using the old namespace for Welcome messages configuration." + end + describe "check_gun_pool_options/0" do test "await_up_timeout" do config = Config.get(:connections_pool) From 7775b1540f47f792f0afa7c49a2cf058e2f6470e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Sep 2020 11:22:15 -0500 Subject: [PATCH 13/49] Add deprecation warning test for check_hellthread_threshold/0 --- test/config/deprecation_warnings_test.exs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/config/deprecation_warnings_test.exs b/test/config/deprecation_warnings_test.exs index e8140f58e..0efc0843c 100644 --- a/test/config/deprecation_warnings_test.exs +++ b/test/config/deprecation_warnings_test.exs @@ -74,6 +74,14 @@ defmodule Pleroma.Config.DeprecationWarningsTest do end) =~ "Your config is using the old namespace for Welcome messages configuration." end + test "check_hellthread_threshold/0" do + clear_config([:mrf_hellthread, :threshold], 16) + + assert capture_log(fn -> + DeprecationWarnings.check_hellthread_threshold() + end) =~ "You are using the old configuration mechanism for the hellthread filter." + end + describe "check_gun_pool_options/0" do test "await_up_timeout" do config = Config.get(:connections_pool) From 88653c01c92fffb396e32edad203d18607980c04 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Sep 2020 11:34:51 -0500 Subject: [PATCH 14/49] Add test for check_activity_expiration_config/0 --- test/config/deprecation_warnings_test.exs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/config/deprecation_warnings_test.exs b/test/config/deprecation_warnings_test.exs index 0efc0843c..28355d7eb 100644 --- a/test/config/deprecation_warnings_test.exs +++ b/test/config/deprecation_warnings_test.exs @@ -82,6 +82,14 @@ defmodule Pleroma.Config.DeprecationWarningsTest do end) =~ "You are using the old configuration mechanism for the hellthread filter." end + test "check_activity_expiration_config/0" do + clear_config([Pleroma.ActivityExpiration, :enabled], true) + + assert capture_log(fn -> + DeprecationWarnings.check_activity_expiration_config() + end) =~ "Your config is using old namespace for activity expiration configuration." + end + describe "check_gun_pool_options/0" do test "await_up_timeout" do config = Config.get(:connections_pool) From 5e86a2809e37100b54e0fc88db79245e13f684aa Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 23 Sep 2020 11:45:32 +0200 Subject: [PATCH 15/49] transmogrifier: Drop incoming create early if it already exists --- lib/pleroma/web/activity_pub/transmogrifier.ex | 6 +++++- .../activity_pub/transmogrifier/question_handling_test.exs | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index aa6a69463..d7dd9fe6b 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -515,15 +515,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Create", "object" => %{"type" => objtype}} = data, + %{"type" => "Create", "object" => %{"type" => objtype, "id" => obj_id}} = data, _options ) when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do data = Map.put(data, "object", strip_internal_fields(data["object"])) with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), + nil <- Activity.get_create_by_object_ap_id(obj_id), {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do {:ok, activity} + else + %Activity{} = activity -> {:ok, activity} + e -> e end end diff --git a/test/web/activity_pub/transmogrifier/question_handling_test.exs b/test/web/activity_pub/transmogrifier/question_handling_test.exs index 74ee79543..d2822ce75 100644 --- a/test/web/activity_pub/transmogrifier/question_handling_test.exs +++ b/test/web/activity_pub/transmogrifier/question_handling_test.exs @@ -157,12 +157,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do } end - test "returns an error if received a second time" do + test "returns same activity if received a second time" do data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!() assert {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) - assert {:error, {:validate_object, {:error, _}}} = Transmogrifier.handle_incoming(data) + assert {:ok, ^activity} = Transmogrifier.handle_incoming(data) end test "accepts a Question with no content" do From 8af8eb5ce79a35e01415763164533c537e017776 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 23 Sep 2020 12:32:47 -0500 Subject: [PATCH 16/49] Chase Pleroma.RepoStreamer.chunk_stream -> Pleroma.Repo.chunk_stream --- lib/mix/tasks/pleroma/email.ex | 7 ++----- lib/mix/tasks/pleroma/user.ex | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/mix/tasks/pleroma/email.ex b/lib/mix/tasks/pleroma/email.ex index c0bef0386..1f543241a 100644 --- a/lib/mix/tasks/pleroma/email.ex +++ b/lib/mix/tasks/pleroma/email.ex @@ -31,11 +31,8 @@ defmodule Mix.Tasks.Pleroma.Email do confirmation_pending: true, invisible: false }) - |> Pleroma.RepoStreamer.chunk_stream(500) - |> Stream.each(fn users -> - users - |> Enum.each(fn user -> Pleroma.User.try_send_confirmation_email(user) end) - end) + |> Pleroma.Repo.chunk_stream(500, :batches) + |> Stream.each(&Pleroma.User.try_send_confirmation_email(&1)) |> Stream.run() end end diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 8196e34b1..d50205600 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -370,7 +370,7 @@ defmodule Mix.Tasks.Pleroma.User do is_admin: false, invisible: false }) - |> Pleroma.RepoStreamer.chunk_stream(500) + |> Pleroma.Repo.chunk_stream(500, :batches) |> Stream.each(fn users -> users |> Enum.each(fn user -> User.need_confirmation(user, false) end) @@ -388,7 +388,7 @@ defmodule Mix.Tasks.Pleroma.User do is_admin: false, invisible: false }) - |> Pleroma.RepoStreamer.chunk_stream(500) + |> Pleroma.Repo.chunk_stream(500, :batches) |> Stream.each(fn users -> users |> Enum.each(fn user -> User.need_confirmation(user, true) end) From aa1f97a5b6904aec38621c9af79a81b5a7b62e30 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 24 Sep 2020 10:46:09 +0300 Subject: [PATCH 17/49] fix for test on mac --- test/utils_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/utils_test.exs b/test/utils_test.exs index 3a730d545..460f7e0b5 100644 --- a/test/utils_test.exs +++ b/test/utils_test.exs @@ -8,7 +8,7 @@ defmodule Pleroma.UtilsTest do describe "tmp_dir/1" do test "returns unique temporary directory" do {:ok, path} = Pleroma.Utils.tmp_dir("emoji") - assert path =~ ~r/\/tmp\/emoji-(.*)-#{:os.getpid()}-(.*)/ + assert path =~ ~r/\/emoji-(.*)-#{:os.getpid()}-(.*)/ File.rm_rf(path) end end From 165961f56dfa20d27869f53777798a8d4ce572f9 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 24 Sep 2020 12:00:39 +0300 Subject: [PATCH 18/49] don't run in async mode --- test/emoji_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/emoji_test.exs b/test/emoji_test.exs index b36047578..1dd3c58c6 100644 --- a/test/emoji_test.exs +++ b/test/emoji_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.EmojiTest do - use ExUnit.Case, async: true + use ExUnit.Case alias Pleroma.Emoji describe "is_unicode_emoji?/1" do From 35d62a4a5602913de3a91582aa03d5af9ac7a5b0 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 24 Sep 2020 11:12:03 +0200 Subject: [PATCH 19/49] CommonAPI test: Add test for polls --- test/web/common_api/common_api_test.exs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 2eab64e8b..e34f5a49b 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -29,6 +29,23 @@ defmodule Pleroma.Web.CommonAPITest do setup do: clear_config([:instance, :limit]) setup do: clear_config([:instance, :max_pinned_statuses]) + describe "posting polls" do + test "it posts a poll" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "who is the best", + poll: %{expires_in: 600, options: ["reimu", "marisa"]} + }) + + object = Object.normalize(activity) + + assert object.data["type"] == "Question" + assert object.data["oneOf"] |> length() == 2 + end + end + describe "blocking" do setup do blocker = insert(:user) From d0078bc40406939cb584847c90b00aad006812e4 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 24 Sep 2020 15:54:55 +0200 Subject: [PATCH 20/49] User Search: Boost resolved results and exact ap_id matches. --- lib/pleroma/user/search.ex | 45 +++++++++++++++++++++++++++++++++----- test/user_search_test.exs | 19 ++++++++++++++++ 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index b8c648672..408295e0c 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -19,11 +19,21 @@ defmodule Pleroma.User.Search do query_string = format_query(query_string) - maybe_resolve(resolve, for_user, query_string) + # If this returns anything, it should bounce to the top + maybe_resolved = maybe_resolve(resolve, for_user, query_string) + maybe_ap_id_match = User.get_cached_by_ap_id(query_string) + + top_user_ids = + case {maybe_resolved, maybe_ap_id_match} do + {{:ok, %User{} = user}, %User{} = other_user} -> [user.id, other_user.id] + {{:ok, %User{} = user}, _} -> [user.id] + {_, %User{} = user} -> [user.id] + _ -> [] + end results = query_string - |> search_query(for_user, following) + |> search_query(for_user, following, top_user_ids) |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset) results @@ -47,7 +57,7 @@ defmodule Pleroma.User.Search do end end - defp search_query(query_string, for_user, following) do + defp search_query(query_string, for_user, following, top_user_ids) do for_user |> base_query(following) |> filter_blocked_user(for_user) @@ -56,13 +66,20 @@ defmodule Pleroma.User.Search do |> filter_internal_users() |> filter_blocked_domains(for_user) |> fts_search(query_string) + |> select_top_users(top_user_ids) |> trigram_rank(query_string) - |> boost_search_rank(for_user) + |> boost_search_rank(for_user, top_user_ids) |> subquery() |> order_by(desc: :search_rank) |> maybe_restrict_local(for_user) end + defp select_top_users(query, top_user_ids) do + from(u in query, + or_where: u.id in ^top_user_ids + ) + end + defp fts_search(query, query_string) do query_string = to_tsquery(query_string) @@ -180,7 +197,7 @@ defmodule Pleroma.User.Search do defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) - defp boost_search_rank(query, %User{} = for_user) do + defp boost_search_rank(query, %User{} = for_user, top_user_ids) do friends_ids = User.get_friends_ids(for_user) followers_ids = User.get_followers_ids(for_user) @@ -192,6 +209,7 @@ defmodule Pleroma.User.Search do CASE WHEN (?) THEN (?) * 1.5 WHEN (?) THEN (?) * 1.3 WHEN (?) THEN (?) * 1.1 + WHEN (?) THEN 9001 ELSE (?) END """, u.id in ^friends_ids and u.id in ^followers_ids, @@ -200,11 +218,26 @@ defmodule Pleroma.User.Search do u.search_rank, u.id in ^followers_ids, u.search_rank, + u.id in ^top_user_ids, u.search_rank ) } ) end - defp boost_search_rank(query, _for_user), do: query + defp boost_search_rank(query, _for_user, top_user_ids) do + from(u in subquery(query), + select_merge: %{ + search_rank: + fragment( + """ + CASE WHEN (?) THEN 9001 + ELSE (?) END + """, + u.id in ^top_user_ids, + u.search_rank + ) + } + ) + end end diff --git a/test/user_search_test.exs b/test/user_search_test.exs index 8529ce6db..68fda1c53 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -17,6 +17,25 @@ defmodule Pleroma.UserSearchTest do describe "User.search" do setup do: clear_config([:instance, :limit_to_local_content]) + test "returns a resolved user as the first result" do + Pleroma.Config.put([:instance, :limit_to_local_content], false) + user = insert(:user, %{nickname: "no_relation", ap_id: "https://lain.com/users/lain"}) + _user = insert(:user, %{nickname: "com_user"}) + + [first_user, _second_user] = User.search("https://lain.com/users/lain", resolve: true) + + assert first_user.id == user.id + end + + test "returns a user with matching ap_id as the first result" do + user = insert(:user, %{nickname: "no_relation", ap_id: "https://lain.com/users/lain"}) + _user = insert(:user, %{nickname: "com_user"}) + + [first_user, _second_user] = User.search("https://lain.com/users/lain") + + assert first_user.id == user.id + end + test "excludes invisible users from results" do user = insert(:user, %{nickname: "john t1000"}) insert(:user, %{invisible: true, nickname: "john t800"}) From defca4a8b1e5baa03b308e77bf2386f135b2584d Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 24 Sep 2020 15:56:17 +0200 Subject: [PATCH 21/49] Changelog: Add info about user search. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f5d01af3..1a1de99a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed +- Search: Users are now findable by their urls. - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated. - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated. - The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false. From 7cd662f18c202ee2b099305e990d15a01370d6f9 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 24 Sep 2020 18:02:50 +0300 Subject: [PATCH 22/49] Added documentation on installation of `ffmpeg` and `ImageMagick` dependencies. --- CHANGELOG.md | 3 ++- docs/installation/alpine_linux_en.md | 9 +++++++++ docs/installation/arch_linux_en.md | 4 +++- docs/installation/debian_based_en.md | 4 +++- docs/installation/debian_based_jp.md | 4 +++- docs/installation/freebsd_en.md | 2 +- docs/installation/gentoo_en.md | 4 +++- docs/installation/netbsd_en.md | 2 +- docs/installation/openbsd_en.md | 5 +++-- docs/installation/openbsd_fi.md | 2 +- docs/installation/otp_en.md | 6 ++++-- 11 files changed, 33 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f5d01af3..3e5f8a504 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,9 +12,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false. - Users with the `discoverable` field set to false will not show up in searches. - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option). +- **Breaking:** Introduced dependencies on `ffmpeg` and `ImageMagick` software packages. Please refer to documentation in `docs/installation`. ### Added -- Media preview proxy (requires media proxy be enabled; see `:media_preview_proxy` config for more details). +- Media preview proxy (requires `ffmpeg` and `ImageMagick` be installed and media proxy be enabled; see `:media_preview_proxy` config for more details). - Pleroma API: Importing the mutes users from CSV files. - Experimental websocket-based federation between Pleroma instances. - Admin API: Importing emoji from a zip file diff --git a/docs/installation/alpine_linux_en.md b/docs/installation/alpine_linux_en.md index a5683f18c..0b9f5cdae 100644 --- a/docs/installation/alpine_linux_en.md +++ b/docs/installation/alpine_linux_en.md @@ -13,6 +13,8 @@ It assumes that you have administrative rights, either as root or a user with [s * `erlang-parsetools` * `erlang-xmerl` * `git` +* `ffmpeg` +* `ImageMagick` * Development Tools * `cmake` @@ -56,6 +58,13 @@ sudo apk add erlang erlang-runtime-tools erlang-xmerl elixir ```shell sudo apk add erlang-eldap ``` + +### Install ffmpeg and ImageMagick + +```shell +sudo apk add ffmpeg imagemagick +``` + ### Install PostgreSQL * Install Postgresql server: diff --git a/docs/installation/arch_linux_en.md b/docs/installation/arch_linux_en.md index 7fb69dd60..38bb82432 100644 --- a/docs/installation/arch_linux_en.md +++ b/docs/installation/arch_linux_en.md @@ -10,6 +10,8 @@ This guide will assume that you have administrative rights, either as root or a * `git` * `base-devel` * `cmake` +* `ffmpeg` +* `ImageMagick` #### Optional packages used in this guide @@ -27,7 +29,7 @@ sudo pacman -Syu * Install some of the above mentioned programs: ```shell -sudo pacman -S git base-devel elixir cmake +sudo pacman -S git base-devel elixir cmake ffmpeg imagemagick ``` ### Install PostgreSQL diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md index 60c2f47e5..30ee64c01 100644 --- a/docs/installation/debian_based_en.md +++ b/docs/installation/debian_based_en.md @@ -13,6 +13,8 @@ This guide will assume you are on Debian Stretch. This guide should also work wi * `git` * `build-essential` * `cmake` +* `ffmpeg` +* `ImageMagick` #### Optional packages used in this guide @@ -31,7 +33,7 @@ sudo apt full-upgrade * Install some of the above mentioned programs: ```shell -sudo apt install git build-essential postgresql postgresql-contrib cmake +sudo apt install git build-essential postgresql postgresql-contrib cmake ffmpeg imagemagick ``` ### Install Elixir and Erlang diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index c2dd840d3..219f40525 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -17,6 +17,8 @@ - `git` - `build-essential` - `cmake` +- `ffmpeg` +- `ImageMagick` #### このガイドで利用している追加パッケージ @@ -33,7 +35,7 @@ sudo apt full-upgrade * 上記に挙げたパッケージをインストールしておきます。 ``` -sudo apt install git build-essential postgresql postgresql-contrib cmake +sudo apt install git build-essential postgresql postgresql-contrib cmake ffmpeg imagemagick ``` diff --git a/docs/installation/freebsd_en.md b/docs/installation/freebsd_en.md index ca2575d9b..01b410a46 100644 --- a/docs/installation/freebsd_en.md +++ b/docs/installation/freebsd_en.md @@ -7,7 +7,7 @@ This document was written for FreeBSD 12.1, but should be work on future release This assumes the target system has `pkg(8)`. ``` -# pkg install elixir postgresql12-server postgresql12-client postgresql12-contrib git-lite sudo nginx gmake acme.sh cmake +# pkg install elixir postgresql12-server postgresql12-client postgresql12-contrib git-lite sudo nginx gmake acme.sh cmake ffmpeg imagemagick ``` Copy the rc.d scripts to the right directory: diff --git a/docs/installation/gentoo_en.md b/docs/installation/gentoo_en.md index 5a676380c..9a8b54ece 100644 --- a/docs/installation/gentoo_en.md +++ b/docs/installation/gentoo_en.md @@ -29,6 +29,8 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i * `dev-lang/elixir` * `dev-vcs/git` * `dev-util/cmake` +* `media-video/ffmpeg` +* `media-gfx/imagemagick` #### Optional ebuilds used in this guide @@ -47,7 +49,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i * Emerge all required the required and suggested software in one go: ```shell - # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake + # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake media-video/ffmpeg media-gfx/imagemagick ``` If you would not like to install the optional packages, remove them from this line. diff --git a/docs/installation/netbsd_en.md b/docs/installation/netbsd_en.md index 6ad0de2f6..8cf2e4682 100644 --- a/docs/installation/netbsd_en.md +++ b/docs/installation/netbsd_en.md @@ -10,7 +10,7 @@ Pleroma uses. The `mksh` shell is needed to run the Elixir `mix` script. -`# pkgin install acmesh elixir git-base git-docs mksh nginx postgresql11-server postgresql11-client postgresql11-contrib sudo` +`# pkgin install acmesh elixir git-base git-docs mksh nginx postgresql11-server postgresql11-client postgresql11-contrib sudo ffmpeg4 ImageMagick` You can also build these packages using pkgsrc: ``` diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index eee452845..277e5ec43 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -10,16 +10,17 @@ The following packages need to be installed: * elixir * gmake - * ImageMagick * git * postgresql-server * postgresql-contrib * cmake + * ffmpeg + * ImageMagick To install them, run the following command (with doas or as root): ``` -pkg_add elixir gmake ImageMagick git postgresql-server postgresql-contrib cmake +pkg_add elixir gmake git postgresql-server postgresql-contrib cmake ffmpeg ImageMagick ``` Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt. diff --git a/docs/installation/openbsd_fi.md b/docs/installation/openbsd_fi.md index b5b5056a9..2cb03e73e 100644 --- a/docs/installation/openbsd_fi.md +++ b/docs/installation/openbsd_fi.md @@ -16,7 +16,7 @@ Matrix-kanava #freenode_#pleroma:matrix.org ovat hyviä paikkoja löytää apua Asenna tarvittava ohjelmisto: -`# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3 cmake` +`# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3 cmake ffmpeg ImageMagick` Luo postgresql-tietokanta: diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index b7e3bb2ac..8fdf6b5b5 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -27,17 +27,19 @@ Other than things bundled in the OTP release Pleroma depends on: * PostgreSQL (also utilizes extensions in postgresql-contrib) * nginx (could be swapped with another reverse proxy but this guide covers only it) * certbot (for Let's Encrypt certificates, could be swapped with another ACME client, but this guide covers only it) +* ffmpeg (needed for media preview proxy) +* ImageMagick (needed for media preview proxy) === "Alpine" ``` echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories apk update - apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot + apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot ffmpeg imagemagick ``` === "Debian/Ubuntu" ``` - apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot + apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot ffmpeg imagemagick ``` ## Setup From 935ef21b0285975b08037827a33f32bfcbbff951 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 24 Sep 2020 16:47:34 -0500 Subject: [PATCH 23/49] Use the import --- lib/mix/tasks/pleroma/email.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/email.ex b/lib/mix/tasks/pleroma/email.ex index 1f543241a..0e4c87598 100644 --- a/lib/mix/tasks/pleroma/email.ex +++ b/lib/mix/tasks/pleroma/email.ex @@ -6,7 +6,7 @@ defmodule Mix.Tasks.Pleroma.Email do @moduledoc File.read!("docs/administration/CLI_tasks/email.md") def run(["test" | args]) do - Mix.Pleroma.start_pleroma() + start_pleroma() {options, [], []} = OptionParser.parse( From e33360fdb958708661a5bda415b9f06b5e1290d5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 24 Sep 2020 18:23:47 -0500 Subject: [PATCH 24/49] Cannot use batches with User.try_send_confirmation_email/1 --- lib/mix/tasks/pleroma/email.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/email.ex b/lib/mix/tasks/pleroma/email.ex index 0e4c87598..9e989ed46 100644 --- a/lib/mix/tasks/pleroma/email.ex +++ b/lib/mix/tasks/pleroma/email.ex @@ -31,7 +31,7 @@ defmodule Mix.Tasks.Pleroma.Email do confirmation_pending: true, invisible: false }) - |> Pleroma.Repo.chunk_stream(500, :batches) + |> Pleroma.Repo.chunk_stream(500) |> Stream.each(&Pleroma.User.try_send_confirmation_email(&1)) |> Stream.run() end From 3bf3db39f5932601798db8fd34523abc1b60dea7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 24 Sep 2020 18:24:44 -0500 Subject: [PATCH 25/49] Validate emails are sent to the appropriate unconfirmed actors --- test/tasks/email_test.exs | 69 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/test/tasks/email_test.exs b/test/tasks/email_test.exs index c3af7ef68..5393e3573 100644 --- a/test/tasks/email_test.exs +++ b/test/tasks/email_test.exs @@ -6,6 +6,8 @@ defmodule Mix.Tasks.Pleroma.EmailTest do alias Pleroma.Config alias Pleroma.Tests.ObanHelpers + import Pleroma.Factory + setup_all do Mix.shell(Mix.Shell.Process) @@ -17,6 +19,7 @@ defmodule Mix.Tasks.Pleroma.EmailTest do end setup do: clear_config([Pleroma.Emails.Mailer, :enabled], true) + setup do: clear_config([:instance, :account_activation_required], true) describe "pleroma.email test" do test "Sends test email with no given address" do @@ -50,5 +53,71 @@ defmodule Mix.Tasks.Pleroma.EmailTest do html_body: ~r/a test email was requested./i ) end + + test "Sends confirmation emails" do + local_user1 = + insert(:user, %{ + confirmation_pending: true, + confirmation_token: "mytoken", + deactivated: false, + email: "local1@pleroma.com", + local: true + }) + + local_user2 = + insert(:user, %{ + confirmation_pending: true, + confirmation_token: "mytoken", + deactivated: false, + email: "local2@pleroma.com", + local: true + }) + + :ok = Mix.Tasks.Pleroma.Email.run(["resend_confirmation_emails"]) + + ObanHelpers.perform_all() + + assert_email_sent(to: {local_user1.name, local_user1.email}) + assert_email_sent(to: {local_user2.name, local_user2.email}) + end + + test "Does not send confirmation email to inappropriate users" do + # confirmed user + insert(:user, %{ + confirmation_pending: false, + confirmation_token: "mytoken", + deactivated: false, + email: "confirmed@pleroma.com", + local: true + }) + + # remote user + insert(:user, %{ + deactivated: false, + email: "remote@not-pleroma.com", + local: false + }) + + # deactivated user = + insert(:user, %{ + deactivated: true, + email: "deactivated@pleroma.com", + local: false + }) + + # invisible user + insert(:user, %{ + deactivated: false, + email: "invisible@pleroma.com", + local: true, + invisible: true + }) + + :ok = Mix.Tasks.Pleroma.Email.run(["resend_confirmation_emails"]) + + ObanHelpers.perform_all() + + refute_email_sent() + end end end From 81faf540b30d0ecf75779e7a5c16f42b7218cec8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 24 Sep 2020 18:35:20 -0500 Subject: [PATCH 26/49] Add some user feedback --- lib/mix/tasks/pleroma/email.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/mix/tasks/pleroma/email.ex b/lib/mix/tasks/pleroma/email.ex index 9e989ed46..9972cb988 100644 --- a/lib/mix/tasks/pleroma/email.ex +++ b/lib/mix/tasks/pleroma/email.ex @@ -25,6 +25,8 @@ defmodule Mix.Tasks.Pleroma.Email do def run(["resend_confirmation_emails"]) do start_pleroma() + shell_info("Sending emails to all unconfirmed users") + Pleroma.User.Query.build(%{ local: true, deactivated: false, From a8c17ea25a79491328345f5834397eb6821a77f1 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 25 Sep 2020 08:46:14 +0200 Subject: [PATCH 27/49] User Search: Also find user by uri --- lib/pleroma/user/search.ex | 35 ++++++++++++++++++++++++++++------- test/user_search_test.exs | 15 +++++++++++++++ 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index 408295e0c..d747bfa52 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -5,6 +5,7 @@ defmodule Pleroma.User.Search do alias Pleroma.Pagination alias Pleroma.User + alias Pleroma.EctoType.ActivityPub.ObjectValidators.Uri, as: UriType import Ecto.Query @limit 20 @@ -21,15 +22,12 @@ defmodule Pleroma.User.Search do # If this returns anything, it should bounce to the top maybe_resolved = maybe_resolve(resolve, for_user, query_string) - maybe_ap_id_match = User.get_cached_by_ap_id(query_string) top_user_ids = - case {maybe_resolved, maybe_ap_id_match} do - {{:ok, %User{} = user}, %User{} = other_user} -> [user.id, other_user.id] - {{:ok, %User{} = user}, _} -> [user.id] - {_, %User{} = user} -> [user.id] - _ -> [] - end + [] + |> maybe_add_resolved(maybe_resolved) + |> maybe_add_ap_id_match(query_string) + |> maybe_add_uri_match(query_string) results = query_string @@ -39,6 +37,29 @@ defmodule Pleroma.User.Search do results end + defp maybe_add_resolved(list, {:ok, %User{} = user}) do + [user.id | list] + end + + defp maybe_add_resolved(list, _), do: list + + defp maybe_add_ap_id_match(list, query) do + if user = User.get_cached_by_ap_id(query) do + [user.id | list] + else + list + end + end + + defp maybe_add_uri_match(list, query) do + with {:ok, query} <- UriType.cast(query), + %User{} = user <- Pleroma.Repo.get_by(User, uri: query) do + [user.id | list] + else + _ -> list + end + end + defp format_query(query_string) do # Strip the beginning @ off if there is a query query_string = String.trim_leading(query_string, "@") diff --git a/test/user_search_test.exs b/test/user_search_test.exs index 68fda1c53..cc14e9741 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -36,6 +36,21 @@ defmodule Pleroma.UserSearchTest do assert first_user.id == user.id end + test "returns a user with matching uri as the first result" do + user = + insert(:user, %{ + nickname: "no_relation", + ap_id: "https://lain.com/users/lain", + uri: "https://lain.com/@lain" + }) + + _user = insert(:user, %{nickname: "com_user"}) + + [first_user, _second_user] = User.search("https://lain.com/@lain") + + assert first_user.id == user.id + end + test "excludes invisible users from results" do user = insert(:user, %{nickname: "john t1000"}) insert(:user, %{invisible: true, nickname: "john t800"}) From 1e0f3f8514a42b088ed68ece5f9e339ad829e242 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 25 Sep 2020 08:56:58 +0200 Subject: [PATCH 28/49] User search: Make uri matches case insensitive. --- .../20200925065249_make_user_ids_ci.exs | 19 +++++++++++++++++++ test/user_search_test.exs | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 priv/repo/migrations/20200925065249_make_user_ids_ci.exs diff --git a/priv/repo/migrations/20200925065249_make_user_ids_ci.exs b/priv/repo/migrations/20200925065249_make_user_ids_ci.exs new file mode 100644 index 000000000..b7305f137 --- /dev/null +++ b/priv/repo/migrations/20200925065249_make_user_ids_ci.exs @@ -0,0 +1,19 @@ +defmodule Pleroma.Repo.Migrations.MakeUserIdsCI do + use Ecto.Migration + + def up do + alter table(:users) do + modify(:uri, :citext) + end + + create(unique_index(:users, :uri)) + end + + def don do + drop(unique_index(:users, :uri)) + + alter table(:users) do + modify(:uri, :text) + end + end +end diff --git a/test/user_search_test.exs b/test/user_search_test.exs index cc14e9741..b99a77b57 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -41,7 +41,7 @@ defmodule Pleroma.UserSearchTest do insert(:user, %{ nickname: "no_relation", ap_id: "https://lain.com/users/lain", - uri: "https://lain.com/@lain" + uri: "https://lain.com/@Lain" }) _user = insert(:user, %{nickname: "com_user"}) From 05b5241314182c5aab2907e27d4c5f46d7617f56 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 25 Sep 2020 09:12:48 +0200 Subject: [PATCH 29/49] Linter fixes --- lib/pleroma/user/search.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index d747bfa52..03f2c552f 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -3,9 +3,10 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.User.Search do + alias Pleroma.EctoType.ActivityPub.ObjectValidators.Uri, as: UriType alias Pleroma.Pagination alias Pleroma.User - alias Pleroma.EctoType.ActivityPub.ObjectValidators.Uri, as: UriType + import Ecto.Query @limit 20 From 8b84ca4901c378d734cd87ae3e4bf72c508a84bf Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 25 Sep 2020 10:37:59 -0500 Subject: [PATCH 30/49] Simplify the value comparison --- lib/mix/tasks/pleroma/user.ex | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index d50205600..e06262804 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -455,11 +455,7 @@ defmodule Mix.Tasks.Pleroma.User do end defp set_confirmed(user, value) do - {:ok, user} = - case value do - true -> User.need_confirmation(user, false) - false -> User.need_confirmation(user, true) - end + {:ok, user} = User.need_confirmation(user, !value) shell_info("Confirmation pending status of #{user.nickname}: #{user.confirmation_pending}") user From 4a30598b9eac9d231546078191ff91b933d63de3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 25 Sep 2020 12:20:52 -0500 Subject: [PATCH 31/49] Config settings leak and break configdb migration tests when async --- test/config/deprecation_warnings_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/config/deprecation_warnings_test.exs b/test/config/deprecation_warnings_test.exs index 28355d7eb..f81a7b580 100644 --- a/test/config/deprecation_warnings_test.exs +++ b/test/config/deprecation_warnings_test.exs @@ -1,5 +1,5 @@ defmodule Pleroma.Config.DeprecationWarningsTest do - use ExUnit.Case, async: true + use ExUnit.Case use Pleroma.Tests.Helpers import ExUnit.CaptureLog From 93b674b66da31964e838c8632ce8cdd7e722516a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 25 Sep 2020 12:46:49 -0500 Subject: [PATCH 32/49] Fix test failures for NoOpPolicy describe/0 --- test/web/activity_pub/mrf/mrf_test.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/web/activity_pub/mrf/mrf_test.exs b/test/web/activity_pub/mrf/mrf_test.exs index a63b25423..e82c8afa6 100644 --- a/test/web/activity_pub/mrf/mrf_test.exs +++ b/test/web/activity_pub/mrf/mrf_test.exs @@ -61,6 +61,8 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do describe "describe/0" do test "it works as expected with noop policy" do + clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.NoOpPolicy]) + expected = %{ mrf_policies: ["NoOpPolicy"], exclusions: false From 4e4f77108207157a49a627edb03951e2f15b62f1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 26 Sep 2020 19:32:16 +0300 Subject: [PATCH 33/49] Adjusted MediaProxyControllerTest to gracefully fail on missing dependencies. Installation docs update. Added ffmpeg/imagemagick checks to launch checks (if media preview proxy is enabled). Added documentation on installing optional media / graphics packages (imagemagick, ffmpeg, exiftool). --- docs/installation/alpine_linux_en.md | 18 +++++------ docs/installation/arch_linux_en.md | 13 ++++++-- docs/installation/debian_based_en.md | 13 ++++++-- docs/installation/debian_based_jp.md | 12 +++++-- docs/installation/freebsd_en.md | 8 ++++- docs/installation/gentoo_en.md | 13 ++++++-- docs/installation/netbsd_en.md | 4 +++ docs/installation/openbsd_en.md | 13 ++++++++ docs/installation/openbsd_fi.md | 11 +++++++ .../optional/media_graphics_packages.md | 32 +++++++++++++++++++ docs/installation/otp_en.md | 25 ++++++++++++--- lib/pleroma/application.ex | 18 ++++++++++- lib/pleroma/helpers/media_helper.ex | 12 +++++++ .../media_proxy_controller_test.exs | 15 +++++++++ 14 files changed, 180 insertions(+), 27 deletions(-) create mode 100644 docs/installation/optional/media_graphics_packages.md diff --git a/docs/installation/alpine_linux_en.md b/docs/installation/alpine_linux_en.md index 0b9f5cdae..d89c7f46f 100644 --- a/docs/installation/alpine_linux_en.md +++ b/docs/installation/alpine_linux_en.md @@ -13,8 +13,6 @@ It assumes that you have administrative rights, either as root or a user with [s * `erlang-parsetools` * `erlang-xmerl` * `git` -* `ffmpeg` -* `ImageMagick` * Development Tools * `cmake` @@ -22,6 +20,9 @@ It assumes that you have administrative rights, either as root or a user with [s * `nginx` (preferred, example configs for other reverse proxies can be found in the repo) * `certbot` (or any other ACME client for Let’s Encrypt certificates) +* `ImageMagick` +* `ffmpeg` +* `exiftool` ### Prepare the system @@ -31,7 +32,6 @@ It assumes that you have administrative rights, either as root or a user with [s awk 'NR==2' /etc/apk/repositories | sed 's/main/community/' | tee -a /etc/apk/repositories ``` - * Then update the system, if not already done: ```shell @@ -59,12 +59,6 @@ sudo apk add erlang erlang-runtime-tools erlang-xmerl elixir sudo apk add erlang-eldap ``` -### Install ffmpeg and ImageMagick - -```shell -sudo apk add ffmpeg imagemagick -``` - ### Install PostgreSQL * Install Postgresql server: @@ -85,6 +79,12 @@ sudo /etc/init.d/postgresql start sudo rc-update add postgresql ``` +### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)) + +```shell +sudo apk add ffmpeg imagemagick exiftool +``` + ### Install PleromaBE * Add a new system user for the Pleroma service: diff --git a/docs/installation/arch_linux_en.md b/docs/installation/arch_linux_en.md index 38bb82432..724b4660a 100644 --- a/docs/installation/arch_linux_en.md +++ b/docs/installation/arch_linux_en.md @@ -10,13 +10,14 @@ This guide will assume that you have administrative rights, either as root or a * `git` * `base-devel` * `cmake` -* `ffmpeg` -* `ImageMagick` #### Optional packages used in this guide * `nginx` (preferred, example configs for other reverse proxies can be found in the repo) * `certbot` (or any other ACME client for Let’s Encrypt certificates) +* `ImageMagick` +* `ffmpeg` +* `exiftool` ### Prepare the system @@ -29,7 +30,7 @@ sudo pacman -Syu * Install some of the above mentioned programs: ```shell -sudo pacman -S git base-devel elixir cmake ffmpeg imagemagick +sudo pacman -S git base-devel elixir cmake ``` ### Install PostgreSQL @@ -54,6 +55,12 @@ sudo -iu postgres initdb -D /var/lib/postgres/data sudo systemctl enable --now postgresql.service ``` +### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)) + +```shell +sudo pacman -S ffmpeg imagemagick perl-image-exiftool +``` + ### Install PleromaBE * Add a new system user for the Pleroma service: diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md index 30ee64c01..eac499a29 100644 --- a/docs/installation/debian_based_en.md +++ b/docs/installation/debian_based_en.md @@ -13,13 +13,14 @@ This guide will assume you are on Debian Stretch. This guide should also work wi * `git` * `build-essential` * `cmake` -* `ffmpeg` -* `ImageMagick` #### Optional packages used in this guide * `nginx` (preferred, example configs for other reverse proxies can be found in the repo) * `certbot` (or any other ACME client for Let’s Encrypt certificates) +* `ImageMagick` +* `ffmpeg` +* `exiftool` ### Prepare the system @@ -33,7 +34,7 @@ sudo apt full-upgrade * Install some of the above mentioned programs: ```shell -sudo apt install git build-essential postgresql postgresql-contrib cmake ffmpeg imagemagick +sudo apt install git build-essential postgresql postgresql-contrib cmake ``` ### Install Elixir and Erlang @@ -52,6 +53,12 @@ sudo apt update sudo apt install elixir erlang-dev erlang-nox ``` +### Optional packages: [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md) + +```shell +sudo apt install imagemagick ffmpeg libimage-exiftool-perl +``` + ### Install PleromaBE * Add a new system user for the Pleroma service: diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index 219f40525..764afbe1a 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -17,13 +17,14 @@ - `git` - `build-essential` - `cmake` -- `ffmpeg` -- `ImageMagick` #### このガイドで利用している追加パッケージ - `nginx` (おすすめです。他のリバースプロキシを使う場合は、参考となる設定をこのリポジトリから探してください) - `certbot` (または何らかのLet's Encrypt向けACMEクライアント) +- `ImageMagick` +- `ffmpeg` +- `exiftool` ### システムを準備する @@ -38,7 +39,6 @@ sudo apt full-upgrade sudo apt install git build-essential postgresql postgresql-contrib cmake ffmpeg imagemagick ``` - ### ElixirとErlangをインストールします * Erlangのリポジトリをダウンロードおよびインストールします。 @@ -53,6 +53,12 @@ sudo apt update sudo apt install elixir erlang-dev erlang-nox ``` +### オプションパッケージ: [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md) + +```shell +sudo apt install imagemagick ffmpeg libimage-exiftool-perl +``` + ### Pleroma BE (バックエンド) をインストールします * Pleroma用に新しいユーザーを作ります。 diff --git a/docs/installation/freebsd_en.md b/docs/installation/freebsd_en.md index 01b410a46..fdcb06c53 100644 --- a/docs/installation/freebsd_en.md +++ b/docs/installation/freebsd_en.md @@ -7,7 +7,7 @@ This document was written for FreeBSD 12.1, but should be work on future release This assumes the target system has `pkg(8)`. ``` -# pkg install elixir postgresql12-server postgresql12-client postgresql12-contrib git-lite sudo nginx gmake acme.sh cmake ffmpeg imagemagick +# pkg install elixir postgresql12-server postgresql12-client postgresql12-contrib git-lite sudo nginx gmake acme.sh cmake ``` Copy the rc.d scripts to the right directory: @@ -26,6 +26,12 @@ Setup the required services to automatically start at boot, using `sysrc(8)`. # service postgresql start ``` +### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)) + +```shell +# pkg install imagemagick ffmpeg p5-Image-ExifTool +``` + ## Configuring Pleroma Create a user for Pleroma: diff --git a/docs/installation/gentoo_en.md b/docs/installation/gentoo_en.md index 9a8b54ece..638fc4e47 100644 --- a/docs/installation/gentoo_en.md +++ b/docs/installation/gentoo_en.md @@ -29,14 +29,15 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i * `dev-lang/elixir` * `dev-vcs/git` * `dev-util/cmake` -* `media-video/ffmpeg` -* `media-gfx/imagemagick` #### Optional ebuilds used in this guide * `www-servers/nginx` (preferred, example configs for other reverse proxies can be found in the repo) * `app-crypt/certbot` (or any other ACME client for Let’s Encrypt certificates) * `app-crypt/certbot-nginx` (nginx certbot plugin that allows use of the all-powerful `--nginx` flag on certbot) +* `media-gfx/imagemagick` +* `media-video/ffmpeg` +* `media-libs/exiftool` ### Prepare the system @@ -49,7 +50,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i * Emerge all required the required and suggested software in one go: ```shell - # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake media-video/ffmpeg media-gfx/imagemagick + # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake ``` If you would not like to install the optional packages, remove them from this line. @@ -89,6 +90,12 @@ If you do not plan to make any modifications to your Pleroma instance, cloning d Not only does this make it much easier to deploy changes you make, as you can commit and pull from upstream and all that good stuff from the comfort of your local machine then simply `git pull` on your instance server when you're ready to deploy, it also ensures you are compliant with the Affero General Public Licence that Pleroma is licenced under, which stipulates that all network services provided with modified AGPL code must publish their changes on a publicly available internet service and for free. It also makes it much easier to ask for help from and provide help to your fellow Pleroma admins if your public repo always reflects what you are running because it is part of your deployment procedure. +### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)) + +```shell +# emerge --ask media-video/ffmpeg media-gfx/imagemagick media-libs/exiftool +``` + ### Install PleromaBE * Add a new system user for the Pleroma service and set up default directories: diff --git a/docs/installation/netbsd_en.md b/docs/installation/netbsd_en.md index 8cf2e4682..d5fa04fdf 100644 --- a/docs/installation/netbsd_en.md +++ b/docs/installation/netbsd_en.md @@ -44,6 +44,10 @@ pgsql=YES First, run `# /etc/rc.d/pgsql start`. Then, `$ sudo -Hu pgsql -g pgsql createdb`. +### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)) + +`# pkgin install ImageMagick ffmpeg4 p5-Image-ExifTool` + ## Configuring Pleroma Create a user for Pleroma: diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 277e5ec43..8092ac379 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -25,6 +25,19 @@ pkg_add elixir gmake git postgresql-server postgresql-contrib cmake ffmpeg Image Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt. +#### Optional software + +Per [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md): + * ImageMagick + * ffmpeg + * exiftool + +To install the above: + +``` +pkg_add ImageMagick ffmpeg p5-Image-ExifTool +``` + #### Creating the pleroma user Pleroma will be run by a dedicated user, \_pleroma. Before creating it, insert the following lines in login.conf: ``` diff --git a/docs/installation/openbsd_fi.md b/docs/installation/openbsd_fi.md index 2cb03e73e..01cf34ab4 100644 --- a/docs/installation/openbsd_fi.md +++ b/docs/installation/openbsd_fi.md @@ -18,6 +18,17 @@ Asenna tarvittava ohjelmisto: `# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3 cmake ffmpeg ImageMagick` +#### Optional software + +[`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md): + * ImageMagick + * ffmpeg + * exiftool + +Asenna tarvittava ohjelmisto: + +`# pkg_add ImageMagick ffmpeg p5-Image-ExifTool` + Luo postgresql-tietokanta: `# su - _postgresql` diff --git a/docs/installation/optional/media_graphics_packages.md b/docs/installation/optional/media_graphics_packages.md new file mode 100644 index 000000000..cb3d71188 --- /dev/null +++ b/docs/installation/optional/media_graphics_packages.md @@ -0,0 +1,32 @@ +# Optional software packages needed for specific functionality + +For specific Pleroma functionality (which is disabled by default) some or all of the below packages are required: + * `ImageMagic` + * `ffmpeg` + * `exiftool` + +Please refer to documentation in `docs/installation` on how to install them on specific OS. + +Note: the packages are not required with the current default settings of Pleroma. + +## `ImageMagick` + +`ImageMagick` is a set of tools to create, edit, compose, or convert bitmap images. + +It is required for the following Pleroma features: + * `Pleroma.Upload.Filters.Mogrify`, `Pleroma.Upload.Filters.Mogrifun` upload filters (related config: `Plaroma.Upload/filters` in `config/config.exs`) + * Media preview proxy for still images (related config: `media_preview_proxy/enabled` in `config/config.exs`) + +## `ffmpeg` + +`ffmpeg` is software to record, convert and stream audio and video. + +It is required for the following Pleroma features: + * Media preview proxy for videos (related config: `media_preview_proxy/enabled` in `config/config.exs`) + +## `exiftool` + +`exiftool` is media files metadata reader/writer. + +It is required for the following Pleroma features: + * `Pleroma.Upload.Filters.Exiftool` upload filter (related config: `Plaroma.Upload/filters` in `config/config.exs`) diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 8fdf6b5b5..f6b05c887 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -27,19 +27,36 @@ Other than things bundled in the OTP release Pleroma depends on: * PostgreSQL (also utilizes extensions in postgresql-contrib) * nginx (could be swapped with another reverse proxy but this guide covers only it) * certbot (for Let's Encrypt certificates, could be swapped with another ACME client, but this guide covers only it) -* ffmpeg (needed for media preview proxy) -* ImageMagick (needed for media preview proxy) === "Alpine" ``` echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories apk update - apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot ffmpeg imagemagick + apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot ``` === "Debian/Ubuntu" ``` - apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot ffmpeg imagemagick + apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot + ``` + +### Installing optional packages + +Per [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md): + * ImageMagick + * ffmpeg + * exiftool + +=== "Alpine" + ``` + echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories + apk update + apk add imagemagick ffmpeg exiftool + ``` + +=== "Debian/Ubuntu" + ``` + apt install imagemagick ffmpeg libimage-exiftool-perl ``` ## Setup diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 00ec79a2a..d7d8e423e 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -268,7 +268,8 @@ defmodule Pleroma.Application do with true <- filter in filters, false <- Pleroma.Utils.command_available?(command_required) do Logger.error( - "#{filter} is specified in list of Pleroma.Upload filters, but the #{command_required} command is not found" + "#{filter} is specified in list of Pleroma.Upload filters, but the " <> + "#{command_required} command is not found" ) end end @@ -276,5 +277,20 @@ defmodule Pleroma.Application do check_filter.(Pleroma.Upload.Filters.Exiftool, "exiftool") check_filter.(Pleroma.Upload.Filters.Mogrify, "mogrify") check_filter.(Pleroma.Upload.Filters.Mogrifun, "mogrify") + + with true <- Config.get([:media_preview_proxy, :enabled]), + missing_graphics_tools = Pleroma.Helpers.MediaHelper.missing_dependencies(), + [] <- missing_graphics_tools do + :noop + else + false -> + :noop + + missing_graphics_tools -> + Logger.error( + "The following dependencies required by Media preview proxy " <> + "(which is currently enabled) are not installed: #{inspect(missing_graphics_tools)}" + ) + end end end diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex index b6f35a24b..6b799173e 100644 --- a/lib/pleroma/helpers/media_helper.ex +++ b/lib/pleroma/helpers/media_helper.ex @@ -9,6 +9,18 @@ defmodule Pleroma.Helpers.MediaHelper do alias Pleroma.HTTP + require Logger + + def missing_dependencies do + Enum.reduce([imagemagick: "convert", ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc -> + if Pleroma.Utils.command_available?(executable) do + acc + else + [sym | acc] + end + end) + end + def image_resize(url, options) do with executable when is_binary(executable) <- System.find_executable("convert"), {:ok, args} <- prepare_image_resize_args(options), diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs index 33e6873f7..e9b584822 100644 --- a/test/web/media_proxy/media_proxy_controller_test.exs +++ b/test/web/media_proxy/media_proxy_controller_test.exs @@ -81,6 +81,15 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do end describe "Media Preview Proxy" do + def assert_dependencies_installed do + missing_dependencies = Pleroma.Helpers.MediaHelper.missing_dependencies() + + assert missing_dependencies == [], + "Error: missing dependencies (please refer to `docs/installation`): #{ + inspect(missing_dependencies) + }" + end + setup do clear_config([:media_proxy, :enabled], true) clear_config([:media_preview_proxy, :enabled], true) @@ -184,6 +193,8 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do url: url, media_proxy_url: media_proxy_url } do + assert_dependencies_installed() + # Setting a high :min_content_length to ensure this scenario is not affected by its logic clear_config([:media_preview_proxy, :min_content_length], 1_000_000_000) @@ -270,6 +281,8 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do url: url, media_proxy_url: media_proxy_url } do + assert_dependencies_installed() + Tesla.Mock.mock(fn %{method: "head", url: ^media_proxy_url} -> %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/png"}]} @@ -290,6 +303,8 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do url: url, media_proxy_url: media_proxy_url } do + assert_dependencies_installed() + Tesla.Mock.mock(fn %{method: "head", url: ^media_proxy_url} -> %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]} From de993b856bc2145e7c4aaa47767c7edc826798c7 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 28 Sep 2020 09:16:42 +0300 Subject: [PATCH 34/49] added `force` option to the unfollow operation --- docs/API/admin_api.md | 6 +- lib/mix/tasks/pleroma/relay.ex | 13 +++- lib/pleroma/user.ex | 4 +- lib/pleroma/web/activity_pub/relay.ex | 20 ++++- .../admin_api/controllers/relay_controller.ex | 16 +--- .../operations/admin/relay_operation.ex | 12 ++- test/tasks/relay_test.exs | 74 +++++++++++++++++++ test/web/activity_pub/relay_test.exs | 40 ++++++++++ 8 files changed, 160 insertions(+), 25 deletions(-) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 7992db58f..ddcab1a2c 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -349,9 +349,9 @@ Response: ### Unfollow a Relay -Params: - -* `relay_url` +- Params: + - `relay_url` + - *optional* `force`: forcefully unfollow a relay even when the relay is not available. (default is `false`) Response: diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex index a6d8d6c1c..bb808ca47 100644 --- a/lib/mix/tasks/pleroma/relay.ex +++ b/lib/mix/tasks/pleroma/relay.ex @@ -21,10 +21,19 @@ defmodule Mix.Tasks.Pleroma.Relay do end end - def run(["unfollow", target]) do + def run(["unfollow", target | rest]) do start_pleroma() - with {:ok, _activity} <- Relay.unfollow(target) do + {options, [], []} = + OptionParser.parse( + rest, + strict: [force: :boolean], + aliases: [f: :force] + ) + + force = Keyword.get(options, :force, false) + + with {:ok, _activity} <- Relay.unfollow(target, %{force: force}) do # put this task to sleep to allow the genserver to push out the messages :timer.sleep(500) else diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 03be61ccf..71ace1c34 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -915,9 +915,7 @@ defmodule Pleroma.User do FollowingRelationship.unfollow(follower, followed) {:ok, followed} = update_follower_count(followed) - {:ok, follower} = - follower - |> update_following_count() + {:ok, follower} = update_following_count(follower) {:ok, follower, followed} diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex index b65710a94..6606e1780 100644 --- a/lib/pleroma/web/activity_pub/relay.ex +++ b/lib/pleroma/web/activity_pub/relay.ex @@ -30,12 +30,16 @@ defmodule Pleroma.Web.ActivityPub.Relay do end end - @spec unfollow(String.t()) :: {:ok, Activity.t()} | {:error, any()} - def unfollow(target_instance) do + @spec unfollow(String.t(), map()) :: {:ok, Activity.t()} | {:error, any()} + def unfollow(target_instance, opts \\ %{}) do with %User{} = local_user <- get_actor(), - {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance), + {:ok, target_user} <- fetch_target_user(target_instance, opts), {:ok, activity} <- ActivityPub.unfollow(local_user, target_user) do - User.unfollow(local_user, target_user) + case target_user.id do + nil -> User.update_following_count(local_user) + _ -> User.unfollow(local_user, target_user) + end + Logger.info("relay: unfollowed instance: #{target_instance}: id=#{activity.data["id"]}") {:ok, activity} else @@ -43,6 +47,14 @@ defmodule Pleroma.Web.ActivityPub.Relay do end end + defp fetch_target_user(ap_id, opts) do + case {opts[:force], User.get_or_fetch_by_ap_id(ap_id)} do + {_, {:ok, %User{} = user}} -> {:ok, user} + {true, _} -> {:ok, %User{ap_id: ap_id}} + {_, error} -> error + end + end + @spec publish(any()) :: {:ok, Activity.t()} | {:error, any()} def publish(%Activity{data: %{"type" => "Create"}} = activity) do with %User{} = user <- get_actor(), diff --git a/lib/pleroma/web/admin_api/controllers/relay_controller.ex b/lib/pleroma/web/admin_api/controllers/relay_controller.ex index 95d06dde7..6c19f09f7 100644 --- a/lib/pleroma/web/admin_api/controllers/relay_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/relay_controller.ex @@ -33,11 +33,7 @@ defmodule Pleroma.Web.AdminAPI.RelayController do def follow(%{assigns: %{user: admin}, body_params: %{relay_url: target}} = conn, _) do with {:ok, _message} <- Relay.follow(target) do - ModerationLog.insert_log(%{ - action: "relay_follow", - actor: admin, - target: target - }) + ModerationLog.insert_log(%{action: "relay_follow", actor: admin, target: target}) json(conn, %{actor: target, followed_back: target in Relay.following()}) else @@ -48,13 +44,9 @@ defmodule Pleroma.Web.AdminAPI.RelayController do end end - def unfollow(%{assigns: %{user: admin}, body_params: %{relay_url: target}} = conn, _) do - with {:ok, _message} <- Relay.unfollow(target) do - ModerationLog.insert_log(%{ - action: "relay_unfollow", - actor: admin, - target: target - }) + def unfollow(%{assigns: %{user: admin}, body_params: %{relay_url: target} = params} = conn, _) do + with {:ok, _message} <- Relay.unfollow(target, %{force: params[:force]}) do + ModerationLog.insert_log(%{action: "relay_unfollow", actor: admin, target: target}) json(conn, target) else diff --git a/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex b/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex index e06b2d164..f754bb9f5 100644 --- a/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex @@ -56,7 +56,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do operationId: "AdminAPI.RelayController.unfollow", security: [%{"oAuth" => ["write:follows"]}], parameters: admin_api_params(), - requestBody: request_body("Parameters", relay_url()), + requestBody: request_body("Parameters", relay_unfollow()), responses: %{ 200 => Operation.response("Status", "application/json", %Schema{ @@ -91,4 +91,14 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do } } end + + defp relay_unfollow do + %Schema{ + type: :object, + properties: %{ + relay_url: %Schema{type: :string, format: :uri}, + force: %Schema{type: :boolean, default: false} + } + } + end end diff --git a/test/tasks/relay_test.exs b/test/tasks/relay_test.exs index e5225b64c..cf48e7dda 100644 --- a/test/tasks/relay_test.exs +++ b/test/tasks/relay_test.exs @@ -81,6 +81,80 @@ defmodule Mix.Tasks.Pleroma.RelayTest do assert undo_activity.data["object"]["id"] == cancelled_activity.data["id"] refute "#{target_instance}/followers" in User.following(local_user) end + + test "unfollow when relay is dead" do + user = insert(:user) + target_instance = user.ap_id + + Mix.Tasks.Pleroma.Relay.run(["follow", target_instance]) + + %User{ap_id: follower_id} = local_user = Relay.get_actor() + target_user = User.get_cached_by_ap_id(target_instance) + follow_activity = Utils.fetch_latest_follow(local_user, target_user) + User.follow(local_user, target_user) + + assert "#{target_instance}/followers" in User.following(local_user) + + Tesla.Mock.mock(fn %{method: :get, url: ^target_instance} -> + %Tesla.Env{status: 404} + end) + + Pleroma.Repo.delete(user) + Cachex.clear(:user_cache) + + Mix.Tasks.Pleroma.Relay.run(["unfollow", target_instance]) + + cancelled_activity = Activity.get_by_ap_id(follow_activity.data["id"]) + assert cancelled_activity.data["state"] == "accept" + + assert [] == + ActivityPub.fetch_activities( + [], + %{ + type: "Undo", + actor_id: follower_id, + skip_preload: true, + invisible_actors: true + } + ) + end + + test "force unfollow when relay is dead" do + user = insert(:user) + target_instance = user.ap_id + + Mix.Tasks.Pleroma.Relay.run(["follow", target_instance]) + + %User{ap_id: follower_id} = local_user = Relay.get_actor() + target_user = User.get_cached_by_ap_id(target_instance) + follow_activity = Utils.fetch_latest_follow(local_user, target_user) + User.follow(local_user, target_user) + + assert "#{target_instance}/followers" in User.following(local_user) + + Tesla.Mock.mock(fn %{method: :get, url: ^target_instance} -> + %Tesla.Env{status: 404} + end) + + Pleroma.Repo.delete(user) + Cachex.clear(:user_cache) + + Mix.Tasks.Pleroma.Relay.run(["unfollow", target_instance, "--force"]) + + cancelled_activity = Activity.get_by_ap_id(follow_activity.data["id"]) + assert cancelled_activity.data["state"] == "cancelled" + + [undo_activity] = + ActivityPub.fetch_activities( + [], + %{type: "Undo", actor_id: follower_id, skip_preload: true, invisible_actors: true} + ) + + assert undo_activity.data["type"] == "Undo" + assert undo_activity.data["actor"] == local_user.ap_id + assert undo_activity.data["object"]["id"] == cancelled_activity.data["id"] + refute "#{target_instance}/followers" in User.following(local_user) + end end describe "mix pleroma.relay list" do diff --git a/test/web/activity_pub/relay_test.exs b/test/web/activity_pub/relay_test.exs index 9d657ac4f..3284980f7 100644 --- a/test/web/activity_pub/relay_test.exs +++ b/test/web/activity_pub/relay_test.exs @@ -63,6 +63,46 @@ defmodule Pleroma.Web.ActivityPub.RelayTest do assert activity.data["to"] == [user.ap_id] refute "#{user.ap_id}/followers" in User.following(service_actor) end + + test "force unfollow when target service is dead" do + user = insert(:user) + user_ap_id = user.ap_id + user_id = user.id + + Tesla.Mock.mock(fn %{method: :get, url: ^user_ap_id} -> + %Tesla.Env{status: 404} + end) + + service_actor = Relay.get_actor() + CommonAPI.follow(service_actor, user) + assert "#{user.ap_id}/followers" in User.following(service_actor) + + assert Pleroma.Repo.get_by( + Pleroma.FollowingRelationship, + follower_id: service_actor.id, + following_id: user_id + ) + + Pleroma.Repo.delete(user) + Cachex.clear(:user_cache) + + assert {:ok, %Activity{} = activity} = Relay.unfollow(user_ap_id, %{force: true}) + + assert refresh_record(service_actor).following_count == 0 + + refute Pleroma.Repo.get_by( + Pleroma.FollowingRelationship, + follower_id: service_actor.id, + following_id: user_id + ) + + assert activity.actor == "#{Pleroma.Web.Endpoint.url()}/relay" + assert user.ap_id in activity.recipients + assert activity.data["type"] == "Undo" + assert activity.data["actor"] == service_actor.ap_id + assert activity.data["to"] == [user_ap_id] + refute "#{user.ap_id}/followers" in User.following(service_actor) + end end describe "publish/1" do From ec2f70cd23ff77a1580650dd629a84f61df1d486 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 28 Sep 2020 14:14:32 +0200 Subject: [PATCH 35/49] CI: Add ffmpeg --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dc953a929..121e4abfe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,7 +59,7 @@ unit-testing: alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] script: - - apt-get update && apt-get install -y libimage-exiftool-perl + - apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg - mix deps.get - mix ecto.create - mix ecto.migrate @@ -93,7 +93,7 @@ unit-testing-rum: <<: *global_variables RUM_ENABLED: "true" script: - - apt-get update && apt-get install -y libimage-exiftool-perl + - apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg - mix deps.get - mix ecto.create - mix ecto.migrate From 7bc561127da6489862d3b7ea49ebc853c0267729 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 28 Sep 2020 18:15:31 +0300 Subject: [PATCH 36/49] Revert citext user URI migration URI paths are not actually case-insesitive, which caused migration issues on a number of databases. Closes #2188 --- .../20200925065249_make_user_ids_ci.exs | 18 ++++-------------- .../20200928145912_revert_citext_change.exs | 11 +++++++++++ test/user_search_test.exs | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 priv/repo/migrations/20200928145912_revert_citext_change.exs diff --git a/priv/repo/migrations/20200925065249_make_user_ids_ci.exs b/priv/repo/migrations/20200925065249_make_user_ids_ci.exs index b7305f137..8ea0f2cf1 100644 --- a/priv/repo/migrations/20200925065249_make_user_ids_ci.exs +++ b/priv/repo/migrations/20200925065249_make_user_ids_ci.exs @@ -1,19 +1,9 @@ defmodule Pleroma.Repo.Migrations.MakeUserIdsCI do use Ecto.Migration - def up do - alter table(:users) do - modify(:uri, :citext) - end - - create(unique_index(:users, :uri)) - end - - def don do - drop(unique_index(:users, :uri)) - - alter table(:users) do - modify(:uri, :text) - end + def change do + # Migration retired, see + # https://git.pleroma.social/pleroma/pleroma/-/issues/2188 + :noop end end diff --git a/priv/repo/migrations/20200928145912_revert_citext_change.exs b/priv/repo/migrations/20200928145912_revert_citext_change.exs new file mode 100644 index 000000000..ab232f607 --- /dev/null +++ b/priv/repo/migrations/20200928145912_revert_citext_change.exs @@ -0,0 +1,11 @@ +defmodule Pleroma.Repo.Migrations.RevertCitextChange do + use Ecto.Migration + + def change do + alter table(:users) do + modify(:uri, :text) + end + + create_if_not_exists(unique_index(:users, :uri)) + end +end diff --git a/test/user_search_test.exs b/test/user_search_test.exs index b99a77b57..cc14e9741 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -41,7 +41,7 @@ defmodule Pleroma.UserSearchTest do insert(:user, %{ nickname: "no_relation", ap_id: "https://lain.com/users/lain", - uri: "https://lain.com/@Lain" + uri: "https://lain.com/@lain" }) _user = insert(:user, %{nickname: "com_user"}) From 7d5c3883acafc2c84f65f38dd639d4999f14215a Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 29 Sep 2020 16:28:06 +0300 Subject: [PATCH 37/49] [#3031] Refactoring: moved system commands checks to ApplicationRequirements. --- lib/pleroma/application.ex | 34 ------------ lib/pleroma/application_requirements.ex | 72 +++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 39 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index d7d8e423e..e73d89350 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -56,7 +56,6 @@ defmodule Pleroma.Application do Pleroma.ApplicationRequirements.verify!() setup_instrumenters() load_custom_modules() - check_system_commands() Pleroma.Docs.JSON.compile() adapter = Application.get_env(:tesla, :adapter) @@ -260,37 +259,4 @@ defmodule Pleroma.Application do end defp http_children(_, _), do: [] - - defp check_system_commands do - filters = Config.get([Pleroma.Upload, :filters]) - - check_filter = fn filter, command_required -> - with true <- filter in filters, - false <- Pleroma.Utils.command_available?(command_required) do - Logger.error( - "#{filter} is specified in list of Pleroma.Upload filters, but the " <> - "#{command_required} command is not found" - ) - end - end - - check_filter.(Pleroma.Upload.Filters.Exiftool, "exiftool") - check_filter.(Pleroma.Upload.Filters.Mogrify, "mogrify") - check_filter.(Pleroma.Upload.Filters.Mogrifun, "mogrify") - - with true <- Config.get([:media_preview_proxy, :enabled]), - missing_graphics_tools = Pleroma.Helpers.MediaHelper.missing_dependencies(), - [] <- missing_graphics_tools do - :noop - else - false -> - :noop - - missing_graphics_tools -> - Logger.error( - "The following dependencies required by Media preview proxy " <> - "(which is currently enabled) are not installed: #{inspect(missing_graphics_tools)}" - ) - end - end end diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 16f62b6f5..b977257a3 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -9,6 +9,9 @@ defmodule Pleroma.ApplicationRequirements do defmodule VerifyError, do: defexception([:message]) + alias Pleroma.Config + alias Pleroma.Helpers.MediaHelper + import Ecto.Query require Logger @@ -16,7 +19,8 @@ defmodule Pleroma.ApplicationRequirements do @spec verify!() :: :ok | VerifyError.t() def verify! do :ok - |> check_confirmation_accounts! + |> check_system_commands!() + |> check_confirmation_accounts!() |> check_migrations_applied!() |> check_welcome_message_config!() |> check_rum!() @@ -48,7 +52,9 @@ defmodule Pleroma.ApplicationRequirements do if Pleroma.Config.get([:instance, :account_activation_required]) && not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do Logger.error( - "Account activation enabled, but no Mailer settings enabled.\nPlease set config :pleroma, :instance, account_activation_required: false\nOtherwise setup and enable Mailer." + "Account activation enabled, but no Mailer settings enabled.\n" <> + "Please set config :pleroma, :instance, account_activation_required: false\n" <> + "Otherwise setup and enable Mailer." ) {:error, @@ -81,7 +87,9 @@ defmodule Pleroma.ApplicationRequirements do Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end) Logger.error( - "The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true" + "The following migrations were not applied:\n#{down_migrations_text}" <> + "If you want to start Pleroma anyway, set\n" <> + "config :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true" ) {:error, "Unapplied Migrations detected"} @@ -124,14 +132,22 @@ defmodule Pleroma.ApplicationRequirements do case {setting, migrate} do {true, false} -> Logger.error( - "Use `RUM` index is enabled, but were not applied migrations for it.\nIf you want to start Pleroma anyway, set\nconfig :pleroma, :database, rum_enabled: false\nOtherwise apply the following migrations:\n`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`" + "Use `RUM` index is enabled, but were not applied migrations for it.\n" <> + "If you want to start Pleroma anyway, set\n" <> + "config :pleroma, :database, rum_enabled: false\n" <> + "Otherwise apply the following migrations:\n" <> + "`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`" ) {:error, "Unapplied RUM Migrations detected"} {false, true} -> Logger.error( - "Detected applied migrations to use `RUM` index, but `RUM` isn't enable in settings.\nIf you want to use `RUM`, set\nconfig :pleroma, :database, rum_enabled: true\nOtherwise roll `RUM` migrations back.\n`mix ecto.rollback --migrations-path priv/repo/optional_migrations/rum_indexing/`" + "Detected applied migrations to use `RUM` index, but `RUM` isn't enable in settings.\n" <> + "If you want to use `RUM`, set\n" <> + "config :pleroma, :database, rum_enabled: true\n" <> + "Otherwise roll `RUM` migrations back.\n" <> + "`mix ecto.rollback --migrations-path priv/repo/optional_migrations/rum_indexing/`" ) {:error, "RUM Migrations detected"} @@ -140,4 +156,50 @@ defmodule Pleroma.ApplicationRequirements do :ok end end + + defp check_system_commands!(:ok) do + filter_commands_statuses = [ + check_filter(Pleroma.Upload.Filters.Exiftool, "exiftool"), + check_filter(Pleroma.Upload.Filters.Mogrify, "mogrify"), + check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify") + ] + + preview_proxy_commands_status = + if !Config.get([:media_preview_proxy, :enabled]) or + MediaHelper.missing_dependencies() == [] do + true + else + Logger.error( + "The following dependencies required by Media preview proxy " <> + "(which is currently enabled) are not installed: " <> + inspect(MediaHelper.missing_dependencies()) + ) + + false + end + + if Enum.all?([preview_proxy_commands_status | filter_commands_statuses], & &1) do + :ok + else + {:error, + "System commands missing. Check logs and see `docs/installation` for more details."} + end + end + + defp check_system_commands!(result), do: result + + defp check_filter(filter, command_required) do + filters = Config.get([Pleroma.Upload, :filters]) + + if filter in filters and not Pleroma.Utils.command_available?(command_required) do + Logger.error( + "#{filter} is specified in list of Pleroma.Upload filters, but the " <> + "#{command_required} command is not found" + ) + + false + else + true + end + end end From 9a56ec25cb4de1284a03cac156a7664bd65b628f Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 29 Sep 2020 16:24:02 +0200 Subject: [PATCH 38/49] Changelog: Add info about relay unfollowing --- CHANGELOG.md | 1 + docs/API/admin_api.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80fbc078d..0d3daa60e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`) - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`) +- Mix task option for force-unfollowing relays ### Changed diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index ddcab1a2c..7bf13daef 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -351,7 +351,7 @@ Response: - Params: - `relay_url` - - *optional* `force`: forcefully unfollow a relay even when the relay is not available. (default is `false`) + - *optional* `force`: forcefully unfollow a relay even when the relay is not available. (default is `false`) Response: From a24f80badf2c24a12de1e99a429aa23e2b40b40c Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 29 Sep 2020 14:30:18 +0000 Subject: [PATCH 39/49] Apply 1 suggestion(s) to 1 file(s) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e5f8a504..8b0f72eb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking:** Introduced dependencies on `ffmpeg` and `ImageMagick` software packages. Please refer to documentation in `docs/installation`. ### Added -- Media preview proxy (requires `ffmpeg` and `ImageMagick` be installed and media proxy be enabled; see `:media_preview_proxy` config for more details). +- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details). - Pleroma API: Importing the mutes users from CSV files. - Experimental websocket-based federation between Pleroma instances. - Admin API: Importing emoji from a zip file From ae262846389aabfc71c12c2eb903cbe367140184 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 29 Sep 2020 17:34:49 +0300 Subject: [PATCH 40/49] [#3031] Adjusted changelog entry. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b0f72eb3..f11585113 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false. - Users with the `discoverable` field set to false will not show up in searches. - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option). -- **Breaking:** Introduced dependencies on `ffmpeg` and `ImageMagick` software packages. Please refer to documentation in `docs/installation`. +- Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`. ### Added - Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details). From b3a9ba09ec5867d240c3769ae4c3fbf598f68d92 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 29 Sep 2020 12:16:15 -0500 Subject: [PATCH 41/49] More robust expires_at timestamp processing --- lib/pleroma/user.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 71ace1c34..09ea80793 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2291,7 +2291,9 @@ defmodule Pleroma.User do # if pinned activity was scheduled for deletion, we reschedule it for deletion if data["expires_at"] do - {:ok, expires_at, _} = DateTime.from_iso8601(data["expires_at"]) + # MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation + {:ok, expires_at} = + data["expires_at"] |> Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast() Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ activity_id: id, From 006b62fd12adadbf698419990ab13bf6f1e901b2 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 29 Sep 2020 21:49:04 +0300 Subject: [PATCH 42/49] OpenAPI CastAndValidate: filter out empty params Closes #2198 --- CHANGELOG.md | 5 +++++ lib/pleroma/web/api_spec/cast_and_validate.ex | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80fbc078d..983ddd628 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,11 @@ switched to a new configuration mechanism, however it was not officially removed - Add documented-but-missing chat pagination. - Allow sending out emails again. +## Unreleased (Patch) + +### Changed +- API: Empty parameter values for integer parameters are now ignored in non-strict validaton mode. + ## [2.1.2] - 2020-09-17 ### Security diff --git a/lib/pleroma/web/api_spec/cast_and_validate.ex b/lib/pleroma/web/api_spec/cast_and_validate.ex index fbfc27d6f..6d1a7ebbc 100644 --- a/lib/pleroma/web/api_spec/cast_and_validate.ex +++ b/lib/pleroma/web/api_spec/cast_and_validate.ex @@ -115,6 +115,10 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do %{reason: :unexpected_field, name: name, path: [name]}, params -> Map.delete(params, name) + # Filter out empty params + %{reason: :invalid_type, path: [name_atom], value: ""}, params -> + Map.delete(params, to_string(name_atom)) + %{reason: :invalid_enum, name: nil, path: path, value: value}, params -> path = path |> Enum.reverse() |> tl() |> Enum.reverse() |> list_items_to_string() update_in(params, path, &List.delete(&1, value)) From 90fee49c52799a7d6ad890ecc49d146ab6ad8455 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 30 Sep 2020 14:14:41 +0200 Subject: [PATCH 43/49] User search: Once again, change uri handling. They can indeed be non-unique. --- lib/pleroma/user/search.ex | 5 +++-- .../20200930082320_user_ur_is_index_part_three.exs | 8 ++++++++ test/user_search_test.exs | 6 ++++++ 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 priv/repo/migrations/20200930082320_user_ur_is_index_part_three.exs diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index 03f2c552f..35a828008 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -54,8 +54,9 @@ defmodule Pleroma.User.Search do defp maybe_add_uri_match(list, query) do with {:ok, query} <- UriType.cast(query), - %User{} = user <- Pleroma.Repo.get_by(User, uri: query) do - [user.id | list] + q = from(u in User, where: u.uri == ^query, select: u.id), + users = Pleroma.Repo.all(q) do + users ++ list else _ -> list end diff --git a/priv/repo/migrations/20200930082320_user_ur_is_index_part_three.exs b/priv/repo/migrations/20200930082320_user_ur_is_index_part_three.exs new file mode 100644 index 000000000..816c6526e --- /dev/null +++ b/priv/repo/migrations/20200930082320_user_ur_is_index_part_three.exs @@ -0,0 +1,8 @@ +defmodule Pleroma.Repo.Migrations.UserURIsIndexPartThree do + use Ecto.Migration + + def change do + drop_if_exists(unique_index(:users, :uri)) + create_if_not_exists(index(:users, :uri)) + end +end diff --git a/test/user_search_test.exs b/test/user_search_test.exs index cc14e9741..c4b805005 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -36,6 +36,12 @@ defmodule Pleroma.UserSearchTest do assert first_user.id == user.id end + test "doesn't die if two users have the same uri" do + insert(:user, %{uri: "https://gensokyo.2hu/@raymoo"}) + insert(:user, %{uri: "https://gensokyo.2hu/@raymoo"}) + assert [_first_user, _second_user] = User.search("https://gensokyo.2hu/@raymoo") + end + test "returns a user with matching uri as the first result" do user = insert(:user, %{ From 1ae39441e6d516f8f1a0dd3ed47de9d8427477c5 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 30 Sep 2020 14:17:21 +0200 Subject: [PATCH 44/49] Migrations: Nullify old unique index on users.uri --- priv/repo/migrations/20200928145912_revert_citext_change.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/repo/migrations/20200928145912_revert_citext_change.exs b/priv/repo/migrations/20200928145912_revert_citext_change.exs index ab232f607..685a98533 100644 --- a/priv/repo/migrations/20200928145912_revert_citext_change.exs +++ b/priv/repo/migrations/20200928145912_revert_citext_change.exs @@ -6,6 +6,6 @@ defmodule Pleroma.Repo.Migrations.RevertCitextChange do modify(:uri, :text) end - create_if_not_exists(unique_index(:users, :uri)) + # create_if_not_exists(unique_index(:users, :uri)) end end From cee1883b35b5453782bea1e34ff9cbf1bde52f7c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Sep 2020 07:53:19 -0500 Subject: [PATCH 45/49] Update Oban to 2.1.0 --- mix.exs | 2 +- mix.lock | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mix.exs b/mix.exs index 18f748672..b9ce8c500 100644 --- a/mix.exs +++ b/mix.exs @@ -122,7 +122,7 @@ defmodule Pleroma.Mixfile do {:ecto_enum, "~> 1.4"}, {:ecto_sql, "~> 3.4.4"}, {:postgrex, ">= 0.15.5"}, - {:oban, "~> 2.0.0"}, + {:oban, "~> 2.1.0"}, {:gettext, "~> 0.18"}, {:pbkdf2_elixir, "~> 1.2"}, {:bcrypt_elixir, "~> 2.2"}, diff --git a/mix.lock b/mix.lock index adb3f024a..2603f70c0 100644 --- a/mix.lock +++ b/mix.lock @@ -24,11 +24,11 @@ "crypt": {:git, "https://github.com/msantos/crypt.git", "f63a705f92c26955977ee62a313012e309a4d77a", [ref: "f63a705f92c26955977ee62a313012e309a4d77a"]}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, "db_connection": {:hex, :db_connection, "2.2.2", "3bbca41b199e1598245b716248964926303b5d4609ff065125ce98bcd368939e", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "642af240d8a8affb93b4ba5a6fcd2bbcbdc327e1a524b825d383711536f8070c"}, - "decimal": {:hex, :decimal, "1.8.1", "a4ef3f5f3428bdbc0d35374029ffcf4ede8533536fa79896dd450168d9acdf3c", [:mix], [], "hexpm", "3cb154b00225ac687f6cbd4acc4b7960027c757a5152b369923ead9ddbca7aec"}, + "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"}, "earmark_parser": {:hex, :earmark_parser, "1.4.10", "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", [:mix], [], "hexpm", "8e2d5370b732385db2c9b22215c3f59c84ac7dda7ed7e544d7c459496ae519c0"}, - "ecto": {:hex, :ecto, "3.4.5", "2bcd262f57b2c888b0bd7f7a28c8a48aa11dc1a2c6a858e45dd8f8426d504265", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8c6d1d4d524559e9b7a062f0498e2c206122552d63eacff0a6567ffe7a8e8691"}, + "ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, "ecto_sql": {:hex, :ecto_sql, "3.4.5", "30161f81b167d561a9a2df4329c10ae05ff36eca7ccc84628f2c8b9fa1e43323", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31990c6a3579b36a3c0841d34a94c275e727de8b84f58509da5f1b2032c98ac2"}, "eimp": {:hex, :eimp, "1.0.14", "fc297f0c7e2700457a95a60c7010a5f1dcb768a083b6d53f49cd94ab95a28f22", [:rebar3], [{:p1_utils, "1.0.18", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "501133f3112079b92d9e22da8b88bf4f0e13d4d67ae9c15c42c30bd25ceb83b6"}, @@ -59,7 +59,7 @@ "httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "aa2c74bd271af34239a3948779612f87df2422c2fdcfdbcec28d9c105f0773fe"}, "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "4bdd305eb64e18b0273864920695cb18d7a2021f31a11b9c5fbcd9a253f936e2"}, "inet_cidr": {:hex, :inet_cidr, "1.0.4", "a05744ab7c221ca8e395c926c3919a821eb512e8f36547c062f62c4ca0cf3d6e", [:mix], [], "hexpm", "64a2d30189704ae41ca7dbdd587f5291db5d1dda1414e0774c29ffc81088c1bc"}, - "jason": {:hex, :jason, "1.2.1", "12b22825e22f468c02eb3e4b9985f3d0cb8dc40b9bd704730efa11abd2708c44", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b659b8571deedf60f79c5a608e15414085fa141344e2716fbd6988a084b5f993"}, + "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"}, "joken": {:hex, :joken, "2.2.0", "2daa1b12be05184aff7b5ace1d43ca1f81345962285fff3f88db74927c954d3a", [:mix], [{:jose, "~> 1.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "b4f92e30388206f869dd25d1af628a1d99d7586e5cf0672f64d4df84c4d2f5e9"}, "jose": {:hex, :jose, "1.10.1", "16d8e460dae7203c6d1efa3f277e25b5af8b659febfc2f2eb4bacf87f128b80a", [:mix, :rebar3], [], "hexpm", "3c7ddc8a9394b92891db7c2771da94bf819834a1a4c92e30857b7d582e2f8257"}, "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"}, @@ -79,7 +79,7 @@ "nimble_parsec": {:hex, :nimble_parsec, "0.6.0", "32111b3bf39137144abd7ba1cce0914533b2d16ef35e8abc5ec8be6122944263", [:mix], [], "hexpm", "27eac315a94909d4dc68bc07a4a83e06c8379237c5ea528a9acff4ca1c873c52"}, "nimble_pool": {:hex, :nimble_pool, "0.1.0", "ffa9d5be27eee2b00b0c634eb649aa27f97b39186fec3c493716c2a33e784ec6", [:mix], [], "hexpm", "343a1eaa620ddcf3430a83f39f2af499fe2370390d4f785cd475b4df5acaf3f9"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, - "oban": {:hex, :oban, "2.0.0", "e6ce70d94dd46815ec0882a1ffb7356df9a9d5b8a40a64ce5c2536617a447379", [:mix], [{:ecto_sql, ">= 3.4.3", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cf574813bd048b98a698aa587c21367d2e06842d4e1b1993dcd6a696e9e633bd"}, + "oban": {:hex, :oban, "2.1.0", "034144686f7e76a102b5d67731f098d98a9e4a52b07c25ad580a01f83a7f1cf5", [:mix], [{:ecto_sql, ">= 3.4.3", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c6f067fa3b308ed9e0e6beb2b34277c9c4e48bf95338edabd8f4a757a26e04c2"}, "open_api_spex": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", "f296ac0924ba3cf79c7a588c4c252889df4c2edd", [ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"]}, "p1_utils": {:hex, :p1_utils, "1.0.18", "3fe224de5b2e190d730a3c5da9d6e8540c96484cf4b4692921d1e28f0c32b01c", [:rebar3], [], "hexpm", "1fc8773a71a15553b179c986b22fbeead19b28fe486c332d4929700ffeb71f88"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, @@ -95,7 +95,7 @@ "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, - "postgrex": {:hex, :postgrex, "0.15.5", "aec40306a622d459b01bff890fa42f1430dac61593b122754144ad9033a2152f", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "ed90c81e1525f65a2ba2279dbcebf030d6d13328daa2f8088b9661eb9143af7f"}, + "postgrex": {:hex, :postgrex, "0.15.6", "a464c72010a56e3214fe2b99c1a76faab4c2bb0255cabdef30dea763a3569aa2", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "f99268325ac8f66ffd6c4964faab9e70fbf721234ab2ad238c00f9530b8cdd55"}, "pot": {:hex, :pot, "0.11.0", "61bad869a94534739dd4614a25a619bc5c47b9970e9a0ea5bef4628036fc7a16", [:rebar3], [], "hexpm", "57ee6ee6bdeb639661ffafb9acefe3c8f966e45394de6a766813bb9e1be4e54b"}, "prometheus": {:hex, :prometheus, "4.6.0", "20510f381db1ccab818b4cf2fac5fa6ab5cc91bc364a154399901c001465f46f", [:mix, :rebar3], [], "hexpm", "4905fd2992f8038eccd7aa0cd22f40637ed618c0bed1f75c05aacec15b7545de"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.3", "3dd4da1812b8e0dbee81ea58bb3b62ed7588f2eae0c9e97e434c46807ff82311", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "8d66289f77f913b37eda81fd287340c17e61a447549deb28efc254532b2bed82"}, @@ -120,5 +120,5 @@ "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm", "1d1848c40487cdb0b30e8ed975e34e025860c02e419cb615d255849f3427439d"}, "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"}, "web_push_encryption": {:hex, :web_push_encryption, "0.3.0", "598b5135e696fd1404dc8d0d7c0fa2c027244a4e5d5e5a98ba267f14fdeaabc8", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "f10bdd1afe527ede694749fb77a2f22f146a51b054c7fa541c9fd920fba7c875"}, - "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []} + "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []}, } From 49229107e812d649e66e36e32e238db66b353b0f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Sep 2020 12:32:54 -0500 Subject: [PATCH 46/49] Make it possible for Varnish logs to contain the true scheme used by clients instead of always reporting http:// --- installation/pleroma.vcl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/installation/pleroma.vcl b/installation/pleroma.vcl index 154747aa6..57b6d17b3 100644 --- a/installation/pleroma.vcl +++ b/installation/pleroma.vcl @@ -1,3 +1,4 @@ +# Recommended varnishncsa logging format: '%h %l %u %t "%m %{X-Forwarded-Proto}i://%{Host}i%U%q %H" %s %b "%{Referer}i" "%{User-agent}i"' vcl 4.1; import std; @@ -14,8 +15,11 @@ acl purge { sub vcl_recv { # Redirect HTTP to HTTPS if (std.port(server.ip) != 443) { + set req.http.X-Forwarded-Proto = "http"; set req.http.x-redir = "https://" + req.http.host + req.url; return (synth(750, "")); + } else { + set req.http.X-Forwarded-Proto = "https"; } # CHUNKED SUPPORT From b3015db841536c26934f43374ed75fb77a11ff68 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Sep 2020 12:49:51 -0500 Subject: [PATCH 47/49] Syntax error --- installation/pleroma.vcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installation/pleroma.vcl b/installation/pleroma.vcl index 57b6d17b3..13dad784c 100644 --- a/installation/pleroma.vcl +++ b/installation/pleroma.vcl @@ -109,7 +109,7 @@ sub vcl_hash { sub vcl_backend_fetch { # Be more lenient for slow servers on the fediverse - if bereq.url ~ "^/proxy/" { + if (bereq.url ~ "^/proxy/") { set bereq.first_byte_timeout = 300s; } From cbdaabad345914e7424e614032056ff86e21142f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 1 Oct 2020 13:32:11 +0300 Subject: [PATCH 48/49] web push http_client fix --- config/benchmark.exs | 2 -- config/config.exs | 2 +- config/test.exs | 2 -- lib/pleroma/http/web_push.ex | 12 ++++++++++++ lib/pleroma/web/push/impl.ex | 12 ++++++------ test/support/web_push_http_client_mock.ex | 23 ----------------------- test/web/push/impl_test.exs | 22 ++-------------------- 7 files changed, 21 insertions(+), 54 deletions(-) create mode 100644 lib/pleroma/http/web_push.ex delete mode 100644 test/support/web_push_http_client_mock.ex diff --git a/config/benchmark.exs b/config/benchmark.exs index e867253eb..5567ff26e 100644 --- a/config/benchmark.exs +++ b/config/benchmark.exs @@ -59,8 +59,6 @@ config :web_push_encryption, :vapid_details, "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4", private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA" -config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock - config :pleroma, Pleroma.ScheduledActivity, daily_user_limit: 2, total_user_limit: 3, diff --git a/config/config.exs b/config/config.exs index 00624bf00..2e6b0796a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -809,7 +809,7 @@ config :tzdata, :http_client, Pleroma.HTTP.Tzdata config :ex_aws, http_client: Pleroma.HTTP.ExAws -config :web_push_encryption, http_client: Pleroma.HTTP +config :web_push_encryption, http_client: Pleroma.HTTP.WebPush config :pleroma, :instances_favicons, enabled: false diff --git a/config/test.exs b/config/test.exs index 93a0e2a61..95f860f2f 100644 --- a/config/test.exs +++ b/config/test.exs @@ -83,8 +83,6 @@ config :web_push_encryption, :vapid_details, "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4", private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA" -config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock - config :pleroma, Oban, queues: false, crontab: false, diff --git a/lib/pleroma/http/web_push.ex b/lib/pleroma/http/web_push.ex new file mode 100644 index 000000000..78148a12e --- /dev/null +++ b/lib/pleroma/http/web_push.ex @@ -0,0 +1,12 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.WebPush do + @moduledoc false + + def post(url, payload, headers) do + list_headers = Map.to_list(headers) + Pleroma.HTTP.post(url, payload, list_headers) + end +end diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index 16368485e..da535aa68 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.Push.Impl do @types ["Create", "Follow", "Announce", "Like", "Move"] @doc "Performs sending notifications for user subscriptions" - @spec perform(Notification.t()) :: list(any) | :error + @spec perform(Notification.t()) :: list(any) | :error | {:error, :unknown_type} def perform( %{ activity: %{data: %{"type" => activity_type}} = activity, @@ -64,20 +64,20 @@ defmodule Pleroma.Web.Push.Impl do @doc "Push message to web" def push_message(body, sub, api_key, subscription) do case WebPushEncryption.send_web_push(body, sub, api_key) do - {:ok, %{status_code: code}} when 400 <= code and code < 500 -> + {:ok, %{status: code}} when code in 400..499 -> Logger.debug("Removing subscription record") Repo.delete!(subscription) :ok - {:ok, %{status_code: code}} when 200 <= code and code < 300 -> + {:ok, %{status: code}} when code in 200..299 -> :ok - {:ok, %{status_code: code}} -> + {:ok, %{status: code}} -> Logger.error("Web Push Notification failed with code: #{code}") :error - _ -> - Logger.error("Web Push Notification failed with unknown error") + error -> + Logger.error("Web Push Notification failed with #{inspect(error)}") :error end end diff --git a/test/support/web_push_http_client_mock.ex b/test/support/web_push_http_client_mock.ex deleted file mode 100644 index 3cd12957d..000000000 --- a/test/support/web_push_http_client_mock.ex +++ /dev/null @@ -1,23 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.WebPushHttpClientMock do - def get(url, headers \\ [], options \\ []) do - { - res, - %Tesla.Env{status: status} - } = Pleroma.HTTP.request(:get, url, "", headers, options) - - {res, %{status_code: status}} - end - - def post(url, body, headers \\ [], options \\ []) do - { - res, - %Tesla.Env{status: status} - } = Pleroma.HTTP.request(:post, url, body, headers, options) - - {res, %{status_code: status}} - end -end diff --git a/test/web/push/impl_test.exs b/test/web/push/impl_test.exs index c7c17e156..6cab46696 100644 --- a/test/web/push/impl_test.exs +++ b/test/web/push/impl_test.exs @@ -5,6 +5,8 @@ defmodule Pleroma.Web.Push.ImplTest do use Pleroma.DataCase + import Pleroma.Factory + alias Pleroma.Notification alias Pleroma.Object alias Pleroma.User @@ -12,10 +14,6 @@ defmodule Pleroma.Web.Push.ImplTest do alias Pleroma.Web.CommonAPI alias Pleroma.Web.Push.Impl alias Pleroma.Web.Push.Subscription - alias Pleroma.Web.WebPushHttpClientMock - - import Mock - import Pleroma.Factory setup do Tesla.Mock.mock(fn @@ -80,22 +78,6 @@ defmodule Pleroma.Web.Push.ImplTest do assert Impl.push_message(@message, @sub, @api_key, %Subscription{}) == :ok end - test_with_mock "uses WebPushHttpClientMock as an HTTP client", WebPushHttpClientMock, - post: fn _, _, _ -> {:ok, %{status_code: 200}} end do - Impl.push_message(@message, @sub, @api_key, %Subscription{}) - assert_called(WebPushHttpClientMock.post("https://example.com/example/1234", :_, :_)) - end - - test_with_mock "uses Pleroma.HTTP as an HTTP client", Pleroma.HTTP, - post: fn _, _, _ -> {:ok, %{status_code: 200}} end do - client = Application.get_env(:web_push_encryption, :http_client) - on_exit(fn -> Application.put_env(:web_push_encryption, :http_client, client) end) - Application.put_env(:web_push_encryption, :http_client, Pleroma.HTTP) - - Impl.push_message(@message, @sub, @api_key, %Subscription{}) - assert_called(Pleroma.HTTP.post("https://example.com/example/1234", :_, :_)) - end - @tag capture_log: true test "fail message sending" do assert Impl.push_message( From 8d093a68a6b54eff232566dfd7f52e9c8682398a Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 5 Oct 2020 13:58:12 +0200 Subject: [PATCH 49/49] Docs: Modify docs so the postgres config is harder to get wrong. --- docs/installation/otp_en.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index f6b05c887..92584d80d 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -101,6 +101,8 @@ It is encouraged to check [Optimizing your PostgreSQL performance](../configurat If you are using PostgreSQL 12 or higher, add this to your Ecto database configuration ```elixir +# +config :pleroma, Pleroma.Repo, prepare: :named, parameters: [ plan_cache_mode: "force_custom_plan"