From d0c1997d483f918b46bdf45cecef82d8aabcb5f1 Mon Sep 17 00:00:00 2001 From: Sean King Date: Sat, 19 Mar 2022 23:33:37 -0600 Subject: [PATCH 001/123] Rewrite integration-test websocket client with Mint.WebSocket --- mix.exs | 3 +- mix.lock | 1 + .../integration/mastodon_websocket_test.exs | 16 +- test/support/websocket_client.ex | 195 +++++++++++++++--- 4 files changed, 178 insertions(+), 37 deletions(-) diff --git a/mix.exs b/mix.exs index 7893b8438..f84191321 100644 --- a/mix.exs +++ b/mix.exs @@ -208,7 +208,8 @@ defmodule Pleroma.Mixfile do {:excoveralls, "0.12.3", only: :test}, {:hackney, "~> 1.18.0", override: true}, {:mox, "~> 1.0", only: :test}, - {:websocket_client, git: "https://github.com/jeremyong/websocket_client.git", only: :test} + {:mint, "~> 1.4", only: :test, override: true}, + {:mint_web_socket, "~> 0.3.0", only: :test} ] ++ oauth_deps() end diff --git a/mix.lock b/mix.lock index 817240538..81f7e4399 100644 --- a/mix.lock +++ b/mix.lock @@ -79,6 +79,7 @@ "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "mint": {:hex, :mint, "1.4.0", "cd7d2451b201fc8e4a8fd86257fb3878d9e3752899eb67b0c5b25b180bde1212", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "10a99e144b815cbf8522dccbc8199d15802440fc7a64d67b6853adb6fa170217"}, + "mint_web_socket": {:hex, :mint_web_socket, "0.3.0", "c9e130dcc778d673fd713eb66434e16cf7d89cee0754e75f26f8bd9a9e592b63", [:mix], [{:mint, "~> 1.4 and >= 1.4.1", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "0605bc3fa684e1a7719b22a3f74be4de5e6a16dd43ac18ebcea72e2adc33b532"}, "mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"}, "mock": {:hex, :mock, "0.3.7", "75b3bbf1466d7e486ea2052a73c6e062c6256fb429d6797999ab02fa32f29e03", [:mix], [{:meck, "~> 0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "4da49a4609e41fd99b7836945c26f373623ea968cfb6282742bcb94440cf7e5c"}, "mogrify": {:hex, :mogrify, "0.9.1", "a26f107c4987477769f272bd0f7e3ac4b7b75b11ba597fd001b877beffa9c068", [:mix], [], "hexpm", "134edf189337d2125c0948bf0c228fdeef975c594317452d536224069a5b7f05"}, diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs index 2d4c7f63b..5599ce030 100644 --- a/test/pleroma/integration/mastodon_websocket_test.exs +++ b/test/pleroma/integration/mastodon_websocket_test.exs @@ -28,21 +28,21 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do qs -> @path <> qs end - WebsocketClient.start_link(self(), path, headers) + WebsocketClient.connect(self(), path, headers) end test "refuses invalid requests" do capture_log(fn -> - assert {:error, {404, _}} = start_socket() - assert {:error, {404, _}} = start_socket("?stream=ncjdk") + assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 404}} = start_socket() + assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 404}} = start_socket("?stream=ncjdk") Process.sleep(30) end) end test "requires authentication and a valid token for protected streams" do capture_log(fn -> - assert {:error, {401, _}} = start_socket("?stream=user&access_token=aaaaaaaaaaaa") - assert {:error, {401, _}} = start_socket("?stream=user") + assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = start_socket("?stream=user&access_token=aaaaaaaaaaaa") + assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = start_socket("?stream=user") Process.sleep(30) end) end @@ -102,7 +102,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") capture_log(fn -> - assert {:error, {401, _}} = start_socket("?stream=user") + assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = start_socket("?stream=user") Process.sleep(30) end) end @@ -111,7 +111,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") capture_log(fn -> - assert {:error, {401, _}} = start_socket("?stream=user:notification") + assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = start_socket("?stream=user:notification") Process.sleep(30) end) end @@ -120,7 +120,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) capture_log(fn -> - assert {:error, {401, _}} = + assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", "I am a friend"}]) Process.sleep(30) diff --git a/test/support/websocket_client.ex b/test/support/websocket_client.ex index d149b324e..afcd0e8c7 100644 --- a/test/support/websocket_client.ex +++ b/test/support/websocket_client.ex @@ -3,60 +3,199 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Integration.WebsocketClient do - # https://github.com/phoenixframework/phoenix/blob/master/test/support/websocket_client.exs + @moduledoc """ + A WebSocket client used to test Mastodon API streaming + + Based on Phoenix Framework's WebsocketClient + https://github.com/phoenixframework/phoenix/blob/master/test/support/websocket_client.exs + """ + + use GenServer + import Kernel, except: [send: 2] + + defstruct [ + :conn, + :request_ref, + :websocket, + :caller, + :status, + :resp_headers, + :sender, + closing?: false + ] @doc """ - Starts the WebSocket server for given ws URL. Received Socket.Message's - are forwarded to the sender pid + Starts the WebSocket client for given ws URL. `Phoenix.Socket.Message`s + received from the server are forwarded to the sender pid. """ - def start_link(sender, url, headers \\ []) do - :crypto.start() - :ssl.start() - - :websocket_client.start_link( - String.to_charlist(url), - __MODULE__, - [sender], - extra_headers: headers - ) + def connect(sender, url, headers \\ []) do + with {:ok, socket} <- GenServer.start_link(__MODULE__, {sender}), + {:ok, :connected} <- GenServer.call(socket, {:connect, url, headers}) do + {:ok, socket} + end end @doc """ Closes the socket """ def close(socket) do - send(socket, :close) + GenServer.cast(socket, :close) end @doc """ Sends a low-level text message to the client. """ def send_text(server_pid, msg) do - send(server_pid, {:text, msg}) + GenServer.call(server_pid, {:text, msg}) end @doc false - def init([sender], _conn_state) do - {:ok, %{sender: sender}} - end + def init({sender}) do + state = %__MODULE__{sender: sender} - @doc false - def websocket_handle(frame, _conn_state, state) do - send(state.sender, frame) {:ok, state} end @doc false - def websocket_info({:text, msg}, _conn_state, state) do - {:reply, {:text, msg}, state} - end + def handle_call({:connect, url, headers}, from, state) do + uri = URI.parse(url) - def websocket_info(:close, _conn_state, _state) do - {:close, <<>>, "done"} + http_scheme = + case uri.scheme do + "ws" -> :http + "wss" -> :https + end + + ws_scheme = + case uri.scheme do + "ws" -> :ws + "wss" -> :wss + end + + path = + case uri.query do + nil -> uri.path + query -> uri.path <> "?" <> query + end + + with {:ok, conn} <- Mint.HTTP.connect(http_scheme, uri.host, uri.port), + {:ok, conn, ref} <- Mint.WebSocket.upgrade(ws_scheme, conn, path, headers) do + state = %{state | conn: conn, request_ref: ref, caller: from} + {:noreply, state} + else + {:error, reason} -> + {:reply, {:error, reason}, state} + + {:error, conn, reason} -> + {:reply, {:error, reason}, put_in(state.conn, conn)} + end end @doc false - def websocket_terminate(_reason, _conn_state, _state) do - :ok + def handle_info(message, state) do + case Mint.WebSocket.stream(state.conn, message) do + {:ok, conn, responses} -> + state = put_in(state.conn, conn) |> handle_responses(responses) + if state.closing?, do: do_close(state), else: {:noreply, state} + + {:error, conn, reason, _responses} -> + state = put_in(state.conn, conn) |> reply({:error, reason}) + {:noreply, state} + + :unknown -> + {:noreply, state} + end + end + + defp do_close(state) do + # Streaming a close frame may fail if the server has already closed + # for writing. + _ = stream_frame(state, :close) + Mint.HTTP.close(state.conn) + {:stop, :normal, state} + end + + defp handle_responses(state, responses) + + defp handle_responses(%{request_ref: ref} = state, [{:status, ref, status} | rest]) do + put_in(state.status, status) + |> handle_responses(rest) + end + + defp handle_responses(%{request_ref: ref} = state, [{:headers, ref, resp_headers} | rest]) do + put_in(state.resp_headers, resp_headers) + |> handle_responses(rest) + end + + defp handle_responses(%{request_ref: ref} = state, [{:done, ref} | rest]) do + case Mint.WebSocket.new(state.conn, ref, state.status, state.resp_headers) do + {:ok, conn, websocket} -> + %{state | conn: conn, websocket: websocket, status: nil, resp_headers: nil} + |> reply({:ok, :connected}) + |> handle_responses(rest) + + {:error, conn, reason} -> + put_in(state.conn, conn) + |> reply({:error, reason}) + end + end + + defp handle_responses(%{request_ref: ref, websocket: websocket} = state, [ + {:data, ref, data} | rest + ]) + when websocket != nil do + case Mint.WebSocket.decode(websocket, data) do + {:ok, websocket, frames} -> + put_in(state.websocket, websocket) + |> handle_frames(frames) + |> handle_responses(rest) + + {:error, websocket, reason} -> + put_in(state.websocket, websocket) + |> reply({:error, reason}) + end + end + + defp handle_responses(state, [_response | rest]) do + handle_responses(state, rest) + end + + defp handle_responses(state, []), do: state + + defp handle_frames(state, frames) do + {frames, state} = + Enum.flat_map_reduce(frames, state, fn + # prepare to close the connection when a close frame is received + {:close, _code, _data}, state -> + {[], put_in(state.closing?, true)} + + frame, state -> + {[frame], state} + end) + + Enum.each(frames, &Kernel.send(state.sender, &1)) + + state + end + + defp reply(state, response) do + if state.caller, do: GenServer.reply(state.caller, response) + put_in(state.caller, nil) + end + + # Encodes a frame as a binary and sends it along the wire, keeping `conn` + # and `websocket` up to date in `state`. + defp stream_frame(state, frame) do + with {:ok, websocket, data} <- Mint.WebSocket.encode(state.websocket, frame), + state = put_in(state.websocket, websocket), + {:ok, conn} <- Mint.WebSocket.stream_request_body(state.conn, state.request_ref, data) do + {:ok, put_in(state.conn, conn)} + else + {:error, %Mint.WebSocket{} = websocket, reason} -> + {:error, put_in(state.websocket, websocket), reason} + + {:error, conn, reason} -> + {:error, put_in(state.conn, conn), reason} + end end end From 4194559ea6d3e0f219ae8e77b468782ac115d134 Mon Sep 17 00:00:00 2001 From: Sean King Date: Sun, 20 Mar 2022 17:26:07 -0600 Subject: [PATCH 002/123] Fix lint errors --- .../integration/mastodon_websocket_test.exs | 21 ++++++++++++++----- test/support/websocket_client.ex | 4 ++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs index 5599ce030..16525c740 100644 --- a/test/pleroma/integration/mastodon_websocket_test.exs +++ b/test/pleroma/integration/mastodon_websocket_test.exs @@ -34,15 +34,22 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do test "refuses invalid requests" do capture_log(fn -> assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 404}} = start_socket() - assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 404}} = start_socket("?stream=ncjdk") + + assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 404}} = + start_socket("?stream=ncjdk") + Process.sleep(30) end) end test "requires authentication and a valid token for protected streams" do capture_log(fn -> - assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = start_socket("?stream=user&access_token=aaaaaaaaaaaa") - assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = start_socket("?stream=user") + assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = + start_socket("?stream=user&access_token=aaaaaaaaaaaa") + + assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = + start_socket("?stream=user") + Process.sleep(30) end) end @@ -102,7 +109,9 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") capture_log(fn -> - assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = start_socket("?stream=user") + assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = + start_socket("?stream=user") + Process.sleep(30) end) end @@ -111,7 +120,9 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") capture_log(fn -> - assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = start_socket("?stream=user:notification") + assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = + start_socket("?stream=user:notification") + Process.sleep(30) end) end diff --git a/test/support/websocket_client.ex b/test/support/websocket_client.ex index afcd0e8c7..43f2854de 100644 --- a/test/support/websocket_client.ex +++ b/test/support/websocket_client.ex @@ -188,8 +188,8 @@ defmodule Pleroma.Integration.WebsocketClient do defp stream_frame(state, frame) do with {:ok, websocket, data} <- Mint.WebSocket.encode(state.websocket, frame), state = put_in(state.websocket, websocket), - {:ok, conn} <- Mint.WebSocket.stream_request_body(state.conn, state.request_ref, data) do - {:ok, put_in(state.conn, conn)} + {:ok, conn} <- Mint.WebSocket.stream_request_body(state.conn, state.request_ref, data) do + {:ok, put_in(state.conn, conn)} else {:error, %Mint.WebSocket{} = websocket, reason} -> {:error, put_in(state.websocket, websocket), reason} From e606b9ab3f5028ddac1f52e855d7f6da4999dbc5 Mon Sep 17 00:00:00 2001 From: duponin Date: Wed, 18 May 2022 20:05:42 +0200 Subject: [PATCH 003/123] add missing extra application to start the SSH BBS --- mix.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 0651781cc..1df92ed53 100644 --- a/mix.exs +++ b/mix.exs @@ -80,7 +80,8 @@ defmodule Pleroma.Mixfile do :quack, :fast_sanitize, :os_mon, - :ssl + :ssl, + :esshd ], included_applications: [:ex_syslogger] ] From 39c47073a3c6fd3da068d5a4c9def18f3847ff32 Mon Sep 17 00:00:00 2001 From: duponin Date: Wed, 18 May 2022 20:06:16 +0200 Subject: [PATCH 004/123] fix Ctrl-c catch on SSH BBS --- lib/pleroma/bbs/handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index a3b623bdf..47f5a920e 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -123,7 +123,7 @@ defmodule Pleroma.BBS.Handler do loop(%{state | counter: state.counter + 1}) - {:error, :interrupted} -> + {:input, ^input, {:error, :interrupted}} -> IO.puts("Caught Ctrl+C...") loop(%{state | counter: state.counter + 1}) From 5086d6d5e9ff68d6a7a82fd3ad6dbc0bad0b599c Mon Sep 17 00:00:00 2001 From: duponin Date: Thu, 19 May 2022 00:56:20 +0200 Subject: [PATCH 005/123] add thread show in BBS frontend --- lib/pleroma/bbs/handler.ex | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index 47f5a920e..f1ac0c687 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -53,6 +53,7 @@ defmodule Pleroma.BBS.Handler do IO.puts("home - Show the home timeline") IO.puts("p - Post the given text") IO.puts("r - Reply to the post with the given id") + IO.puts("t - Show a thread from the given id") IO.puts("quit - Quit") state @@ -73,6 +74,33 @@ defmodule Pleroma.BBS.Handler do state end + def handle_command(%{user: user} = state, "t " <> activity_id) do + with %Activity{} = activity <- Activity.get_by_id(activity_id) do + activities = + ActivityPub.fetch_activities_for_context(activity.data["context"], %{ + blocking_user: user, + user: user, + exclude_id: activity.id + }) + + case activities do + [] -> + activity_id + |> Activity.get_by_id() + |> puts_activity() + + _ -> + activities + |> Enum.reverse() + |> Enum.each(&puts_activity/1) + end + else + _e -> IO.puts("An error occured when trying to show the thread...") + end + + state + end + def handle_command(%{user: user} = state, "p " <> text) do text = String.trim(text) From b128e1d6c5bbc78874d05af2676550de80ae85c7 Mon Sep 17 00:00:00 2001 From: duponin Date: Thu, 19 May 2022 01:38:13 +0200 Subject: [PATCH 006/123] decode HTML to be human readable in BBS --- lib/pleroma/bbs/handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index f1ac0c687..c2491a20c 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -43,7 +43,7 @@ defmodule Pleroma.BBS.Handler do def puts_activity(activity) do status = Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{activity: activity}) IO.puts("-- #{status.id} by #{status.account.display_name} (#{status.account.acct})") - IO.puts(HTML.strip_tags(status.content)) + IO.puts(status.content |> HTML.strip_tags() |> HtmlEntities.decode()) IO.puts("") end From 33ced2c2ed9391ec95aae2205bb30d987ceac86d Mon Sep 17 00:00:00 2001 From: duponin Date: Sat, 21 May 2022 04:17:34 +0200 Subject: [PATCH 007/123] BBS: put a new line for each HTML break in an activity Otherwise it would just put each line on the first one, which is not really readable --- lib/pleroma/bbs/handler.ex | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index c2491a20c..d641de9ac 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -42,9 +42,14 @@ defmodule Pleroma.BBS.Handler do def puts_activity(activity) do status = Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{activity: activity}) + IO.puts("-- #{status.id} by #{status.account.display_name} (#{status.account.acct})") - IO.puts(status.content |> HTML.strip_tags() |> HtmlEntities.decode()) - IO.puts("") + + status.content + |> String.split("
") + |> Enum.map(&HTML.strip_tags/1) + |> Enum.map(&HtmlEntities.decode/1) + |> Enum.map(&IO.puts/1) end def handle_command(state, "help") do From c04c7f9e45eec680afc0bf6c145fa55fc3f56ea8 Mon Sep 17 00:00:00 2001 From: duponin Date: Sat, 21 May 2022 05:10:22 +0200 Subject: [PATCH 008/123] BBS: show notifactions --- lib/pleroma/bbs/handler.ex | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index d641de9ac..e0174efe1 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -52,6 +52,40 @@ defmodule Pleroma.BBS.Handler do |> Enum.map(&IO.puts/1) end + def puts_notification(activity, user) do + notification = + Pleroma.Web.MastodonAPI.NotificationView.render("show.json", %{ + notification: activity, + for: user + }) + + IO.puts( + "== (#{notification.type}) #{notification.status.id} by #{notification.account.display_name} (#{notification.account.acct})" + ) + + notification.status.content + |> String.split("
") + |> Enum.map(&HTML.strip_tags/1) + |> Enum.map(&HtmlEntities.decode/1) + |> (fn x -> + case x do + [content] -> + "> " <> content + + [head | _tail] -> + # "> " <> hd <> "..." + head + |> String.to_charlist() + |> Enum.take(80) + |> List.to_string() + |> (fn x -> "> " <> x <> "..." end).() + end + end).() + |> IO.puts() + + IO.puts("") + end + def handle_command(state, "help") do IO.puts("Available commands:") IO.puts("help - This help") @@ -59,6 +93,7 @@ defmodule Pleroma.BBS.Handler do IO.puts("p - Post the given text") IO.puts("r - Reply to the post with the given id") IO.puts("t - Show a thread from the given id") + IO.puts("n - Show notifications") IO.puts("quit - Quit") state @@ -106,6 +141,14 @@ defmodule Pleroma.BBS.Handler do state end + def handle_command(%{user: user} = state, "n") do + user + |> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(%{}) + |> Enum.each(&puts_notification(&1, user)) + + state + end + def handle_command(%{user: user} = state, "p " <> text) do text = String.trim(text) From e3e8ff06f9c588563003ba9855f2d38b9d6e08b7 Mon Sep 17 00:00:00 2001 From: duponin Date: Sat, 21 May 2022 05:10:48 +0200 Subject: [PATCH 009/123] BBS: mark notification as read --- lib/pleroma/bbs/handler.ex | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index e0174efe1..7314453af 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -94,6 +94,7 @@ defmodule Pleroma.BBS.Handler do IO.puts("r - Reply to the post with the given id") IO.puts("t - Show a thread from the given id") IO.puts("n - Show notifications") + IO.puts("n read - Mark all notifactions as read") IO.puts("quit - Quit") state @@ -141,6 +142,13 @@ defmodule Pleroma.BBS.Handler do state end + def handle_command(%{user: user} = state, "n read") do + Pleroma.Notification.clear(user) + IO.puts("All notifications are marked as read") + + state + end + def handle_command(%{user: user} = state, "n") do user |> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(%{}) From a4659d993d1493406e9df4a26ada35cba50511c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Sat, 21 May 2022 23:23:55 +0000 Subject: [PATCH 010/123] =?UTF-8?q?Apply=20H=C3=A9l=C3=A8ne=20suggestions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/pleroma/bbs/handler.ex | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index 7314453af..a8f2fd37b 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -75,9 +75,7 @@ defmodule Pleroma.BBS.Handler do [head | _tail] -> # "> " <> hd <> "..." head - |> String.to_charlist() - |> Enum.take(80) - |> List.to_string() + |> String.slice(1, 80) |> (fn x -> "> " <> x <> "..." end).() end end).() @@ -136,7 +134,7 @@ defmodule Pleroma.BBS.Handler do |> Enum.each(&puts_activity/1) end else - _e -> IO.puts("An error occured when trying to show the thread...") + _e -> IO.puts("Could not show this thread...") end state @@ -144,7 +142,7 @@ defmodule Pleroma.BBS.Handler do def handle_command(%{user: user} = state, "n read") do Pleroma.Notification.clear(user) - IO.puts("All notifications are marked as read") + IO.puts("All notifications were marked as read") state end From fffd9059d67fb719c38dc014de1fa750dd5be8b4 Mon Sep 17 00:00:00 2001 From: duponin Date: Sun, 22 May 2022 02:39:38 +0200 Subject: [PATCH 011/123] BBS: add post favourite feature --- lib/pleroma/bbs/handler.ex | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index a8f2fd37b..631307f02 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -93,6 +93,7 @@ defmodule Pleroma.BBS.Handler do IO.puts("t - Show a thread from the given id") IO.puts("n - Show notifications") IO.puts("n read - Mark all notifactions as read") + IO.puts("f - Favourites the post with the given id") IO.puts("quit - Quit") state @@ -167,6 +168,19 @@ defmodule Pleroma.BBS.Handler do state end + def handle_command(%{user: user} = state, "f " <> id) do + id = String.trim(id) + + with %Activity{} = activity <- Activity.get_by_id(id), + {:ok, _activity} <- CommonAPI.favorite(user, activity) do + IO.puts("Favourited!") + else + _e -> IO.puts("Could not Favourite...") + end + + state + end + def handle_command(state, "home") do user = state.user From 5951d637a98402ad0e1d11d220c9374fc02d5bcd Mon Sep 17 00:00:00 2001 From: duponin Date: Sun, 22 May 2022 02:40:56 +0200 Subject: [PATCH 012/123] BBS: show post ID when posted --- lib/pleroma/bbs/handler.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index 631307f02..fecabb878 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -159,8 +159,8 @@ defmodule Pleroma.BBS.Handler do def handle_command(%{user: user} = state, "p " <> text) do text = String.trim(text) - with {:ok, _activity} <- CommonAPI.post(user, %{status: text}) do - IO.puts("Posted!") + with {:ok, activity} <- CommonAPI.post(user, %{status: text}) do + IO.puts("Posted! ID: #{activity.id}") else _e -> IO.puts("Could not post...") end From 5ca1ac041f011df458af7ebe057b39c1cc9548d0 Mon Sep 17 00:00:00 2001 From: duponin Date: Sun, 22 May 2022 03:19:24 +0200 Subject: [PATCH 013/123] BBS: add repeat functionality --- lib/pleroma/bbs/handler.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index fecabb878..27799338f 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -94,6 +94,7 @@ defmodule Pleroma.BBS.Handler do IO.puts("n - Show notifications") IO.puts("n read - Mark all notifactions as read") IO.puts("f - Favourites the post with the given id") + IO.puts("R - Repeat the post with the given id") IO.puts("quit - Quit") state From 547def67a76854aa4c9c8438eb1ee4dfa36fd8ac Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 29 May 2022 11:36:00 -0400 Subject: [PATCH 014/123] Allow Updates by every actor on the same origin --- .../object_validators/update_validator.ex | 4 +++- .../update_handling_test.exs | 24 ++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/update_validator.ex b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex index a5def312e..1e940a400 100644 --- a/lib/pleroma/web/activity_pub/object_validators/update_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex @@ -51,7 +51,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator do with actor = get_field(cng, :actor), object = get_field(cng, :object), {:ok, object_id} <- ObjectValidators.ObjectID.cast(object), - true <- actor == object_id do + actor_uri <- URI.parse(actor), + object_uri <- URI.parse(object_id), + true <- actor_uri.host == object_uri.host do cng else _e -> diff --git a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs index 94bc5a89b..f2a22d370 100644 --- a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs @@ -32,7 +32,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateHandlingTest do test "returns an error if the object can't be updated by the actor", %{ valid_update: valid_update } do - other_user = insert(:user) + other_user = insert(:user, local: false) update = valid_update @@ -40,5 +40,27 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateHandlingTest do assert {:error, _cng} = ObjectValidator.validate(update, []) end + + test "validates as long as the object is same-origin with the actor", %{ + valid_update: valid_update + } do + other_user = insert(:user) + + update = + valid_update + |> Map.put("actor", other_user.ap_id) + + assert {:ok, _update, []} = ObjectValidator.validate(update, []) + end + + test "validates if the object is not of an Actor type" do + note = insert(:note) + updated_note = note.data |> Map.put("content", "edited content") + other_user = insert(:user) + + {:ok, update, _} = Builder.update(other_user, updated_note) + + assert {:ok, _update, []} = ObjectValidator.validate(update, []) + end end end From 0f6a5eb9a299629f295372f4d5ecdd9083a19717 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 29 May 2022 12:54:57 -0400 Subject: [PATCH 015/123] Handle Note and Question Updates --- lib/pleroma/web/activity_pub/side_effects.ex | 82 ++++++++++++++++--- .../web/activity_pub/side_effects_test.exs | 23 ++++++ 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index b997c15db..aeddf3ed8 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -153,23 +153,25 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # Tasks this handles: # - Update the user + # - Update a non-user object (Note, Question, etc.) # # For a local user, we also get a changeset with the full information, so we # can update non-federating, non-activitypub settings as well. @impl true def handle(%{data: %{"type" => "Update", "object" => updated_object}} = object, meta) do - if changeset = Keyword.get(meta, :user_update_changeset) do - changeset - |> User.update_and_set_cache() + updated_object_id = updated_object["id"] + + with {_, true} <- {:has_id, is_binary(updated_object_id)}, + {_, user} <- {:user, Pleroma.User.get_by_ap_id(updated_object_id)} do + if user do + handle_update_user(object, meta) + else + handle_update_object(object, meta) + end else - {:ok, new_user_data} = ActivityPub.user_data_from_user_object(updated_object) - - User.get_by_ap_id(updated_object["id"]) - |> User.remote_user_changeset(new_user_data) - |> User.update_and_set_cache() + _ -> + {:ok, object, meta} end - - {:ok, object, meta} end # Tasks this handles: @@ -390,6 +392,66 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do {:ok, object, meta} end + defp handle_update_user( + %{data: %{"type" => "Update", "object" => updated_object}} = object, + meta + ) do + if changeset = Keyword.get(meta, :user_update_changeset) do + changeset + |> User.update_and_set_cache() + else + {:ok, new_user_data} = ActivityPub.user_data_from_user_object(updated_object) + + User.get_by_ap_id(updated_object["id"]) + |> User.remote_user_changeset(new_user_data) + |> User.update_and_set_cache() + end + + {:ok, object, meta} + end + + @updatable_object_types ["Note", "Question"] + # We do not allow poll options to be changed, but the poll description can be. + @updatable_fields [ + "source", + "tag", + "updated", + "emoji", + "content", + "summary", + "sensitive", + "attachment", + "generator" + ] + defp handle_update_object( + %{data: %{"type" => "Update", "object" => updated_object}} = object, + meta + ) do + orig_object = Object.get_by_ap_id(updated_object["id"]) + orig_object_data = orig_object.data + + if orig_object_data["type"] in @updatable_object_types do + updated_object_data = + @updatable_fields + |> Enum.reduce( + orig_object_data, + fn field, acc -> + if Map.has_key?(updated_object, field) do + Map.put(acc, field, updated_object[field]) + else + Map.drop(acc, [field]) + end + end + ) + + orig_object + |> Object.change(%{data: updated_object_data}) + |> Object.update_and_set_cache() + end + + {:ok, object, meta} + end + def handle_object_creation(%{"type" => "ChatMessage"} = object, _activity, meta) do with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do actor = User.get_cached_by_ap_id(object.data["actor"]) diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 64c4a8c14..f72753116 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -140,6 +140,29 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do end end + describe "update notes" do + setup do + user = insert(:user) + note = insert(:note, user: user) + + updated_note = + note.data + |> Map.put("summary", "edited summary") + |> Map.put("content", "edited content") + + {:ok, update_data, []} = Builder.update(user, updated_note) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) + + %{user: user, object_id: note.id, update_data: update_data, update: update} + end + + test "it updates the note", %{object_id: object_id, update: update} do + {:ok, _, _} = SideEffects.handle(update) + new_note = Pleroma.Object.get_by_id(object_id) + assert %{"summary" => "edited summary", "content" => "edited content"} = new_note.data + end + end + describe "EmojiReact objects" do setup do poster = insert(:user) From 5e8aac0e07cf54d527643e9793b92f3c0b3826e2 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 29 May 2022 13:54:16 -0400 Subject: [PATCH 016/123] Record edit history for Note and Question Updates --- lib/pleroma/web/activity_pub/side_effects.ex | 34 +++++++++++++++++ priv/static/schemas/litepub-0.1.jsonld | 3 +- .../web/activity_pub/side_effects_test.exs | 37 ++++++++++++++++++- 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index aeddf3ed8..c4d56fa20 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -410,6 +410,26 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do {:ok, object, meta} end + defp history_for_object(object) do + with history <- Map.get(object, "formerRepresentations"), + true <- is_map(history), + "OrderedCollection" <- Map.get(history, "type"), + true <- is_list(Map.get(history, "orderedItems")), + true <- is_integer(Map.get(history, "totalItems")) do + history + else + _ -> history_skeleton() + end + end + + defp history_skeleton do + %{ + "type" => "OrderedCollection", + "totalItems" => 0, + "orderedItems" => [] + } + end + @updatable_object_types ["Note", "Question"] # We do not allow poll options to be changed, but the poll description can be. @updatable_fields [ @@ -431,6 +451,19 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do orig_object_data = orig_object.data if orig_object_data["type"] in @updatable_object_types do + # Put edit history + # Note that we may have got the edit history by first fetching the object + history = history_for_object(orig_object_data) + + latest_history_item = + orig_object_data + |> Map.drop(["id", "formerRepresentations"]) + + new_history = + history + |> Map.put("orderedItems", [latest_history_item | history["orderedItems"]]) + |> Map.put("totalItems", history["totalItems"] + 1) + updated_object_data = @updatable_fields |> Enum.reduce( @@ -443,6 +476,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end end ) + |> Map.put("formerRepresentations", new_history) orig_object |> Object.change(%{data: updated_object_data}) diff --git a/priv/static/schemas/litepub-0.1.jsonld b/priv/static/schemas/litepub-0.1.jsonld index 946099a6e..650118475 100644 --- a/priv/static/schemas/litepub-0.1.jsonld +++ b/priv/static/schemas/litepub-0.1.jsonld @@ -36,7 +36,8 @@ "@id": "as:alsoKnownAs", "@type": "@id" }, - "vcard": "http://www.w3.org/2006/vcard/ns#" + "vcard": "http://www.w3.org/2006/vcard/ns#", + "formerRepresentations": "litepub:formerRepresentations" } ] } diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index f72753116..5c60504d4 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -153,7 +153,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do {:ok, update_data, []} = Builder.update(user, updated_note) {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) - %{user: user, object_id: note.id, update_data: update_data, update: update} + %{user: user, note: note, object_id: note.id, update_data: update_data, update: update} end test "it updates the note", %{object_id: object_id, update: update} do @@ -161,6 +161,41 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do new_note = Pleroma.Object.get_by_id(object_id) assert %{"summary" => "edited summary", "content" => "edited content"} = new_note.data end + + test "it records the original note in formerRepresentations", %{ + note: note, + object_id: object_id, + update: update + } do + {:ok, _, _} = SideEffects.handle(update) + %{data: new_note} = Pleroma.Object.get_by_id(object_id) + assert %{"summary" => "edited summary", "content" => "edited content"} = new_note + + assert [Map.drop(note.data, ["id", "formerRepresentations"])] == + new_note["formerRepresentations"]["orderedItems"] + + assert new_note["formerRepresentations"]["totalItems"] == 1 + end + + test "it puts the original note at the front of formerRepresentations", %{ + note: note, + object_id: object_id, + update: update + } do + {:ok, _, _} = SideEffects.handle(update) + %{data: first_edit} = Pleroma.Object.get_by_id(object_id) + {:ok, _, _} = SideEffects.handle(update) + %{data: new_note} = Pleroma.Object.get_by_id(object_id) + assert %{"summary" => "edited summary", "content" => "edited content"} = new_note + + original_version = Map.drop(note.data, ["id", "formerRepresentations"]) + first_edit = Map.drop(first_edit, ["id", "formerRepresentations"]) + + assert [first_edit, original_version] == + new_note["formerRepresentations"]["orderedItems"] + + assert new_note["formerRepresentations"]["totalItems"] == 2 + end end describe "EmojiReact objects" do From 8acfe95f3e9d4183fd513cfe828500c852db4d5f Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 29 May 2022 22:16:03 -0400 Subject: [PATCH 017/123] Allow updating polls --- lib/pleroma/web/activity_pub/side_effects.ex | 81 ++++++++++++++----- .../web/activity_pub/side_effects_test.exs | 64 ++++++++++++++- 2 files changed, 125 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index c4d56fa20..ac327280c 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -443,14 +443,29 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do "attachment", "generator" ] - defp handle_update_object( - %{data: %{"type" => "Update", "object" => updated_object}} = object, - meta - ) do - orig_object = Object.get_by_ap_id(updated_object["id"]) - orig_object_data = orig_object.data + defp update_content_fields(orig_object_data, updated_object) do + @updatable_fields + |> Enum.reduce( + %{data: orig_object_data, updated: false}, + fn field, %{data: data, updated: updated} -> + updated = updated or Map.get(updated_object, field) != Map.get(orig_object_data, field) - if orig_object_data["type"] in @updatable_object_types do + data = + if Map.has_key?(updated_object, field) do + Map.put(data, field, updated_object[field]) + else + Map.drop(data, [field]) + end + + %{data: data, updated: updated} + end + ) + end + + defp maybe_update_history(updated_object, orig_object_data, updated) do + if not updated do + updated_object + else # Put edit history # Note that we may have got the edit history by first fetching the object history = history_for_object(orig_object_data) @@ -464,19 +479,47 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do |> Map.put("orderedItems", [latest_history_item | history["orderedItems"]]) |> Map.put("totalItems", history["totalItems"] + 1) + updated_object + |> Map.put("formerRepresentations", new_history) + end + end + + defp maybe_update_poll(to_be_updated, updated_object) do + choice_key = fn data -> + if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf" + end + + with true <- to_be_updated["type"] == "Question", + key <- choice_key.(updated_object), + true <- key == choice_key.(to_be_updated), + orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])), + new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])), + true <- orig_choices == new_choices do + # Choices are the same, but counts are different + to_be_updated + |> Map.put(key, updated_object[key]) + else + # Choices (or vote type) have changed, do not allow this + _ -> to_be_updated + end + end + + defp handle_update_object( + %{data: %{"type" => "Update", "object" => updated_object}} = object, + meta + ) do + orig_object = Object.get_by_ap_id(updated_object["id"]) + orig_object_data = orig_object.data + + if orig_object_data["type"] in @updatable_object_types do + %{data: updated_object_data, updated: updated} = + orig_object_data + |> update_content_fields(updated_object) + updated_object_data = - @updatable_fields - |> Enum.reduce( - orig_object_data, - fn field, acc -> - if Map.has_key?(updated_object, field) do - Map.put(acc, field, updated_object[field]) - else - Map.drop(acc, [field]) - end - end - ) - |> Map.put("formerRepresentations", new_history) + updated_object_data + |> maybe_update_history(orig_object_data, updated) + |> maybe_update_poll(updated_object) orig_object |> Object.change(%{data: updated_object_data}) diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 5c60504d4..62394b058 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -178,15 +178,24 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do end test "it puts the original note at the front of formerRepresentations", %{ + user: user, note: note, object_id: object_id, update: update } do {:ok, _, _} = SideEffects.handle(update) %{data: first_edit} = Pleroma.Object.get_by_id(object_id) + + second_updated_note = + note.data + |> Map.put("summary", "edited summary 2") + |> Map.put("content", "edited content 2") + + {:ok, second_update_data, []} = Builder.update(user, second_updated_note) + {:ok, update, _meta} = ActivityPub.persist(second_update_data, local: true) {:ok, _, _} = SideEffects.handle(update) %{data: new_note} = Pleroma.Object.get_by_id(object_id) - assert %{"summary" => "edited summary", "content" => "edited content"} = new_note + assert %{"summary" => "edited summary 2", "content" => "edited content 2"} = new_note original_version = Map.drop(note.data, ["id", "formerRepresentations"]) first_edit = Map.drop(first_edit, ["id", "formerRepresentations"]) @@ -196,6 +205,59 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do assert new_note["formerRepresentations"]["totalItems"] == 2 end + + test "it does not prepend to formerRepresentations if no actual changes are made", %{ + note: note, + object_id: object_id, + update: update + } do + {:ok, _, _} = SideEffects.handle(update) + %{data: _first_edit} = Pleroma.Object.get_by_id(object_id) + + {:ok, _, _} = SideEffects.handle(update) + %{data: new_note} = Pleroma.Object.get_by_id(object_id) + assert %{"summary" => "edited summary", "content" => "edited content"} = new_note + + original_version = Map.drop(note.data, ["id", "formerRepresentations"]) + + assert [original_version] == + new_note["formerRepresentations"]["orderedItems"] + + assert new_note["formerRepresentations"]["totalItems"] == 1 + end + end + + describe "update questions" do + setup do + user = insert(:user) + question = insert(:question, user: user) + + %{user: user, data: question.data, id: question.id} + end + + test "allows updating choice count without generating edit history", %{ + user: user, + data: data, + id: id + } do + new_choices = + data["oneOf"] + |> Enum.map(fn choice -> put_in(choice, ["replies", "totalItems"], 5) end) + + updated_question = data |> Map.put("oneOf", new_choices) + + {:ok, update_data, []} = Builder.update(user, updated_question) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) + + {:ok, _, _} = SideEffects.handle(update) + + %{data: new_question} = Pleroma.Object.get_by_id(id) + + assert [%{"replies" => %{"totalItems" => 5}}, %{"replies" => %{"totalItems" => 5}}] = + new_question["oneOf"] + + refute Map.has_key?(new_question, "formerRepresentations") + end end describe "EmojiReact objects" do From c004eb0fa2c0a754a0fb839a961e35f406c57445 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 29 May 2022 23:50:31 -0400 Subject: [PATCH 018/123] Implement mastodon api for showing edit history --- lib/pleroma/object.ex | 20 +++++ lib/pleroma/web/activity_pub/side_effects.ex | 22 +---- .../api_spec/operations/status_operation.ex | 82 +++++++++++++++++++ .../controllers/status_controller.ex | 30 ++++++- .../web/mastodon_api/views/status_view.ex | 65 +++++++++++++++ lib/pleroma/web/router.ex | 3 + .../controllers/status_controller_test.exs | 46 +++++++++++ 7 files changed, 245 insertions(+), 23 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index fe264b5e0..a893f2c1a 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -425,4 +425,24 @@ defmodule Pleroma.Object do end def object_data_hashtags(_), do: [] + + def history_for(object) do + with history <- Map.get(object, "formerRepresentations"), + true <- is_map(history), + "OrderedCollection" <- Map.get(history, "type"), + true <- is_list(Map.get(history, "orderedItems")), + true <- is_integer(Map.get(history, "totalItems")) do + history + else + _ -> history_skeleton() + end + end + + defp history_skeleton do + %{ + "type" => "OrderedCollection", + "totalItems" => 0, + "orderedItems" => [] + } + end end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index ac327280c..894c0ceef 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -410,26 +410,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do {:ok, object, meta} end - defp history_for_object(object) do - with history <- Map.get(object, "formerRepresentations"), - true <- is_map(history), - "OrderedCollection" <- Map.get(history, "type"), - true <- is_list(Map.get(history, "orderedItems")), - true <- is_integer(Map.get(history, "totalItems")) do - history - else - _ -> history_skeleton() - end - end - - defp history_skeleton do - %{ - "type" => "OrderedCollection", - "totalItems" => 0, - "orderedItems" => [] - } - end - @updatable_object_types ["Note", "Question"] # We do not allow poll options to be changed, but the poll description can be. @updatable_fields [ @@ -468,7 +448,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do else # Put edit history # Note that we may have got the edit history by first fetching the object - history = history_for_object(orig_object_data) + history = Object.history_for(orig_object_data) latest_history_item = orig_object_data diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 639f24d49..e5322707f 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -6,9 +6,13 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do alias OpenApiSpex.Operation alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.AccountOperation + alias Pleroma.Web.ApiSpec.Schemas.Account alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.Attachment alias Pleroma.Web.ApiSpec.Schemas.BooleanLike + alias Pleroma.Web.ApiSpec.Schemas.Emoji alias Pleroma.Web.ApiSpec.Schemas.FlakeID + alias Pleroma.Web.ApiSpec.Schemas.Poll alias Pleroma.Web.ApiSpec.Schemas.ScheduledStatus alias Pleroma.Web.ApiSpec.Schemas.Status alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope @@ -434,6 +438,29 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do } end + def show_history_operation do + %Operation{ + tags: ["Retrieve status history"], + summary: "Status history", + description: "View history of a status", + operationId: "StatusController.show_history", + security: [%{"oAuth" => ["read:statuses"]}], + parameters: [ + id_param() + ], + responses: %{ + 200 => status_history_response(), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } + end + + def show_source_operation do + end + + def update_operation do + end + def array_of_statuses do %Schema{type: :array, items: Status, example: [Status.schema().example]} end @@ -579,6 +606,61 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do Operation.response("Status", "application/json", Status) end + defp status_history_response do + Operation.response( + "Status History", + "application/json", + %Schema{ + title: "Status history", + description: "Response schema for history of a status", + type: :array, + items: %Schema{ + type: :object, + properties: %{ + account: %Schema{ + allOf: [Account], + description: "The account that authored this status" + }, + content: %Schema{ + type: :string, + format: :html, + description: "HTML-encoded status content" + }, + sensitive: %Schema{ + type: :boolean, + description: "Is this status marked as sensitive content?" + }, + spoiler_text: %Schema{ + type: :string, + description: + "Subject or summary line, below which status content is collapsed until expanded" + }, + created_at: %Schema{ + type: :string, + format: "date-time", + description: "The date when this status was created" + }, + media_attachments: %Schema{ + type: :array, + items: Attachment, + description: "Media that is attached to this status" + }, + emojis: %Schema{ + type: :array, + items: Emoji, + description: "Custom emoji to be used when rendering status content" + }, + poll: %Schema{ + allOf: [Poll], + nullable: true, + description: "The poll attached to the status" + } + } + } + } + ) + end + defp context do %Schema{ title: "StatusContext", diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 42a95bdc5..72d85f1ec 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -38,7 +38,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do :index, :show, :card, - :context + :context, + :show_history, + :show_source ] ) @@ -49,7 +51,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do :create, :delete, :reblog, - :unreblog + :unreblog, + :update ] ) @@ -191,6 +194,29 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do create(%Plug.Conn{conn | body_params: params}, %{}) end + @doc "GET /api/v1/statuses/:id/history" + def show_history(%{assigns: %{user: user}} = conn, %{id: id} = params) do + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + true <- Visibility.visible_for_user?(activity, user) do + try_render(conn, "history.json", + activity: activity, + for: user, + with_direct_conversation_id: true, + with_muted: Map.get(params, :with_muted, false) + ) + else + _ -> {:error, :not_found} + end + end + + @doc "GET /api/v1/statuses/:id/source" + def show_source(%{assigns: %{user: _user}} = _conn, %{id: _id} = _params) do + end + + @doc "PUT /api/v1/statuses/:id" + def update(%{assigns: %{user: _user}} = _conn, %{id: _id} = _params) do + end + @doc "GET /api/v1/statuses/:id" def show(%{assigns: %{user: user}} = conn, %{id: id} = params) do with %Activity{} = activity <- Activity.get_by_id_with_object(id), diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 1ebfd6740..c50e0d3da 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -384,6 +384,71 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do nil end + def render("history.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do + object = Object.normalize(activity, fetch: false) + + hashtags = Object.hashtags(object) + + user = CommonAPI.get_user(activity.data["actor"]) + + past_history = + Object.history_for(object.data) + |> Map.get("orderedItems") + |> Enum.map(&Map.put(&1, "id", object.data["id"])) + |> Enum.map(&%Object{data: &1, id: object.id}) + + history = [object | past_history] + + individual_opts = + opts + |> Map.put(:as, :object) + |> Map.put(:user, user) + |> Map.put(:hashtags, hashtags) + + render_many(history, StatusView, "history_item.json", individual_opts) + end + + def render( + "history_item.json", + %{activity: activity, user: user, object: object, hashtags: hashtags} = opts + ) do + sensitive = object.data["sensitive"] || Enum.member?(hashtags, "nsfw") + + attachment_data = object.data["attachment"] || [] + attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment) + + created_at = Utils.to_masto_date(object.data["updated"] || object.data["published"]) + + content = + object + |> render_content() + + content_html = + content + |> Activity.HTML.get_cached_scrubbed_html_for_activity( + User.html_filter_policy(opts[:for]), + activity, + "mastoapi:content" + ) + + summary = object.data["summary"] || "" + + %{ + account: + AccountView.render("show.json", %{ + user: user, + for: opts[:for] + }), + content: content_html, + sensitive: sensitive, + spoiler_text: summary, + created_at: created_at, + media_attachments: attachments, + emojis: build_emojis(object.data["emoji"]), + poll: render(PollView, "show.json", object: object, for: opts[:for]) + } + end + def render("card.json", %{rich_media: rich_media, page_url: page_url}) do page_url_data = URI.parse(page_url) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ceb6c3cfd..2d2e5365e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -552,6 +552,9 @@ defmodule Pleroma.Web.Router do get("/bookmarks", StatusController, :bookmarks) post("/statuses", StatusController, :create) + get("/statuses/:id/history", StatusController, :show_history) + get("/statuses/:id/source", StatusController, :show_source) + put("/statuses/:id", StatusController, :update) delete("/statuses/:id", StatusController, :delete) post("/statuses/:id/reblog", StatusController, :reblog) post("/statuses/:id/unreblog", StatusController, :unreblog) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index dc6912b7b..d98dc0a92 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -1990,4 +1990,50 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do } = result end end + + describe "get status history" do + setup do + oauth_access(["read:statuses"]) + end + + test "unedited post", %{conn: conn} do + activity = insert(:note_activity) + + conn = get(conn, "/api/v1/statuses/#{activity.id}/history") + + assert [_] = json_response_and_validate_schema(conn, 200) + end + + test "edited post", %{conn: conn} do + note = + insert( + :note, + data: %{ + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "mew mew 2", + "summary" => "title 2" + }, + %{ + "type" => "Note", + "content" => "mew mew 1", + "summary" => "title 1" + } + ], + "totalItems" => 2 + } + } + ) + + activity = insert(:note_activity, note: note) + + conn = get(conn, "/api/v1/statuses/#{activity.id}/history") + + assert [_, %{"spoiler_text" => "title 2"}, %{"spoiler_text" => "title 1"}] = + json_response_and_validate_schema(conn, 200) + end + end end From 393b50884607f9aca4d6e08bf429c8fe8f426f96 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 30 May 2022 00:59:23 -0400 Subject: [PATCH 019/123] Implement viewing source --- .../api_spec/operations/status_operation.ex | 36 +++++++++++++++++++ .../controllers/status_controller.ex | 11 +++++- .../web/mastodon_api/views/status_view.ex | 10 ++++++ .../controllers/status_controller_test.exs | 18 ++++++++++ 4 files changed, 74 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index e5322707f..617aba460 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -456,6 +456,20 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do end def show_source_operation do + %Operation{ + tags: ["Retrieve status source"], + summary: "Status source", + description: "View source of a status", + operationId: "StatusController.show_source", + security: [%{"oAuth" => ["read:statuses"]}], + parameters: [ + id_param() + ], + responses: %{ + 200 => status_source_response(), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } end def update_operation do @@ -661,6 +675,28 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do ) end + defp status_source_response do + Operation.response( + "Status Source", + "application/json", + %Schema{ + type: :object, + properties: %{ + id: FlakeID, + text: %Schema{ + type: :string, + description: "Raw source of status content" + }, + spoiler_text: %Schema{ + type: :string, + description: + "Subject or summary line, below which status content is collapsed until expanded" + } + } + } + ) + end + defp context do %Schema{ title: "StatusContext", diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 72d85f1ec..ea9e08aa8 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -210,7 +210,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do end @doc "GET /api/v1/statuses/:id/source" - def show_source(%{assigns: %{user: _user}} = _conn, %{id: _id} = _params) do + def show_source(%{assigns: %{user: user}} = conn, %{id: id} = _params) do + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + true <- Visibility.visible_for_user?(activity, user) do + try_render(conn, "source.json", + activity: activity, + for: user + ) + else + _ -> {:error, :not_found} + end end @doc "PUT /api/v1/statuses/:id" diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index c50e0d3da..8d4685ffa 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -449,6 +449,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do } end + def render("source.json", %{activity: %{data: %{"object" => _object}} = activity} = _opts) do + object = Object.normalize(activity, fetch: false) + + %{ + id: activity.id, + text: Map.get(object.data, "source", ""), + spoiler_text: Map.get(object.data, "summary", "") + } + end + def render("card.json", %{rich_media: rich_media, page_url: page_url}) do page_url_data = URI.parse(page_url) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index d98dc0a92..f27cf5048 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2036,4 +2036,22 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do json_response_and_validate_schema(conn, 200) end end + + describe "get status source" do + setup do + oauth_access(["read:statuses"]) + end + + test "it returns the source", %{conn: conn} do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "mew mew #abc", spoiler_text: "#def"}) + + conn = get(conn, "/api/v1/statuses/#{activity.id}/source") + + id = activity.id + + assert %{"id" => ^id, "text" => "mew mew #abc", "spoiler_text" => "#def"} = json_response_and_validate_schema(conn, 200) + end + end end From b613a9ec6b68972c81dfe2f0175572bc7bd547f9 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 31 May 2022 14:29:12 -0400 Subject: [PATCH 020/123] Implement mastodon api for editing status --- lib/pleroma/constants.ex | 24 ++++++ lib/pleroma/web/activity_pub/builder.ex | 13 ++- lib/pleroma/web/activity_pub/side_effects.ex | 16 +--- .../api_spec/operations/status_operation.ex | 72 ++++++++++++++++- lib/pleroma/web/common_api.ex | 36 +++++++++ .../controllers/status_controller.ex | 21 ++++- test/pleroma/web/common_api_test.exs | 29 +++++++ .../controllers/status_controller_test.exs | 79 ++++++++++++++++++- 8 files changed, 271 insertions(+), 19 deletions(-) diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index a42c71d23..bbb95104f 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -27,4 +27,28 @@ defmodule Pleroma.Constants do do: ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css) ) + + const(status_updatable_fields, + do: [ + "source", + "tag", + "updated", + "emoji", + "content", + "summary", + "sensitive", + "attachment", + "generator" + ] + ) + + const(actor_types, + do: [ + "Application", + "Group", + "Organization", + "Person", + "Service" + ] + ) end diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index 5b25138a4..532047599 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -218,10 +218,16 @@ defmodule Pleroma.Web.ActivityPub.Builder do end end - # Retricted to user updates for now, always public @spec update(User.t(), Object.t()) :: {:ok, map(), keyword()} def update(actor, object) do - to = [Pleroma.Constants.as_public(), actor.follower_address] + {to, cc} = + if object["type"] in Pleroma.Constants.actor_types() do + # User updates, always public + {[Pleroma.Constants.as_public(), actor.follower_address], []} + else + # Status updates, follow the recipients in the object + {object["to"] || [], object["cc"] || []} + end {:ok, %{ @@ -229,7 +235,8 @@ defmodule Pleroma.Web.ActivityPub.Builder do "type" => "Update", "actor" => actor.ap_id, "object" => object, - "to" => to + "to" => to, + "cc" => cc }, []} end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 894c0ceef..49054c320 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -25,6 +25,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do alias Pleroma.Web.Streamer alias Pleroma.Workers.PollWorker + require Pleroma.Constants require Logger @cachex Pleroma.Config.get([:cachex, :provider], Cachex) @@ -411,20 +412,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end @updatable_object_types ["Note", "Question"] - # We do not allow poll options to be changed, but the poll description can be. - @updatable_fields [ - "source", - "tag", - "updated", - "emoji", - "content", - "summary", - "sensitive", - "attachment", - "generator" - ] defp update_content_fields(orig_object_data, updated_object) do - @updatable_fields + Pleroma.Constants.status_updatable_fields() |> Enum.reduce( %{data: orig_object_data, updated: false}, fn field, %{data: data, updated: updated} -> @@ -502,6 +491,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do |> maybe_update_poll(updated_object) orig_object + |> Repo.preload(:hashtags) |> Object.change(%{data: updated_object_data}) |> Object.update_and_set_cache() end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 617aba460..c69307a4d 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -473,6 +473,22 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do end def update_operation do + %Operation{ + tags: ["Update status"], + summary: "Update status", + description: "Change the content of a status", + operationId: "StatusController.update", + security: [%{"oAuth" => ["write:statuses"]}], + parameters: [ + id_param() + ], + requestBody: request_body("Parameters", update_request(), required: true), + responses: %{ + 200 => status_response(), + 403 => Operation.response("Forbidden", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } end def array_of_statuses do @@ -578,6 +594,60 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do } end + defp update_request do + %Schema{ + title: "StatusUpdateRequest", + type: :object, + properties: %{ + status: %Schema{ + type: :string, + nullable: true, + description: + "Text content of the status. If `media_ids` is provided, this becomes optional. Attaching a `poll` is optional while `status` is provided." + }, + media_ids: %Schema{ + nullable: true, + type: :array, + items: %Schema{type: :string}, + description: "Array of Attachment ids to be attached as media." + }, + poll: poll_params(), + sensitive: %Schema{ + allOf: [BooleanLike], + nullable: true, + description: "Mark status and attached media as sensitive?" + }, + spoiler_text: %Schema{ + type: :string, + nullable: true, + description: + "Text to be shown as a warning or subject before the actual content. Statuses are generally collapsed behind this field." + }, + content_type: %Schema{ + type: :string, + nullable: true, + description: + "The MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint." + }, + to: %Schema{ + type: :array, + nullable: true, + items: %Schema{type: :string}, + description: + "A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply" + } + }, + example: %{ + "status" => "What time is it?", + "sensitive" => "false", + "poll" => %{ + "options" => ["Cofe", "Adventure"], + "expires_in" => 420 + } + } + } + end + def poll_params do %Schema{ nullable: true, @@ -690,7 +760,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do spoiler_text: %Schema{ type: :string, description: - "Subject or summary line, below which status content is collapsed until expanded" + "Subject or summary line, below which status content is collapsed until expanded" } } } diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 1b95ee89c..e60c26053 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -402,6 +402,42 @@ defmodule Pleroma.Web.CommonAPI do end end + def update(user, orig_activity, changes) do + with orig_object <- Object.normalize(orig_activity), + {:ok, new_object} <- make_update_data(user, orig_object, changes), + {:ok, update_data, _} <- Builder.update(user, new_object), + {:ok, update, _} <- Pipeline.common_pipeline(update_data, local: true) do + {:ok, update} + else + _ -> {:error, nil} + end + end + + defp make_update_data(user, orig_object, changes) do + kept_params = %{ + visibility: Visibility.get_visibility(orig_object) + } + + params = Map.merge(changes, kept_params) + + with {:ok, draft} <- ActivityDraft.create(user, params) do + change = + Pleroma.Constants.status_updatable_fields() + |> Enum.reduce(orig_object.data, fn key, acc -> + if Map.has_key?(draft.object, key) do + acc |> Map.put(key, Map.get(draft.object, key)) + else + acc |> Map.drop([key]) + end + end) + |> Map.put("updated", Utils.make_date()) + + {:ok, change} + else + _ -> {:error, nil} + end + end + @spec pin(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, term()} def pin(id, %User{} = user) do with %Activity{} = activity <- create_activity_by_id(id), diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index ea9e08aa8..fa86e9dc0 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -223,7 +223,26 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do end @doc "PUT /api/v1/statuses/:id" - def update(%{assigns: %{user: _user}} = _conn, %{id: _id} = _params) do + def update(%{assigns: %{user: user}, body_params: body_params} = conn, %{id: id} = params) do + with {_, %Activity{}} = {_, activity} <- {:activity, Activity.get_by_id_with_object(id)}, + {_, true} <- {:visible, Visibility.visible_for_user?(activity, user)}, + {_, true} <- {:is_create, activity.data["type"] == "Create"}, + actor <- Activity.user_actor(activity), + {_, true} <- {:own_status, actor.id == user.id}, + changes <- body_params |> put_application(conn), + {_, {:ok, _update_activity}} <- {:pipeline, CommonAPI.update(user, activity, changes)}, + {_, %Activity{}} = {_, activity} <- {:refetched, Activity.get_by_id_with_object(id)} do + try_render(conn, "show.json", + activity: activity, + for: user, + with_direct_conversation_id: true, + with_muted: Map.get(params, :with_muted, false) + ) + else + {:own_status, _} -> {:error, :forbidden} + {:pipeline, _} -> {:error, :internal_server_error} + _ -> {:error, :not_found} + end end @doc "GET /api/v1/statuses/:id" diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index b502aaa03..af91fdf74 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1541,4 +1541,33 @@ defmodule Pleroma.Web.CommonAPITest do end end end + + describe "update/3" do + test "updates a post" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{status: "foo1", spoiler_text: "title 1"}) + + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2"}) + + updated_object = Object.normalize(updated) + assert updated_object.data["content"] == "updated 2" + assert Map.get(updated_object.data, "summary", "") == "" + assert Map.has_key?(updated_object.data, "updated") + end + + test "does not change visibility" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{status: "foo1", spoiler_text: "title 1", visibility: "private"}) + + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2"}) + + updated_object = Object.normalize(updated) + assert updated_object.data["content"] == "updated 2" + assert Map.get(updated_object.data, "summary", "") == "" + assert Visibility.get_visibility(updated_object) == "private" + assert Visibility.get_visibility(updated) == "private" + end + end end diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index f27cf5048..d4ca2d618 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2051,7 +2051,84 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do id = activity.id - assert %{"id" => ^id, "text" => "mew mew #abc", "spoiler_text" => "#def"} = json_response_and_validate_schema(conn, 200) + assert %{"id" => ^id, "text" => "mew mew #abc", "spoiler_text" => "#def"} = + json_response_and_validate_schema(conn, 200) + end + end + + describe "update status" do + setup do + oauth_access(["write:statuses"]) + end + + test "it updates the status", %{conn: conn, user: user} do + {:ok, activity} = CommonAPI.post(user, %{status: "mew mew #abc", spoiler_text: "#def"}) + + response = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "edited", + "spoiler_text" => "lol" + }) + |> json_response_and_validate_schema(200) + + assert response["content"] == "edited" + assert response["spoiler_text"] == "lol" + end + + test "it does not update visibility", %{conn: conn, user: user} do + {:ok, activity} = + CommonAPI.post(user, %{ + status: "mew mew #abc", + spoiler_text: "#def", + visibility: "private" + }) + + response = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "edited", + "spoiler_text" => "lol" + }) + |> json_response_and_validate_schema(200) + + assert response["visibility"] == "private" + end + + test "it refuses to update when original post is not by the user", %{conn: conn} do + another_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(another_user, %{status: "mew mew #abc", spoiler_text: "#def"}) + + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "edited", + "spoiler_text" => "lol" + }) + |> json_response_and_validate_schema(:forbidden) + end + + test "it returns 404 if the user cannot see the post", %{conn: conn} do + another_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(another_user, %{ + status: "mew mew #abc", + spoiler_text: "#def", + visibility: "private" + }) + + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "edited", + "spoiler_text" => "lol" + }) + |> json_response_and_validate_schema(:not_found) end end end From 410e177b2ac3177f0645d7728b2ea922ba3c24d3 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 1 Jun 2022 12:02:03 -0400 Subject: [PATCH 021/123] Strip internal fields in formerRepresentation --- .../web/activity_pub/transmogrifier.ex | 19 ++++++- .../web/activity_pub/transmogrifier_test.exs | 54 +++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index a70330f0e..5750396a4 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -902,7 +902,24 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def strip_internal_fields(object) do - Map.drop(object, Pleroma.Constants.object_internal_fields()) + outer = Map.drop(object, Pleroma.Constants.object_internal_fields()) + + case outer do + %{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> + update_in( + outer["formerRepresentations"]["orderedItems"], + &Enum.map( + &1, + fn + item when is_map(item) -> Map.drop(item, Pleroma.Constants.object_internal_fields()) + item -> item + end + ) + ) + + _ -> + outer + end end defp strip_internal_tags(%{"tag" => tags} = object) do diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index 335fe1a30..dae07cf21 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -575,4 +575,58 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert Transmogrifier.fix_attachments(object) == expected end end + + describe "strip_internal_fields/1" do + test "it strips internal fields in formerRepresentations" do + original = %{ + "formerRepresentations" => %{ + "orderedItems" => [ + %{"generator" => %{}} + ] + } + } + + stripped = Transmogrifier.strip_internal_fields(original) + + refute Map.has_key?( + Enum.at(stripped["formerRepresentations"]["orderedItems"], 0), + "generator" + ) + end + + test "it strips internal fields in maybe badly-formed formerRepresentations" do + original = %{ + "formerRepresentations" => %{ + "orderedItems" => [ + %{"generator" => %{}}, + "https://example.com/1" + ] + } + } + + stripped = Transmogrifier.strip_internal_fields(original) + + refute Map.has_key?( + Enum.at(stripped["formerRepresentations"]["orderedItems"], 0), + "generator" + ) + + assert Enum.at(stripped["formerRepresentations"]["orderedItems"], 1) == + "https://example.com/1" + end + + test "it ignores if formerRepresentations does not look like an OrderedCollection" do + original = %{ + "formerRepresentations" => %{ + "items" => [ + %{"generator" => %{}} + ] + } + } + + stripped = Transmogrifier.strip_internal_fields(original) + + assert Map.has_key?(Enum.at(stripped["formerRepresentations"]["items"], 0), "generator") + end + end end From fa31ae50e6ec44a3921a60d2a6c19e864f0511e7 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 1 Jun 2022 19:30:50 -0400 Subject: [PATCH 022/123] Inject history when object is refetched --- lib/pleroma/object.ex | 22 +++ lib/pleroma/object/fetcher.ex | 27 +++ lib/pleroma/web/activity_pub/side_effects.ex | 24 +-- test/pleroma/object/fetcher_test.exs | 187 +++++++++++++++++++ 4 files changed, 237 insertions(+), 23 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index a893f2c1a..670ab8743 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -445,4 +445,26 @@ defmodule Pleroma.Object do "orderedItems" => [] } end + + def maybe_update_history(updated_object, orig_object_data, updated) do + if not updated do + updated_object + else + # Put edit history + # Note that we may have got the edit history by first fetching the object + history = Object.history_for(orig_object_data) + + latest_history_item = + orig_object_data + |> Map.drop(["id", "formerRepresentations"]) + + new_history = + history + |> Map.put("orderedItems", [latest_history_item | history["orderedItems"]]) + |> Map.put("totalItems", history["totalItems"] + 1) + + updated_object + |> Map.put("formerRepresentations", new_history) + end + end end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index deb3dc711..ce816c1fc 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -26,8 +26,35 @@ defmodule Pleroma.Object.Fetcher do end defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do + has_history? = fn + %{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> true + _ -> false + end + internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields()) + remote_history_exists? = has_history?.(new_data) + + # If the remote history exists, we treat that as the only source of truth. + new_data = + if has_history?.(old_data) and not remote_history_exists? do + Map.put(new_data, "formerRepresentations", old_data["formerRepresentations"]) + else + new_data + end + + # If the remote does not have history information, we need to manage it ourselves + new_data = + if not remote_history_exists? do + changed? = + Pleroma.Constants.status_updatable_fields() + |> Enum.any?(fn field -> Map.get(old_data, field) != Map.get(new_data, field) end) + + new_data |> Object.maybe_update_history(old_data, changed?) + else + new_data + end + Map.merge(new_data, internal_fields) end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 49054c320..52a343de7 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -431,28 +431,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do ) end - defp maybe_update_history(updated_object, orig_object_data, updated) do - if not updated do - updated_object - else - # Put edit history - # Note that we may have got the edit history by first fetching the object - history = Object.history_for(orig_object_data) - - latest_history_item = - orig_object_data - |> Map.drop(["id", "formerRepresentations"]) - - new_history = - history - |> Map.put("orderedItems", [latest_history_item | history["orderedItems"]]) - |> Map.put("totalItems", history["totalItems"] + 1) - - updated_object - |> Map.put("formerRepresentations", new_history) - end - end - defp maybe_update_poll(to_be_updated, updated_object) do choice_key = fn data -> if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf" @@ -487,7 +465,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do updated_object_data = updated_object_data - |> maybe_update_history(orig_object_data, updated) + |> Object.maybe_update_history(orig_object_data, updated) |> maybe_update_poll(updated_object) orig_object diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index 98130f434..5a79e064f 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -269,4 +269,191 @@ defmodule Pleroma.Object.FetcherTest do refute called(Pleroma.Signature.sign(:_, :_)) end end + + describe "refetching" do + setup do + object1 = %{ + "id" => "https://mastodon.social/1", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "type" => "Note", + "content" => "test 1", + "bcc" => [], + "bto" => [], + "cc" => [], + "to" => [], + "summary" => "" + } + + object2 = %{ + "id" => "https://mastodon.social/2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "type" => "Note", + "content" => "test 2", + "bcc" => [], + "bto" => [], + "cc" => [], + "to" => [], + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "orig 2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "bcc" => [], + "bto" => [], + "cc" => [], + "to" => [], + "summary" => "" + } + ], + "totalItems" => 1 + } + } + + mock(fn + %{ + method: :get, + url: "https://mastodon.social/1" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: Jason.encode!(object1) + } + + %{ + method: :get, + url: "https://mastodon.social/2" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: Jason.encode!(object2) + } + + %{ + method: :get, + url: "https://mastodon.social/users/emelie/collections/featured" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: + Jason.encode!(%{ + "id" => "https://mastodon.social/users/emelie/collections/featured", + "type" => "OrderedCollection", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "orderedItems" => [], + "totalItems" => 0 + }) + } + + env -> + apply(HttpRequestMock, :request, [env]) + end) + + %{object1: object1, object2: object2} + end + + test "it keeps formerRepresentations if remote does not have this attr", %{object1: object1} do + full_object1 = + object1 + |> Map.merge(%{ + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "orig 2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "bcc" => [], + "bto" => [], + "cc" => [], + "to" => [], + "summary" => "" + } + ], + "totalItems" => 1 + } + }) + + {:ok, o} = Object.create(full_object1) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{"formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]}} = + refetched.data + end + + test "it uses formerRepresentations from remote if possible", %{object2: object2} do + {:ok, o} = Object.create(object2) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{"formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]}} = + refetched.data + end + + test "it replaces formerRepresentations with the one from remote", %{object2: object2} do + full_object2 = + object2 + |> Map.merge(%{ + "content" => "mew mew #def", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"type" => "Note", "content" => "mew mew 2"} + ], + "totalItems" => 1 + } + }) + + {:ok, o} = Object.create(full_object2) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{ + "content" => "test 2", + "formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]} + } = refetched.data + end + + test "it adds to formerRepresentations if the remote does not have one and the object has changed", + %{object1: object1} do + full_object1 = + object1 + |> Map.merge(%{ + "content" => "mew mew #def", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"type" => "Note", "content" => "mew mew 1"} + ], + "totalItems" => 1 + } + }) + + {:ok, o} = Object.create(full_object1) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{ + "content" => "test 1", + "formerRepresentations" => %{ + "orderedItems" => [ + %{"content" => "mew mew #def"}, + %{"content" => "mew mew 1"} + ], + "totalItems" => 2 + } + } = refetched.data + end + end end From 8bac8147d4079c0ba0a54753bbab904e46dadbfc Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 3 Jun 2022 21:15:17 -0400 Subject: [PATCH 023/123] Stream out edits --- lib/pleroma/web/activity_pub/activity_pub.ex | 13 ++++++-- lib/pleroma/web/activity_pub/side_effects.ex | 6 ++++ lib/pleroma/web/streamer.ex | 14 +++++++++ lib/pleroma/web/views/streamer_view.ex | 31 ++++++++++++++++++ test/pleroma/web/streamer_test.exs | 33 ++++++++++++++++++++ 5 files changed, 95 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 064f93b22..179e6763b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -190,7 +190,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def notify_and_stream(activity) do Notification.create_notifications(activity) - conversation = create_or_bump_conversation(activity, activity.actor) + original_activity = + case activity do + %{data: %{"type" => "Update"}, object: %{data: %{"id" => id}}} -> + Activity.get_create_by_object_ap_id_with_object(id) + + _ -> + activity + end + + conversation = create_or_bump_conversation(original_activity, original_activity.actor) participations = get_participations(conversation) stream_out(activity) stream_out_participations(participations) @@ -256,7 +265,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do @impl true def stream_out(%Activity{data: %{"type" => data_type}} = activity) - when data_type in ["Create", "Announce", "Delete"] do + when data_type in ["Create", "Announce", "Delete", "Update"] do activity |> Topics.get_activity_topics() |> Streamer.stream(activity) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 52a343de7..05f9b9bd9 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -472,6 +472,12 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do |> Repo.preload(:hashtags) |> Object.change(%{data: updated_object_data}) |> Object.update_and_set_cache() + + if updated do + object + |> Activity.normalize() + |> ActivityPub.notify_and_stream() + end end {:ok, object, meta} diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index ff7f62a1e..8b7fb985b 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -296,6 +296,20 @@ defmodule Pleroma.Web.Streamer do defp push_to_socket(_topic, %Activity{data: %{"type" => "Delete"}}), do: :noop + defp push_to_socket(topic, %Activity{data: %{"type" => "Update"}} = item) do + anon_render = StreamerView.render("status_update.json", item) + + Registry.dispatch(@registry, topic, fn list -> + Enum.each(list, fn {pid, auth?} -> + if auth? do + send(pid, {:render_with_user, StreamerView, "status_update.json", item}) + else + send(pid, {:text, anon_render}) + end + end) + end) + end + defp push_to_socket(topic, item) do anon_render = StreamerView.render("update.json", item) diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index 16c2b7d61..797762d90 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -25,6 +25,22 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end + def render("status_update.json", %Activity{} = activity, %User{} = user) do + activity = Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) + + %{ + event: "status.update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "show.json", + activity: activity, + for: user + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + def render("notification.json", %Notification{} = notify, %User{} = user) do %{ event: "notification", @@ -51,6 +67,21 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end + def render("status_update.json", %Activity{} = activity) do + activity = Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) + + %{ + event: "status.update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "show.json", + activity: activity + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + def render("chat_update.json", %{chat_message_reference: cm_ref}) do # Explicitly giving the cmr for the object here, so we don't accidentally # send a later 'last_message' that was inserted between inserting this and diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 4d4fed070..5ceaf763f 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -442,6 +442,20 @@ defmodule Pleroma.Web.StreamerTest do "state" => "follow_accept" } = Jason.decode!(payload) end + + test "it streams edits in the 'user' stream", %{user: user, token: oauth_token} do + sender = insert(:user) + {:ok, _, _, _} = CommonAPI.follow(user, sender) + + {:ok, activity} = CommonAPI.post(sender, %{status: "hey"}) + + Streamer.get_topic_and_add_socket("user", user, oauth_token) + {:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew"}) + edited = Pleroma.Activity.normalize(edited) + + assert_receive {:render_with_user, _, "status_update.json", ^edited} + refute Streamer.filtered_by_user?(user, edited) + end end describe "public streams" do @@ -484,6 +498,25 @@ defmodule Pleroma.Web.StreamerTest do assert_receive {:text, event} assert %{"event" => "delete", "payload" => ^activity_id} = Jason.decode!(event) end + + test "it streams edits in the 'public' stream" do + sender = insert(:user) + + Streamer.get_topic_and_add_socket("public", nil, nil) + {:ok, activity} = CommonAPI.post(sender, %{status: "hey"}) + assert_receive {:text, _} + + {:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew"}) + + edited = Pleroma.Activity.normalize(edited) + + %{id: activity_id} = Pleroma.Activity.get_create_by_object_ap_id(edited.object.data["id"]) + + assert_receive {:text, event} + assert %{"event" => "status.update", "payload" => payload} = Jason.decode!(event) + assert %{"id" => ^activity_id} = Jason.decode!(payload) + refute Streamer.filtered_by_user?(sender, edited) + end end describe "thread_containment/2" do From fdaa8640838c741500839f66fc7902c88d449afd Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 3 Jun 2022 21:17:16 -0400 Subject: [PATCH 024/123] Test that own edits are streamed --- test/pleroma/web/streamer_test.exs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 5ceaf763f..56cbf1b7b 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -456,6 +456,17 @@ defmodule Pleroma.Web.StreamerTest do assert_receive {:render_with_user, _, "status_update.json", ^edited} refute Streamer.filtered_by_user?(user, edited) end + + test "it streams own edits in the 'user' stream", %{user: user, token: oauth_token} do + {:ok, activity} = CommonAPI.post(user, %{status: "hey"}) + + Streamer.get_topic_and_add_socket("user", user, oauth_token) + {:ok, edited} = CommonAPI.update(user, activity, %{status: "mew mew"}) + edited = Pleroma.Activity.normalize(edited) + + assert_receive {:render_with_user, _, "status_update.json", ^edited} + refute Streamer.filtered_by_user?(user, edited) + end end describe "public streams" do From 3249ac1f12b69718cacc193c020e8bdccf167a9e Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 3 Jun 2022 21:47:40 -0400 Subject: [PATCH 025/123] Show edited_at in MastodonAPI/show --- lib/pleroma/web/api_spec/schemas/status.ex | 6 ++++++ .../web/mastodon_api/views/status_view.ex | 11 +++++++++++ .../web/mastodon_api/views/status_view_test.exs | 16 ++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 6e6e30315..f803caec2 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -73,6 +73,12 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do format: "date-time", description: "The date when this status was created" }, + edited_at: %Schema{ + type: :string, + format: "date-time", + nullable: true, + description: "The date when this status was last edited" + }, emojis: %Schema{ type: :array, items: Emoji, diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 8d4685ffa..4afba4b33 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -258,6 +258,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do created_at = Utils.to_masto_date(object.data["published"]) + edited_at = + with %{"updated" => updated} <- object.data, + date <- Utils.to_masto_date(updated), + true <- date != "" do + date + else + _ -> + nil + end + reply_to = get_reply_to(activity, opts) reply_to_user = reply_to && CommonAPI.get_user(reply_to.data["actor"]) @@ -346,6 +356,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do content: content_html, text: opts[:with_source] && object.data["source"], created_at: created_at, + edited_at: edited_at, reblogs_count: announcement_count, replies_count: object.data["repliesCount"] || 0, favourites_count: like_count, diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index 5d81c92b9..44cc003f3 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -246,6 +246,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do content: HTML.filter_tags(object_data["content"]), text: nil, created_at: created_at, + edited_at: nil, reblogs_count: 0, replies_count: 0, favourites_count: 0, @@ -708,4 +709,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do status = StatusView.render("show.json", activity: visible, for: poster) assert status.pleroma.parent_visible end + + test "it shows edited_at" do + poster = insert(:user) + + {:ok, post} = CommonAPI.post(poster, %{status: "hey"}) + + status = StatusView.render("show.json", activity: post) + refute status.edited_at + + {:ok, _} = CommonAPI.update(poster, post, %{status: "mew mew"}) + edited = Pleroma.Activity.normalize(post) + + status = StatusView.render("show.json", activity: edited) + assert status.edited_at + end end From 72ac940618efe56e14f02e23e5af75ae275a5c26 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 3 Jun 2022 21:50:49 -0400 Subject: [PATCH 026/123] Fix SideEffectsTest --- test/pleroma/web/activity_pub/side_effects_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 62394b058..c709ac75a 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -144,6 +144,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do setup do user = insert(:user) note = insert(:note, user: user) + _note_activity = insert(:note_activity, note: note) updated_note = note.data From fe2d4778eee5e8b4fe24f8e1d16d1065e9430027 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 4 Jun 2022 12:56:56 -0400 Subject: [PATCH 027/123] Expose content type of status sources --- .../API/differences_in_mastoapi_responses.md | 4 +++ .../api_spec/operations/status_operation.ex | 4 +++ lib/pleroma/web/common_api/activity_draft.ex | 5 ++- lib/pleroma/web/common_api/utils.ex | 2 +- .../web/mastodon_api/views/status_view.ex | 27 ++++++++++++-- .../mastodon_api/views/status_view_test.exs | 36 +++++++++++++++++++ 6 files changed, 73 insertions(+), 5 deletions(-) diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md index 73c46fff8..4007c63c8 100644 --- a/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/development/API/differences_in_mastoapi_responses.md @@ -40,6 +40,10 @@ Has these additional fields under the `pleroma` object: - `parent_visible`: If the parent of this post is visible to the user or not. - `pinned_at`: a datetime (iso8601) when status was pinned, `null` otherwise. +The `GET /api/v1/statuses/:id/source` endpoint additionally has the following attributes: + +- `content_type`: The content type of the status source. + ## Scheduled statuses Has these additional fields in `params`: diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index c69307a4d..e921128c7 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -761,6 +761,10 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do type: :string, description: "Subject or summary line, below which status content is collapsed until expanded" + }, + content_type: %Schema{ + type: :string, + description: "The content type of the source" } } } diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 7c21c8c3a..9af635da8 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -224,7 +224,10 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do object = note_data |> Map.put("emoji", emoji) - |> Map.put("source", draft.status) + |> Map.put("source", %{ + "content" => draft.status, + "mediaType" => Utils.get_content_type(draft.params[:content_type]) + }) |> Map.put("generator", draft.params[:generator]) %__MODULE__{draft | object: object} diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index ce850b038..4c6a26384 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -219,7 +219,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do |> maybe_add_attachments(draft.attachments, attachment_links) end - defp get_content_type(content_type) do + def get_content_type(content_type) do if Enum.member?(Config.get([:instance, :allowed_post_formats]), content_type) do content_type else diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 4afba4b33..f798b2624 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -354,7 +354,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do reblog: nil, card: card, content: content_html, - text: opts[:with_source] && object.data["source"], + text: opts[:with_source] && get_source_text(object.data["source"]), created_at: created_at, edited_at: edited_at, reblogs_count: announcement_count, @@ -465,8 +465,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do %{ id: activity.id, - text: Map.get(object.data, "source", ""), - spoiler_text: Map.get(object.data, "summary", "") + text: get_source_text(Map.get(object.data, "source", "")), + spoiler_text: Map.get(object.data, "summary", ""), + content_type: get_source_content_type(object.data["source"]) } end @@ -687,4 +688,24 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end defp build_image_url(_, _), do: nil + + defp get_source_text(%{"content" => content} = _source) do + content + end + + defp get_source_text(source) when is_binary(source) do + source + end + + defp get_source_text(_) do + "" + end + + defp get_source_content_type(%{"mediaType" => type} = _source) do + type + end + + defp get_source_content_type(_source) do + Utils.get_content_type(nil) + end end diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index 44cc003f3..297889449 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -724,4 +724,40 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do status = StatusView.render("show.json", activity: edited) assert status.edited_at end + + test "with a source object" do + note = + insert(:note, + data: %{"source" => %{"content" => "object source", "mediaType" => "text/markdown"}} + ) + + activity = insert(:note_activity, note: note) + + status = StatusView.render("show.json", activity: activity, with_source: true) + assert status.text == "object source" + end + + describe "source.json" do + test "with a source object, renders both source and content type" do + note = + insert(:note, + data: %{"source" => %{"content" => "object source", "mediaType" => "text/markdown"}} + ) + + activity = insert(:note_activity, note: note) + + status = StatusView.render("source.json", activity: activity) + assert status.text == "object source" + assert status.content_type == "text/markdown" + end + + test "with a source string, renders source and put text/plain as the content type" do + note = insert(:note, data: %{"source" => "string source"}) + activity = insert(:note_activity, note: note) + + status = StatusView.render("source.json", activity: activity) + assert status.text == "string source" + assert status.content_type == "text/plain" + end + end end From 97eabb20473d962065cb32782737ee10c794617b Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 4 Jun 2022 13:15:07 -0400 Subject: [PATCH 028/123] Fix CommonAPITest --- test/pleroma/web/common_api_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index af91fdf74..2f1197c37 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -586,7 +586,7 @@ defmodule Pleroma.Web.CommonAPITest do object = Object.normalize(activity, fetch: false) assert object.data["content"] == "

2hu

alert('xss')" - assert object.data["source"] == post + assert object.data["source"]["content"] == post end test "it filters out obviously bad tags when accepting a post as Markdown" do @@ -603,7 +603,7 @@ defmodule Pleroma.Web.CommonAPITest do object = Object.normalize(activity, fetch: false) assert object.data["content"] == "

2hu

" - assert object.data["source"] == post + assert object.data["source"]["content"] == post end test "it does not allow replies to direct messages that are not direct messages themselves" do From 06a3998013aca1f74c563d261d050543056c1255 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 5 Jun 2022 15:02:25 -0400 Subject: [PATCH 029/123] Create Update notifications --- lib/pleroma/notification.ex | 22 +++++++- ...85734_add_update_to_notifications_enum.exs | 51 +++++++++++++++++++ test/pleroma/notification_test.exs | 46 +++++++++++++++++ 3 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 52fd2656b..82aeb1802 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -385,7 +385,7 @@ defmodule Pleroma.Notification do end def create_notifications(%Activity{data: %{"type" => type}} = activity, options) - when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag"] do + when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag", "Update"] do do_create_notifications(activity, options) end @@ -439,6 +439,9 @@ defmodule Pleroma.Notification do activity |> type_from_activity_object() + "Update" -> + "update" + t -> raise "No notification type for activity type #{t}" end @@ -513,7 +516,7 @@ defmodule Pleroma.Notification do def get_notified_from_activity(activity, local_only \\ true) def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only) - when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact", "Flag"] do + when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact", "Flag", "Update"] do potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity) potential_receivers = @@ -553,6 +556,21 @@ defmodule Pleroma.Notification do (User.all_superusers() |> Enum.map(fn user -> user.ap_id end)) -- [actor] end + # Update activity: notify all who repeated this + def get_potential_receiver_ap_ids(%{data: %{"type" => "Update", "actor" => actor}} = activity) do + with %Object{data: %{"id" => object_id}} <- Object.normalize(activity, fetch: false) do + repeaters = + Activity.Queries.by_type("Announce") + |> Activity.Queries.by_object_id(object_id) + |> Activity.with_joined_user_actor() + |> where([a, u], u.local) + |> select([a, u], u.ap_id) + |> Repo.all() + + repeaters -- [actor] + end + end + def get_potential_receiver_ap_ids(activity) do [] |> Utils.maybe_notify_to_recipients(activity) diff --git a/priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs b/priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs new file mode 100644 index 000000000..0656c885f --- /dev/null +++ b/priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs @@ -0,0 +1,51 @@ +defmodule Pleroma.Repo.Migrations.AddUpdateToNotificationsEnum do + use Ecto.Migration + + @disable_ddl_transaction true + + def up do + """ + alter type notification_type add value 'update' + """ + |> execute() + end + + # 20210717000000_add_poll_to_notifications_enum.exs + def down do + alter table(:notifications) do + modify(:type, :string) + end + + """ + delete from notifications where type = 'update' + """ + |> execute() + + """ + drop type if exists notification_type + """ + |> execute() + + """ + create type notification_type as enum ( + 'follow', + 'follow_request', + 'mention', + 'move', + 'pleroma:emoji_reaction', + 'pleroma:chat_mention', + 'reblog', + 'favourite', + 'pleroma:report', + 'poll' + ) + """ + |> execute() + + """ + alter table notifications + alter column type type notification_type using (type::notification_type) + """ + |> execute() + end +end diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index 805764ea4..a000c0efd 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -127,6 +127,28 @@ defmodule Pleroma.NotificationTest do subscriber_notifications = Notification.for_user(subscriber) assert Enum.empty?(subscriber_notifications) end + + test "it sends edited notifications to those who repeated a status" do + user = insert(:user) + repeated_user = insert(:user) + other_user = insert(:user) + + {:ok, activity_one} = + CommonAPI.post(user, %{ + status: "hey @#{other_user.nickname}!" + }) + + {:ok, _activity_two} = CommonAPI.repeat(activity_one.id, repeated_user) + + {:ok, _edit_activity} = + CommonAPI.update(user, activity_one, %{ + status: "hey @#{other_user.nickname}! mew mew" + }) + + assert [%{type: "reblog"}] = Notification.for_user(user) + assert [%{type: "update"}] = Notification.for_user(repeated_user) + assert [%{type: "mention"}] = Notification.for_user(other_user) + end end test "create_poll_notifications/1" do @@ -839,6 +861,30 @@ defmodule Pleroma.NotificationTest do assert [other_user] == enabled_receivers assert [] == disabled_receivers end + + test "it sends edited notifications to those who repeated a status" do + user = insert(:user) + repeated_user = insert(:user) + other_user = insert(:user) + + {:ok, activity_one} = + CommonAPI.post(user, %{ + status: "hey @#{other_user.nickname}!" + }) + + {:ok, _activity_two} = CommonAPI.repeat(activity_one.id, repeated_user) + + {:ok, edit_activity} = + CommonAPI.update(user, activity_one, %{ + status: "hey @#{other_user.nickname}! mew mew" + }) + + {enabled_receivers, _disabled_receivers} = + Notification.get_notified_from_activity(edit_activity) + + assert repeated_user in enabled_receivers + assert other_user not in enabled_receivers + end end describe "notification lifecycle" do From 532f6ae3ede9b0795a164ca170314b95d5113fc8 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 5 Jun 2022 16:34:42 -0400 Subject: [PATCH 030/123] Return update notification in mastodon api --- .../controllers/notification_controller.ex | 1 + .../mastodon_api/views/notification_view.ex | 15 ++++++++--- .../views/notification_view_test.exs | 26 +++++++++++++++++++ 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index 932bc6423..e93930771 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -51,6 +51,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do move pleroma:emoji_reaction poll + update } def index(%{assigns: %{user: user}} = conn, params) do params = diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 0dc7f3beb..b5b5b2376 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -19,7 +19,11 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView - @parent_types ~w{Like Announce EmojiReact} + defp object_id_for(%{data: %{"object" => %{"id" => id}}}) when is_binary(id), do: id + + defp object_id_for(%{data: %{"object" => id}}) when is_binary(id), do: id + + @parent_types ~w{Like Announce EmojiReact Update} def render("index.json", %{notifications: notifications, for: reading_user} = opts) do activities = Enum.map(notifications, & &1.activity) @@ -30,7 +34,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do %{data: %{"type" => type}} -> type in @parent_types end) - |> Enum.map(& &1.data["object"]) + |> Enum.map(&object_id_for/1) |> Activity.create_by_object_ap_id() |> Activity.with_preloaded_object(:left) |> Pleroma.Repo.all() @@ -78,9 +82,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do parent_activity_fn = fn -> if opts[:parent_activities] do - Activity.Queries.find_by_object_ap_id(opts[:parent_activities], activity.data["object"]) + Activity.Queries.find_by_object_ap_id(opts[:parent_activities], object_id_for(activity)) else - Activity.get_create_by_object_ap_id(activity.data["object"]) + Activity.get_create_by_object_ap_id(object_id_for(activity)) end end @@ -109,6 +113,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do "reblog" -> put_status(response, parent_activity_fn.(), reading_user, status_render_opts) + "update" -> + put_status(response, parent_activity_fn.(), reading_user, status_render_opts) + "move" -> put_target(response, activity, reading_user, %{}) diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs index 8e4c9136a..d3d74f5cd 100644 --- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs @@ -237,6 +237,32 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do test_notifications_rendering([notification], moderator_user, [expected]) end + test "Edit notification" do + user = insert(:user) + repeat_user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "mew"}) + {:ok, _} = CommonAPI.repeat(activity.id, repeat_user) + {:ok, update} = CommonAPI.update(user, activity, %{status: "mew mew"}) + + user = Pleroma.User.get_by_ap_id(user.ap_id) + activity = Pleroma.Activity.normalize(activity) + update = Pleroma.Activity.normalize(update) + + {:ok, [notification]} = Notification.create_notifications(update) + + expected = %{ + id: to_string(notification.id), + pleroma: %{is_seen: false, is_muted: false}, + type: "update", + account: AccountView.render("show.json", %{user: user, for: repeat_user}), + created_at: Utils.to_masto_date(notification.inserted_at), + status: StatusView.render("show.json", %{activity: activity, for: repeat_user}) + } + + test_notifications_rendering([notification], repeat_user, [expected]) + end + test "muted notification" do user = insert(:user) another_user = insert(:user) From d2d3532e5f3e5bcedc91fd0f5ac4ca69043348db Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 5 Jun 2022 16:35:01 -0400 Subject: [PATCH 031/123] Lint --- lib/pleroma/notification.ex | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 82aeb1802..2906c599d 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -516,7 +516,16 @@ defmodule Pleroma.Notification do def get_notified_from_activity(activity, local_only \\ true) def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only) - when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact", "Flag", "Update"] do + when type in [ + "Create", + "Like", + "Announce", + "Follow", + "Move", + "EmojiReact", + "Flag", + "Update" + ] do potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity) potential_receivers = From 237b220d71bfe7db66db12549851fb93900a060a Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 8 Jun 2022 11:05:48 -0400 Subject: [PATCH 032/123] Add object id to uploaded attachments --- lib/pleroma/upload.ex | 2 ++ test/pleroma/upload_test.exs | 30 ++++++++++++++++-------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 242813dcd..7480c57a6 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -36,6 +36,7 @@ defmodule Pleroma.Upload do alias Ecto.UUID alias Pleroma.Config alias Pleroma.Maps + alias Pleroma.Web.ActivityPub.Utils require Logger @type source :: @@ -88,6 +89,7 @@ defmodule Pleroma.Upload do {:ok, url_spec} <- Pleroma.Uploaders.Uploader.put_file(opts.uploader, upload) do {:ok, %{ + "id" => Utils.generate_object_id(), "type" => opts.activity_type, "mediaType" => upload.content_type, "url" => [ diff --git a/test/pleroma/upload_test.exs b/test/pleroma/upload_test.exs index f2795f985..6584c2def 100644 --- a/test/pleroma/upload_test.exs +++ b/test/pleroma/upload_test.exs @@ -49,20 +49,22 @@ defmodule Pleroma.UploadTest do test "it returns file" do File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") - assert Upload.store(@upload_file) == - {:ok, - %{ - "name" => "image.jpg", - "type" => "Document", - "mediaType" => "image/jpeg", - "url" => [ - %{ - "href" => "http://localhost:4001/media/post-process-file.jpg", - "mediaType" => "image/jpeg", - "type" => "Link" - } - ] - }} + assert {:ok, result} = Upload.store(@upload_file) + + assert result == + %{ + "id" => result["id"], + "name" => "image.jpg", + "type" => "Document", + "mediaType" => "image/jpeg", + "url" => [ + %{ + "href" => "http://localhost:4001/media/post-process-file.jpg", + "mediaType" => "image/jpeg", + "type" => "Link" + } + ] + } Task.await(Agent.get(TestUploaderSuccess, fn task_pid -> task_pid end)) end From aafd7a687dea7595ee9431451d8e170fc3ff909e Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 8 Jun 2022 11:45:24 -0400 Subject: [PATCH 033/123] Return the corresponding object id in attachment view --- lib/pleroma/web/common_api/utils.ex | 8 ++++++-- .../web/mastodon_api/views/status_view.ex | 13 +++++++++++-- .../controllers/status_controller_test.exs | 19 +++++++++++++++++++ test/support/factory.ex | 12 ++++++++++++ 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 4c6a26384..5fc8c3220 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -37,7 +37,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do def attachments_from_ids_no_descs(ids) do Enum.map(ids, fn media_id -> - case Repo.get(Object, media_id) do + case get_attachment(media_id) do %Object{data: data} -> data _ -> nil end @@ -51,13 +51,17 @@ defmodule Pleroma.Web.CommonAPI.Utils do {_, descs} = Jason.decode(descs_str) Enum.map(ids, fn media_id -> - with %Object{data: data} <- Repo.get(Object, media_id) do + with %Object{data: data} <- get_attachment(media_id) do Map.put(data, "name", descs[media_id]) end end) |> Enum.reject(&is_nil/1) end + defp get_attachment(media_id) do + Repo.get(Object, media_id) + end + @spec get_to_and_cc(ActivityDraft.t()) :: {list(String.t()), list(String.t())} def get_to_and_cc(%{in_reply_to_conversation: %Participation{} = participation}) do diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index f798b2624..43f5fa02e 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -523,10 +523,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do true -> "unknown" end - <> = :crypto.hash(:md5, href) + attachment_id = + with {_, ap_id} when is_binary(ap_id) <- {:ap_id, attachment["id"]}, + {_, %Object{data: _object_data, id: object_id}} <- + {:object, Object.get_by_ap_id(ap_id)} do + to_string(object_id) + else + _ -> + <> = :crypto.hash(:md5, href) + to_string(attachment["id"] || hash_id) + end %{ - id: to_string(attachment["id"] || hash_id), + id: attachment_id, url: href, remote_url: href, preview_url: href_preview, diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index d4ca2d618..3bfe5ea79 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2077,6 +2077,25 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do assert response["spoiler_text"] == "lol" end + test "it updates the attachments", %{conn: conn, user: user} do + attachment = insert(:attachment, user: user) + attachment_id = to_string(attachment.id) + + {:ok, activity} = CommonAPI.post(user, %{status: "mew mew #abc", spoiler_text: "#def"}) + + response = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "mew mew #abc", + "spoiler_text" => "#def", + "media_ids" => [attachment_id] + }) + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^attachment_id}] = response["media_attachments"] + end + test "it does not update visibility", %{conn: conn, user: user} do {:ok, activity} = CommonAPI.post(user, %{ diff --git a/test/support/factory.ex b/test/support/factory.ex index 09456debf..aaadae9bd 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -111,6 +111,18 @@ defmodule Pleroma.Factory do } end + def attachment_factory(attrs \\ %{}) do + user = attrs[:user] || insert(:user) + + data = + attachment_data(user.ap_id, nil) + |> Map.put("id", Pleroma.Web.ActivityPub.Utils.generate_object_id()) + + %Pleroma.Object{ + data: merge_attributes(data, Map.get(attrs, :data, %{})) + } + end + def attachment_note_factory(attrs \\ %{}) do user = attrs[:user] || insert(:user) {length, attrs} = Map.pop(attrs, :length, 1) From c3593639adfdd6f9e086aaab18bda5c83bcfcc8b Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Thu, 9 Jun 2022 11:39:51 -0400 Subject: [PATCH 034/123] Fix incorrectly cached content after editing --- lib/pleroma/activity/html.ex | 36 ++++++++++++++++ lib/pleroma/application.ex | 1 + lib/pleroma/web/activity_pub/side_effects.ex | 25 +++++++---- .../web/mastodon_api/views/status_view.ex | 41 ++++++++++++++++--- .../controllers/status_controller_test.exs | 16 +++++++- test/pleroma/web/metadata/utils_test.exs | 16 +++++++- 6 files changed, 119 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/activity/html.ex b/lib/pleroma/activity/html.ex index 071a89c8d..706b2d36c 100644 --- a/lib/pleroma/activity/html.ex +++ b/lib/pleroma/activity/html.ex @@ -8,6 +8,40 @@ defmodule Pleroma.Activity.HTML do @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + # We store a list of cache keys related to an activity in a + # separate cache, scrubber_management_cache. It has the same + # size as scrubber_cache (see application.ex). Every time we add + # a cache to scrubber_cache, we update scrubber_management_cache. + # + # The most recent write of a certain key in the management cache + # is the same as the most recent write of any record related to that + # key in the main cache. + # Assuming LRW ( https://hexdocs.pm/cachex/Cachex.Policy.LRW.html ), + # this means when the management cache is evicted by cachex, all + # related records in the main cache will also have been evicted. + + defp get_cache_keys_for(activity_id) do + with {:ok, list} when is_list(list) <- @cachex.get(:scrubber_management_cache, activity_id) do + list + else + _ -> [] + end + end + + defp add_cache_key_for(activity_id, additional_key) do + current = get_cache_keys_for(activity_id) + + unless additional_key in current do + @cachex.put(:scrubber_management_cache, activity_id, [additional_key | current]) + end + end + + def invalidate_cache_for(activity_id) do + keys = get_cache_keys_for(activity_id) + Enum.map(keys, &@cachex.del(:scrubber_cache, &1)) + @cachex.del(:scrubber_management_cache, activity_id) + end + def get_cached_scrubbed_html_for_activity( content, scrubbers, @@ -19,6 +53,8 @@ defmodule Pleroma.Activity.HTML do @cachex.fetch!(:scrubber_cache, key, fn _key -> object = Object.normalize(activity, fetch: false) + + add_cache_key_for(activity.id, key) HTML.ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback) end) end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index d808bc732..e6b733f9b 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -189,6 +189,7 @@ defmodule Pleroma.Application do build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500), build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000), build_cachex("scrubber", limit: 2500), + build_cachex("scrubber_management", limit: 2500), build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 05f9b9bd9..d387d9362 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -455,7 +455,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do %{data: %{"type" => "Update", "object" => updated_object}} = object, meta ) do - orig_object = Object.get_by_ap_id(updated_object["id"]) + orig_object_ap_id = updated_object["id"] + orig_object = Object.get_by_ap_id(orig_object_ap_id) orig_object_data = orig_object.data if orig_object_data["type"] in @updatable_object_types do @@ -468,15 +469,21 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do |> Object.maybe_update_history(orig_object_data, updated) |> maybe_update_poll(updated_object) - orig_object - |> Repo.preload(:hashtags) - |> Object.change(%{data: updated_object_data}) - |> Object.update_and_set_cache() + changeset = + orig_object + |> Repo.preload(:hashtags) + |> Object.change(%{data: updated_object_data}) - if updated do - object - |> Activity.normalize() - |> ActivityPub.notify_and_stream() + with {:ok, new_object} <- Repo.update(changeset), + {:ok, _} <- Object.invalid_object_cache(new_object), + {:ok, _} <- Object.set_cache(new_object), + # The metadata/utils.ex uses the object id for the cache. + {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do + if updated do + object + |> Activity.normalize() + |> ActivityPub.notify_and_stream() + end end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 43f5fa02e..9cb2adcf9 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -272,6 +272,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do reply_to_user = reply_to && CommonAPI.get_user(reply_to.data["actor"]) + history_len = + 1 + + (Object.history_for(object.data) + |> Map.get("orderedItems") + |> length()) + + # See render("history.json", ...) for more details + # Here the implicit index of the current content is 0 + chrono_order = history_len - 1 + content = object |> render_content() @@ -281,14 +291,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Activity.HTML.get_cached_scrubbed_html_for_activity( User.html_filter_policy(opts[:for]), activity, - "mastoapi:content" + "mastoapi:content:#{chrono_order}" ) content_plaintext = content |> Activity.HTML.get_cached_stripped_html_for_activity( activity, - "mastoapi:content" + "mastoapi:content:#{chrono_order}" ) summary = object.data["summary"] || "" @@ -410,9 +420,25 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do history = [object | past_history] + history_len = length(history) + + history = + Enum.with_index( + history, + fn object, index -> + %{ + # The history is prepended every time there is a new edit. + # In chrono_order, the oldest item is always at 0, and so on. + # The chrono_order is an invariant kept between edits. + chrono_order: history_len - 1 - index, + object: object + } + end + ) + individual_opts = opts - |> Map.put(:as, :object) + |> Map.put(:as, :item) |> Map.put(:user, user) |> Map.put(:hashtags, hashtags) @@ -421,7 +447,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do def render( "history_item.json", - %{activity: activity, user: user, object: object, hashtags: hashtags} = opts + %{ + activity: activity, + user: user, + item: %{object: object, chrono_order: chrono_order}, + hashtags: hashtags + } = opts ) do sensitive = object.data["sensitive"] || Enum.member?(hashtags, "nsfw") @@ -439,7 +470,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Activity.HTML.get_cached_scrubbed_html_for_activity( User.html_filter_policy(opts[:for]), activity, - "mastoapi:content" + "mastoapi:content:#{chrono_order}" ) summary = object.data["summary"] || "" diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 3bfe5ea79..c077670ed 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2061,9 +2061,15 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do oauth_access(["write:statuses"]) end - test "it updates the status", %{conn: conn, user: user} do + test "it updates the status" do + %{conn: conn, user: user} = oauth_access(["write:statuses", "read:statuses"]) + {:ok, activity} = CommonAPI.post(user, %{status: "mew mew #abc", spoiler_text: "#def"}) + conn + |> get("/api/v1/statuses/#{activity.id}") + |> json_response_and_validate_schema(200) + response = conn |> put_req_header("content-type", "application/json") @@ -2075,6 +2081,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do assert response["content"] == "edited" assert response["spoiler_text"] == "lol" + + response = + conn + |> get("/api/v1/statuses/#{activity.id}") + |> json_response_and_validate_schema(200) + + assert response["content"] == "edited" + assert response["spoiler_text"] == "lol" end test "it updates the attachments", %{conn: conn, user: user} do diff --git a/test/pleroma/web/metadata/utils_test.exs b/test/pleroma/web/metadata/utils_test.exs index ce8ed5683..5f2f4a056 100644 --- a/test/pleroma/web/metadata/utils_test.exs +++ b/test/pleroma/web/metadata/utils_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Metadata.UtilsTest do - use Pleroma.DataCase, async: true + use Pleroma.DataCase, async: false import Pleroma.Factory alias Pleroma.Web.Metadata.Utils @@ -22,6 +22,20 @@ defmodule Pleroma.Web.Metadata.UtilsTest do assert Utils.scrub_html_and_truncate(note) == "Pleroma's really cool!" end + + test "it does not return old content after editing" do + user = insert(:user) + + {:ok, activity} = Pleroma.Web.CommonAPI.post(user, %{status: "mew mew #def"}) + + object = Pleroma.Object.normalize(activity) + assert Utils.scrub_html_and_truncate(object) == "mew mew #def" + + {:ok, update} = Pleroma.Web.CommonAPI.update(user, activity, %{status: "mew mew #abc"}) + update = Pleroma.Activity.normalize(update) + object = Pleroma.Object.normalize(update) + assert Utils.scrub_html_and_truncate(object) == "mew mew #abc" + end end describe "scrub_html_and_truncate/2" do From 27f3d802f2fd6e9d002654993d8eedb92d120055 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 11 Jun 2022 10:35:36 -0400 Subject: [PATCH 035/123] Expose history and source apis to anon users --- .../web/mastodon_api/controllers/status_controller.ex | 10 ++++++---- lib/pleroma/web/router.ex | 4 ++-- .../controllers/status_controller_test.exs | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index fa86e9dc0..e594ea491 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -195,8 +195,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do end @doc "GET /api/v1/statuses/:id/history" - def show_history(%{assigns: %{user: user}} = conn, %{id: id} = params) do - with %Activity{} = activity <- Activity.get_by_id_with_object(id), + def show_history(%{assigns: assigns} = conn, %{id: id} = params) do + with user = assigns[:user], + %Activity{} = activity <- Activity.get_by_id_with_object(id), true <- Visibility.visible_for_user?(activity, user) do try_render(conn, "history.json", activity: activity, @@ -210,8 +211,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do end @doc "GET /api/v1/statuses/:id/source" - def show_source(%{assigns: %{user: user}} = conn, %{id: id} = _params) do - with %Activity{} = activity <- Activity.get_by_id_with_object(id), + def show_source(%{assigns: assigns} = conn, %{id: id} = _params) do + with user = assigns[:user], + %Activity{} = activity <- Activity.get_by_id_with_object(id), true <- Visibility.visible_for_user?(activity, user) do try_render(conn, "source.json", activity: activity, diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 2d2e5365e..4a999f0c2 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -552,8 +552,6 @@ defmodule Pleroma.Web.Router do get("/bookmarks", StatusController, :bookmarks) post("/statuses", StatusController, :create) - get("/statuses/:id/history", StatusController, :show_history) - get("/statuses/:id/source", StatusController, :show_source) put("/statuses/:id", StatusController, :update) delete("/statuses/:id", StatusController, :delete) post("/statuses/:id/reblog", StatusController, :reblog) @@ -611,6 +609,8 @@ defmodule Pleroma.Web.Router do get("/statuses/:id/card", StatusController, :card) get("/statuses/:id/favourited_by", StatusController, :favourited_by) get("/statuses/:id/reblogged_by", StatusController, :reblogged_by) + get("/statuses/:id/history", StatusController, :show_history) + get("/statuses/:id/source", StatusController, :show_source) get("/custom_emojis", CustomEmojiController, :index) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index c077670ed..04f1c17db 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -1993,7 +1993,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do describe "get status history" do setup do - oauth_access(["read:statuses"]) + %{conn: build_conn()} end test "unedited post", %{conn: conn} do @@ -2039,7 +2039,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do describe "get status source" do setup do - oauth_access(["read:statuses"]) + %{conn: build_conn()} end test "it returns the source", %{conn: conn} do From 7451f0e81f1fd378a3ff23d437e3cc6780d62fb4 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 11 Jun 2022 12:02:16 -0400 Subject: [PATCH 036/123] Send the correct update in streamer get_create_by_ap_id_with_object() seems to fetch the old object. Why this happens needs further investigation. --- lib/pleroma/web/streamer.ex | 8 ++++-- lib/pleroma/web/views/streamer_view.ex | 4 --- test/pleroma/web/streamer_test.exs | 37 +++++++++++++++++++++++--- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 8b7fb985b..fe909df0a 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -297,12 +297,16 @@ defmodule Pleroma.Web.Streamer do defp push_to_socket(_topic, %Activity{data: %{"type" => "Delete"}}), do: :noop defp push_to_socket(topic, %Activity{data: %{"type" => "Update"}} = item) do - anon_render = StreamerView.render("status_update.json", item) + create_activity = + Pleroma.Activity.get_create_by_object_ap_id(item.object.data["id"]) + |> Map.put(:object, item.object) + + anon_render = StreamerView.render("status_update.json", create_activity) Registry.dispatch(@registry, topic, fn list -> Enum.each(list, fn {pid, auth?} -> if auth? do - send(pid, {:render_with_user, StreamerView, "status_update.json", item}) + send(pid, {:render_with_user, StreamerView, "status_update.json", create_activity}) else send(pid, {:text, anon_render}) end diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index 797762d90..6a55242b0 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -26,8 +26,6 @@ defmodule Pleroma.Web.StreamerView do end def render("status_update.json", %Activity{} = activity, %User{} = user) do - activity = Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) - %{ event: "status.update", payload: @@ -68,8 +66,6 @@ defmodule Pleroma.Web.StreamerView do end def render("status_update.json", %Activity{} = activity) do - activity = Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) - %{ event: "status.update", payload: diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 56cbf1b7b..4891bf499 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -451,9 +451,9 @@ defmodule Pleroma.Web.StreamerTest do Streamer.get_topic_and_add_socket("user", user, oauth_token) {:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew"}) - edited = Pleroma.Activity.normalize(edited) + create = Pleroma.Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) - assert_receive {:render_with_user, _, "status_update.json", ^edited} + assert_receive {:render_with_user, _, "status_update.json", ^create} refute Streamer.filtered_by_user?(user, edited) end @@ -462,9 +462,9 @@ defmodule Pleroma.Web.StreamerTest do Streamer.get_topic_and_add_socket("user", user, oauth_token) {:ok, edited} = CommonAPI.update(user, activity, %{status: "mew mew"}) - edited = Pleroma.Activity.normalize(edited) + create = Pleroma.Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) - assert_receive {:render_with_user, _, "status_update.json", ^edited} + assert_receive {:render_with_user, _, "status_update.json", ^create} refute Streamer.filtered_by_user?(user, edited) end end @@ -528,6 +528,35 @@ defmodule Pleroma.Web.StreamerTest do assert %{"id" => ^activity_id} = Jason.decode!(payload) refute Streamer.filtered_by_user?(sender, edited) end + + test "it streams multiple edits in the 'public' stream correctly" do + sender = insert(:user) + + Streamer.get_topic_and_add_socket("public", nil, nil) + {:ok, activity} = CommonAPI.post(sender, %{status: "hey"}) + assert_receive {:text, _} + + {:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew"}) + + edited = Pleroma.Activity.normalize(edited) + + %{id: activity_id} = Pleroma.Activity.get_create_by_object_ap_id(edited.object.data["id"]) + + assert_receive {:text, event} + assert %{"event" => "status.update", "payload" => payload} = Jason.decode!(event) + assert %{"id" => ^activity_id} = Jason.decode!(payload) + refute Streamer.filtered_by_user?(sender, edited) + + {:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew 2"}) + + edited = Pleroma.Activity.normalize(edited) + + %{id: activity_id} = Pleroma.Activity.get_create_by_object_ap_id(edited.object.data["id"]) + assert_receive {:text, event} + assert %{"event" => "status.update", "payload" => payload} = Jason.decode!(event) + assert %{"id" => ^activity_id, "content" => "mew mew 2"} = Jason.decode!(payload) + refute Streamer.filtered_by_user?(sender, edited) + end end describe "thread_containment/2" do From 95b39223281a61f3ee7d52776df2713952de3be0 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 11 Jun 2022 16:28:59 -0400 Subject: [PATCH 037/123] Workaround with_index does not support function in Elixir 1.9 --- .../web/mastodon_api/views/status_view.ex | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 9cb2adcf9..6ede89803 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -423,18 +423,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do history_len = length(history) history = - Enum.with_index( - history, - fn object, index -> - %{ - # The history is prepended every time there is a new edit. - # In chrono_order, the oldest item is always at 0, and so on. - # The chrono_order is an invariant kept between edits. - chrono_order: history_len - 1 - index, - object: object - } - end - ) + Enum.zip(history_len..0, history) + |> Enum.map(fn {chrono_order, object} -> + %{ + # The history is prepended every time there is a new edit. + # In chrono_order, the oldest item is always at 0, and so on. + # The chrono_order is an invariant kept between edits. + chrono_order: chrono_order, + object: object + } + end) individual_opts = opts From 44613db853226015207977ee958ebbf4d26f7c00 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 11 Jun 2022 19:52:07 -0400 Subject: [PATCH 038/123] Show original status at the first of history --- lib/pleroma/web/mastodon_api/views/status_view.ex | 11 +++++------ .../controllers/status_controller_test.exs | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 6ede89803..8439431eb 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -418,13 +418,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Enum.map(&Map.put(&1, "id", object.data["id"])) |> Enum.map(&%Object{data: &1, id: object.id}) - history = [object | past_history] - - history_len = length(history) - history = - Enum.zip(history_len..0, history) - |> Enum.map(fn {chrono_order, object} -> + [object | past_history] + # Mastodon expects the original to be at the first + |> Enum.reverse() + |> Enum.with_index() + |> Enum.map(fn {object, chrono_order} -> %{ # The history is prepended every time there is a new edit. # In chrono_order, the oldest item is always at 0, and so on. diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 04f1c17db..05c5d9ed5 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2032,7 +2032,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do conn = get(conn, "/api/v1/statuses/#{activity.id}/history") - assert [_, %{"spoiler_text" => "title 2"}, %{"spoiler_text" => "title 1"}] = + assert [%{"spoiler_text" => "title 1"}, %{"spoiler_text" => "title 2"}, _] = json_response_and_validate_schema(conn, 200) end end From 06da000c5d4fc8d71bd36bfb4cec9cbf4399dfe8 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 21 Jun 2022 12:32:44 -0400 Subject: [PATCH 039/123] Add editing to features --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index ee52475d5..4f613416b 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -68,6 +68,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "shareable_emoji_packs", "multifetch", "pleroma:api/v1/notifications:include_types_filter", + "editing", if Config.get([:activitypub, :blockers_visible]) do "blockers_visible" end, From 01321c88b5ef0d6c236b968cc47eaa8873054b1e Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 24 Jun 2022 10:10:22 -0400 Subject: [PATCH 040/123] Convert incoming Updated object into Pleroma format --- .../web/activity_pub/object_validator.ex | 16 +++++++++ .../article_note_page_validator.ex | 5 ++- .../article_note_page_validator_test.exs | 5 +++ .../update_handling_test.exs | 35 ++++++++++++++++++- 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index f3e31c931..b3105c46e 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -141,6 +141,22 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end end + def validate( + %{"type" => "Update", "object" => %{"type" => objtype} = object} = update_activity, + meta + ) + when objtype in ~w[Question Answer Audio Video Event Article Note Page] do + with {:ok, object_data} <- cast_and_apply(object), + meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), + {:ok, update_activity} <- + update_activity + |> UpdateValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + update_activity = stringify_keys(update_activity) + {:ok, update_activity, meta} + end + end + def validate(%{"type" => type} = object, meta) when type in ~w[Accept Reject Follow Update Like EmojiReact Announce ChatMessage Answer] do diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index ca335bc8a..0a0d30e45 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -49,7 +49,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"]) defp fix_url(data), do: data - defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data + defp fix_tag(%{"tag" => tag} = data) when is_list(tag) do + Map.put(data, "tag", Enum.filter(tag, &is_map/1)) + end + defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag]) defp fix_tag(data), do: Map.drop(data, ["tag"]) diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index f93537ed8..c87b07547 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -31,6 +31,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest test "a basic note validates", %{note: note} do %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note) end + + test "a note from factory validates" do + note = insert(:note) + %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note.data) + end end test "a Note from Roadhouse validates" do diff --git a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs index f2a22d370..b812b0b0e 100644 --- a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs @@ -60,7 +60,40 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateHandlingTest do {:ok, update, _} = Builder.update(other_user, updated_note) - assert {:ok, _update, []} = ObjectValidator.validate(update, []) + assert {:ok, _update, _} = ObjectValidator.validate(update, []) + end + end + + describe "update note" do + test "converts object into Pleroma's format" do + mastodon_tags = [ + %{ + "icon" => %{ + "mediaType" => "image/png", + "type" => "Image", + "url" => "https://somewhere.org/emoji/url/1.png" + }, + "id" => "https://somewhere.org/emoji/1", + "name" => ":some_emoji:", + "type" => "Emoji", + "updated" => "2021-04-07T11:00:00Z" + } + ] + + user = insert(:user) + note = insert(:note, user: user) + + updated_note = + note.data + |> Map.put("content", "edited content") + |> Map.put("tag", mastodon_tags) + + {:ok, update, _} = Builder.update(user, updated_note) + + assert {:ok, _update, meta} = ObjectValidator.validate(update, []) + + assert %{"emoji" => %{"some_emoji" => "https://somewhere.org/emoji/url/1.png"}} = + meta[:object_data] end end end From ee0738319169cf5c7d31dcaf641d9b744c7a72dc Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 24 Jun 2022 10:26:01 -0400 Subject: [PATCH 041/123] Use meta[:object_data] in SideEffects for Update --- lib/pleroma/web/activity_pub/side_effects.ex | 2 + .../web/activity_pub/side_effects_test.exs | 49 ++++++++++++++----- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index d387d9362..aa4183bf6 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -459,6 +459,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do orig_object = Object.get_by_ap_id(orig_object_ap_id) orig_object_data = orig_object.data + updated_object = meta[:object_data] + if orig_object_data["type"] in @updatable_object_types do %{data: updated_object_data, updated: updated} = orig_object_data diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index c709ac75a..8e84db774 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -154,21 +154,44 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do {:ok, update_data, []} = Builder.update(user, updated_note) {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) - %{user: user, note: note, object_id: note.id, update_data: update_data, update: update} + %{ + user: user, + note: note, + object_id: note.id, + update_data: update_data, + update: update, + updated_note: updated_note + } end - test "it updates the note", %{object_id: object_id, update: update} do - {:ok, _, _} = SideEffects.handle(update) + test "it updates the note", %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) new_note = Pleroma.Object.get_by_id(object_id) assert %{"summary" => "edited summary", "content" => "edited content"} = new_note.data end + test "it updates using object_data", %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + updated_note = Map.put(updated_note, "summary", "mew mew") + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + new_note = Pleroma.Object.get_by_id(object_id) + assert %{"summary" => "mew mew", "content" => "edited content"} = new_note.data + end + test "it records the original note in formerRepresentations", %{ note: note, object_id: object_id, - update: update + update: update, + updated_note: updated_note } do - {:ok, _, _} = SideEffects.handle(update) + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) %{data: new_note} = Pleroma.Object.get_by_id(object_id) assert %{"summary" => "edited summary", "content" => "edited content"} = new_note @@ -182,9 +205,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do user: user, note: note, object_id: object_id, - update: update + update: update, + updated_note: updated_note } do - {:ok, _, _} = SideEffects.handle(update) + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) %{data: first_edit} = Pleroma.Object.get_by_id(object_id) second_updated_note = @@ -194,7 +218,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do {:ok, second_update_data, []} = Builder.update(user, second_updated_note) {:ok, update, _meta} = ActivityPub.persist(second_update_data, local: true) - {:ok, _, _} = SideEffects.handle(update) + {:ok, _, _} = SideEffects.handle(update, object_data: second_updated_note) %{data: new_note} = Pleroma.Object.get_by_id(object_id) assert %{"summary" => "edited summary 2", "content" => "edited content 2"} = new_note @@ -210,12 +234,13 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do test "it does not prepend to formerRepresentations if no actual changes are made", %{ note: note, object_id: object_id, - update: update + update: update, + updated_note: updated_note } do - {:ok, _, _} = SideEffects.handle(update) + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) %{data: _first_edit} = Pleroma.Object.get_by_id(object_id) - {:ok, _, _} = SideEffects.handle(update) + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) %{data: new_note} = Pleroma.Object.get_by_id(object_id) assert %{"summary" => "edited summary", "content" => "edited content"} = new_note @@ -250,7 +275,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do {:ok, update_data, []} = Builder.update(user, updated_question) {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) - {:ok, _, _} = SideEffects.handle(update) + {:ok, _, _} = SideEffects.handle(update, object_data: updated_question) %{data: new_question} = Pleroma.Object.get_by_id(id) From e0d6da4e7d52e2cdd0fc5290e4ff3a23da7398f6 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 24 Jun 2022 10:54:11 -0400 Subject: [PATCH 042/123] Fix CommonAPITest --- .../web/activity_pub/object_validators/attachment_validator.ex | 3 ++- .../web/activity_pub/object_validators/common_fields.ex | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index d1c61ac82..ca6e39612 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do @primary_key false embedded_schema do + field(:id, :string) field(:type, :string) field(:mediaType, :string, default: "application/octet-stream") field(:name, :string) @@ -43,7 +44,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do |> fix_url() struct - |> cast(data, [:type, :mediaType, :name, :blurhash]) + |> cast(data, [:id, :type, :mediaType, :name, :blurhash]) |> cast_embed(:url, with: &url_changeset/2) |> validate_inclusion(:type, ~w[Link Document Audio Image Video]) |> validate_required([:type, :mediaType, :url]) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex index 8e768ffbf..a59a6e545 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex @@ -33,6 +33,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do field(:content, :string) field(:published, ObjectValidators.DateTime) + field(:updated, ObjectValidators.DateTime) field(:emoji, ObjectValidators.Emoji, default: %{}) embeds_many(:attachment, AttachmentValidator) end From 99a6f5031638da2eed237f91c6dded9e25717599 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 25 Jun 2022 00:32:22 -0400 Subject: [PATCH 043/123] Unify the logic of updating objects --- lib/pleroma/object.ex | 42 ----- lib/pleroma/object/fetcher.ex | 9 +- lib/pleroma/object/updater.ex | 157 ++++++++++++++++++ lib/pleroma/web/activity_pub/side_effects.ex | 62 ++----- lib/pleroma/web/common_api.ex | 10 +- .../web/mastodon_api/views/status_view.ex | 4 +- 6 files changed, 183 insertions(+), 101 deletions(-) create mode 100644 lib/pleroma/object/updater.ex diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 670ab8743..fe264b5e0 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -425,46 +425,4 @@ defmodule Pleroma.Object do end def object_data_hashtags(_), do: [] - - def history_for(object) do - with history <- Map.get(object, "formerRepresentations"), - true <- is_map(history), - "OrderedCollection" <- Map.get(history, "type"), - true <- is_list(Map.get(history, "orderedItems")), - true <- is_integer(Map.get(history, "totalItems")) do - history - else - _ -> history_skeleton() - end - end - - defp history_skeleton do - %{ - "type" => "OrderedCollection", - "totalItems" => 0, - "orderedItems" => [] - } - end - - def maybe_update_history(updated_object, orig_object_data, updated) do - if not updated do - updated_object - else - # Put edit history - # Note that we may have got the edit history by first fetching the object - history = Object.history_for(orig_object_data) - - latest_history_item = - orig_object_data - |> Map.drop(["id", "formerRepresentations"]) - - new_history = - history - |> Map.put("orderedItems", [latest_history_item | history["orderedItems"]]) - |> Map.put("totalItems", history["totalItems"] + 1) - - updated_object - |> Map.put("formerRepresentations", new_history) - end - end end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index ce816c1fc..d81fdcf24 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -50,7 +50,14 @@ defmodule Pleroma.Object.Fetcher do Pleroma.Constants.status_updatable_fields() |> Enum.any?(fn field -> Map.get(old_data, field) != Map.get(new_data, field) end) - new_data |> Object.maybe_update_history(old_data, changed?) + %{updated_object: updated_object} = + new_data + |> Object.Updater.maybe_update_history(old_data, + updated: changed?, + use_history_in_new_object?: false + ) + + updated_object else new_data end diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex new file mode 100644 index 000000000..03136c38e --- /dev/null +++ b/lib/pleroma/object/updater.ex @@ -0,0 +1,157 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Object.Updater do + require Pleroma.Constants + + def update_content_fields(orig_object_data, updated_object) do + Pleroma.Constants.status_updatable_fields() + |> Enum.reduce( + %{data: orig_object_data, updated: false}, + fn field, %{data: data, updated: updated} -> + updated = updated or Map.get(updated_object, field) != Map.get(orig_object_data, field) + + data = + if Map.has_key?(updated_object, field) do + Map.put(data, field, updated_object[field]) + else + Map.drop(data, [field]) + end + + %{data: data, updated: updated} + end + ) + end + + def maybe_history(object) do + with history <- Map.get(object, "formerRepresentations"), + true <- is_map(history), + "OrderedCollection" <- Map.get(history, "type"), + true <- is_list(Map.get(history, "orderedItems")), + true <- is_integer(Map.get(history, "totalItems")) do + history + else + _ -> nil + end + end + + def history_for(object) do + with history when not is_nil(history) <- maybe_history(object) do + history + else + _ -> history_skeleton() + end + end + + defp history_skeleton do + %{ + "type" => "OrderedCollection", + "totalItems" => 0, + "orderedItems" => [] + } + end + + def maybe_update_history( + updated_object, + orig_object_data, + opts + ) do + updated = opts[:updated] + use_history_in_new_object? = opts[:use_history_in_new_object?] + + if not updated do + %{updated_object: updated_object, used_history_in_new_object?: false} + else + # Put edit history + # Note that we may have got the edit history by first fetching the object + {new_history, used_history_in_new_object?} = + with true <- use_history_in_new_object?, + updated_history when not is_nil(updated_history) <- maybe_history(updated_object) do + {updated_history, true} + else + _ -> + history = history_for(orig_object_data) + + latest_history_item = + orig_object_data + |> Map.drop(["id", "formerRepresentations"]) + + updated_history = + history + |> Map.put("orderedItems", [latest_history_item | history["orderedItems"]]) + |> Map.put("totalItems", history["totalItems"] + 1) + + {updated_history, false} + end + + updated_object = + updated_object + |> Map.put("formerRepresentations", new_history) + + %{updated_object: updated_object, used_history_in_new_object?: used_history_in_new_object?} + end + end + + defp maybe_update_poll(to_be_updated, updated_object) do + choice_key = fn data -> + if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf" + end + + with true <- to_be_updated["type"] == "Question", + key <- choice_key.(updated_object), + true <- key == choice_key.(to_be_updated), + orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])), + new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])), + true <- orig_choices == new_choices do + # Choices are the same, but counts are different + to_be_updated + |> Map.put(key, updated_object[key]) + else + # Choices (or vote type) have changed, do not allow this + _ -> to_be_updated + end + end + + # This calculates the data to be sent as the object of an Update. + # new_data's formerRepresentations is not considered. + # formerRepresentations is added to the returned data. + def make_update_object_data(original_data, new_data, date) do + %{data: updated_data, updated: updated} = + original_data + |> update_content_fields(new_data) + + if not updated do + updated_data + else + %{updated_object: updated_data} = + updated_data + |> maybe_update_history(original_data, updated: updated, use_history_in_new_object?: false) + + updated_data + |> Map.put("updated", date) + end + end + + # This calculates the data of the new Object from an Update. + # new_data's formerRepresentations is considered. + def make_new_object_data_from_update_object(original_data, new_data) do + %{data: updated_data, updated: updated} = + original_data + |> update_content_fields(new_data) + + %{updated_object: updated_data, used_history_in_new_object?: used_history_in_new_object?} = + updated_data + |> maybe_update_history(original_data, updated: updated, use_history_in_new_object?: false) + + updated_data = + updated_data + |> maybe_update_poll(new_data) + + %{ + updated_data: updated_data, + updated: updated, + used_history_in_new_object?: used_history_in_new_object? + } + end +end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index aa4183bf6..7345a6904 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -412,45 +412,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end @updatable_object_types ["Note", "Question"] - defp update_content_fields(orig_object_data, updated_object) do - Pleroma.Constants.status_updatable_fields() - |> Enum.reduce( - %{data: orig_object_data, updated: false}, - fn field, %{data: data, updated: updated} -> - updated = updated or Map.get(updated_object, field) != Map.get(orig_object_data, field) - - data = - if Map.has_key?(updated_object, field) do - Map.put(data, field, updated_object[field]) - else - Map.drop(data, [field]) - end - - %{data: data, updated: updated} - end - ) - end - - defp maybe_update_poll(to_be_updated, updated_object) do - choice_key = fn data -> - if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf" - end - - with true <- to_be_updated["type"] == "Question", - key <- choice_key.(updated_object), - true <- key == choice_key.(to_be_updated), - orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])), - new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])), - true <- orig_choices == new_choices do - # Choices are the same, but counts are different - to_be_updated - |> Map.put(key, updated_object[key]) - else - # Choices (or vote type) have changed, do not allow this - _ -> to_be_updated - end - end - defp handle_update_object( %{data: %{"type" => "Update", "object" => updated_object}} = object, meta @@ -462,14 +423,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do updated_object = meta[:object_data] if orig_object_data["type"] in @updatable_object_types do - %{data: updated_object_data, updated: updated} = - orig_object_data - |> update_content_fields(updated_object) - - updated_object_data = - updated_object_data - |> Object.maybe_update_history(orig_object_data, updated) - |> maybe_update_poll(updated_object) + %{ + updated_data: updated_object_data, + updated: updated, + used_history_in_new_object?: used_history_in_new_object? + } = Object.Updater.make_new_object_data_from_update_object(orig_object_data, updated_object) changeset = orig_object @@ -481,6 +439,16 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do {:ok, _} <- Object.set_cache(new_object), # The metadata/utils.ex uses the object id for the cache. {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do + if used_history_in_new_object? do + with create_activity when not is_nil(create_activity) <- + Pleroma.Activity.get_create_by_object_ap_id(orig_object_ap_id), + {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(create_activity.id) do + nil + else + _ -> nil + end + end + if updated do object |> Activity.normalize() diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index e60c26053..e5a78c102 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -422,15 +422,7 @@ defmodule Pleroma.Web.CommonAPI do with {:ok, draft} <- ActivityDraft.create(user, params) do change = - Pleroma.Constants.status_updatable_fields() - |> Enum.reduce(orig_object.data, fn key, acc -> - if Map.has_key?(draft.object, key) do - acc |> Map.put(key, Map.get(draft.object, key)) - else - acc |> Map.drop([key]) - end - end) - |> Map.put("updated", Utils.make_date()) + Object.Updater.make_update_object_data(orig_object.data, draft.object, Utils.make_date()) {:ok, change} else diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 8439431eb..54e025aae 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -274,7 +274,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do history_len = 1 + - (Object.history_for(object.data) + (Object.Updater.history_for(object.data) |> Map.get("orderedItems") |> length()) @@ -413,7 +413,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do user = CommonAPI.get_user(activity.data["actor"]) past_history = - Object.history_for(object.data) + Object.Updater.history_for(object.data) |> Map.get("orderedItems") |> Enum.map(&Map.put(&1, "id", object.data["id"])) |> Enum.map(&%Object{data: &1, id: object.id}) From 40953a8f5c299e55b3f186bd6fdebe1bbf6e7401 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 25 Jun 2022 01:03:46 -0400 Subject: [PATCH 044/123] Reuse formerRepresentations from remote if possible --- lib/pleroma/object/updater.ex | 8 +++- test/pleroma/object/updater_test.exs | 65 ++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 test/pleroma/object/updater_test.exs diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index 03136c38e..0b21f6c99 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -67,7 +67,7 @@ defmodule Pleroma.Object.Updater do # Note that we may have got the edit history by first fetching the object {new_history, used_history_in_new_object?} = with true <- use_history_in_new_object?, - updated_history when not is_nil(updated_history) <- maybe_history(updated_object) do + updated_history when not is_nil(updated_history) <- maybe_history(opts[:new_data]) do {updated_history, true} else _ -> @@ -142,7 +142,11 @@ defmodule Pleroma.Object.Updater do %{updated_object: updated_data, used_history_in_new_object?: used_history_in_new_object?} = updated_data - |> maybe_update_history(original_data, updated: updated, use_history_in_new_object?: false) + |> maybe_update_history(original_data, + updated: updated, + use_history_in_new_object?: true, + new_data: new_data + ) updated_data = updated_data diff --git a/test/pleroma/object/updater_test.exs b/test/pleroma/object/updater_test.exs new file mode 100644 index 000000000..ef13439b9 --- /dev/null +++ b/test/pleroma/object/updater_test.exs @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Object.UpdaterTest do + use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + alias Pleroma.Object.Updater + + describe "make_update_object_data/3" do + setup do + note = insert(:note) + %{original_data: note.data} + end + + test "it makes an updated field", %{original_data: original_data} do + new_data = Map.put(original_data, "content", "new content") + + date = Pleroma.Web.ActivityPub.Utils.make_date() + update_object_data = Updater.make_update_object_data(original_data, new_data, date) + assert %{"updated" => ^date} = update_object_data + end + + test "it creates formerRepresentations", %{original_data: original_data} do + new_data = Map.put(original_data, "content", "new content") + + date = Pleroma.Web.ActivityPub.Utils.make_date() + update_object_data = Updater.make_update_object_data(original_data, new_data, date) + + history_item = original_data |> Map.drop(["id", "formerRepresentations"]) + + assert %{ + "formerRepresentations" => %{ + "totalItems" => 1, + "orderedItems" => [^history_item] + } + } = update_object_data + end + end + + describe "make_new_object_data_from_update_object/2" do + test "it reuses formerRepresentations if it exists" do + %{data: original_data} = insert(:note) + + new_data = + original_data + |> Map.put("content", "edited") + + date = Pleroma.Web.ActivityPub.Utils.make_date() + update_object_data = Updater.make_update_object_data(original_data, new_data, date) + + %{ + updated_data: _updated_data, + updated: updated, + used_history_in_new_object?: used_history_in_new_object? + } = Updater.make_new_object_data_from_update_object(original_data, update_object_data) + + assert updated + assert used_history_in_new_object? + end + end +end From e98579b1da78b47d6848ec55042640e539e44f6c Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 25 Jun 2022 01:17:09 -0400 Subject: [PATCH 045/123] Verify that formerRepresentation provided in Update is used --- test/pleroma/object/updater_test.exs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/pleroma/object/updater_test.exs b/test/pleroma/object/updater_test.exs index ef13439b9..7e9b44823 100644 --- a/test/pleroma/object/updater_test.exs +++ b/test/pleroma/object/updater_test.exs @@ -52,14 +52,25 @@ defmodule Pleroma.Object.UpdaterTest do date = Pleroma.Web.ActivityPub.Utils.make_date() update_object_data = Updater.make_update_object_data(original_data, new_data, date) + history = update_object_data["formerRepresentations"]["orderedItems"] + + update_object_data = + update_object_data + |> put_in( + ["formerRepresentations", "orderedItems"], + history ++ [Map.put(original_data, "summary", "additional summary")] + ) + |> put_in(["formerRepresentations", "totalItems"], length(history) + 1) + %{ - updated_data: _updated_data, + updated_data: updated_data, updated: updated, used_history_in_new_object?: used_history_in_new_object? } = Updater.make_new_object_data_from_update_object(original_data, update_object_data) assert updated assert used_history_in_new_object? + assert updated_data["formerRepresentations"] == update_object_data["formerRepresentations"] end end end From 9c6dae942d2ec5e2314af1d345cf2aeed504aae8 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 25 Jun 2022 09:23:09 -0400 Subject: [PATCH 046/123] Fix local updates causing emojis to be lost --- lib/pleroma/web/activity_pub/side_effects.ex | 9 ++++++++- test/pleroma/web/common_api_test.exs | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 7345a6904..747f467db 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -420,7 +420,14 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do orig_object = Object.get_by_ap_id(orig_object_ap_id) orig_object_data = orig_object.data - updated_object = meta[:object_data] + updated_object = + if meta[:local] do + # If this is a local Update, we don't process it by transmogrifier, + # so we use the embedded object as-is. + updated_object + else + meta[:object_data] + end if orig_object_data["type"] in @updatable_object_types do %{ diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 2f1197c37..32d6562d7 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1569,5 +1569,20 @@ defmodule Pleroma.Web.CommonAPITest do assert Visibility.get_visibility(updated_object) == "private" assert Visibility.get_visibility(updated) == "private" end + + test "updates a post with emoji" do + [{emoji1, _}, {emoji2, _} | _] = Pleroma.Emoji.get_all() + + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{status: "foo1", spoiler_text: "title 1 :#{emoji1}:"}) + + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2 :#{emoji2}:"}) + + updated_object = Object.normalize(updated) + assert updated_object.data["content"] == "updated 2 :#{emoji2}:" + assert %{^emoji2 => _} = updated_object.data["emoji"] + end end end From 5321fd001233076e7f2b6ea00adeb41ecf7e180a Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 25 Jun 2022 10:03:19 -0400 Subject: [PATCH 047/123] Do not put meta[:object_data] for local Updates --- .../web/activity_pub/object_validator.ex | 12 ++++++- .../update_handling_test.exs | 31 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index b3105c46e..d4bf9c31e 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -146,7 +146,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do meta ) when objtype in ~w[Question Answer Audio Video Event Article Note Page] do - with {:ok, object_data} <- cast_and_apply(object), + with {_, false} <- {:local, Access.get(meta, :local, false)}, + {:ok, object_data} <- cast_and_apply(object), meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), {:ok, update_activity} <- update_activity @@ -154,6 +155,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do |> Ecto.Changeset.apply_action(:insert) do update_activity = stringify_keys(update_activity) {:ok, update_activity, meta} + else + {:local, _} -> + with {:ok, object} <- + update_activity + |> UpdateValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end end end diff --git a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs index b812b0b0e..198c35cd3 100644 --- a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs @@ -95,5 +95,36 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateHandlingTest do assert %{"emoji" => %{"some_emoji" => "https://somewhere.org/emoji/url/1.png"}} = meta[:object_data] end + + test "returns no object_data in meta for a local Update" do + user = insert(:user) + note = insert(:note, user: user) + + updated_note = + note.data + |> Map.put("content", "edited content") + + {:ok, update, _} = Builder.update(user, updated_note) + + assert {:ok, _update, meta} = ObjectValidator.validate(update, local: true) + assert is_nil(meta[:object_data]) + end + + test "returns object_data in meta for a remote Update" do + user = insert(:user) + note = insert(:note, user: user) + + updated_note = + note.data + |> Map.put("content", "edited content") + + {:ok, update, _} = Builder.update(user, updated_note) + + assert {:ok, _update, meta} = ObjectValidator.validate(update, local: false) + assert meta[:object_data] + + assert {:ok, _update, meta} = ObjectValidator.validate(update, []) + assert meta[:object_data] + end end end From 014096aeefe88348323db74e2ab7f81e0184bfee Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 25 Jun 2022 11:20:46 -0400 Subject: [PATCH 048/123] Make outbound transmogrifier aware of edit history --- lib/pleroma/constants.ex | 12 +++ lib/pleroma/web/activity_pub/side_effects.ex | 3 +- .../web/activity_pub/transmogrifier.ex | 52 ++++++++----- .../web/activity_pub/transmogrifier_test.exs | 75 ++++++++++--------- test/pleroma/web/common_api_test.exs | 21 ++++++ 5 files changed, 109 insertions(+), 54 deletions(-) diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index bbb95104f..1b3d09d73 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -42,6 +42,18 @@ defmodule Pleroma.Constants do ] ) + const(updatable_object_types, + do: [ + "Note", + "Question", + "Audio", + "Video", + "Event", + "Article", + "Page" + ] + ) + const(actor_types, do: [ "Application", diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 747f467db..f56e357bf 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -411,7 +411,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do {:ok, object, meta} end - @updatable_object_types ["Note", "Question"] defp handle_update_object( %{data: %{"type" => "Update", "object" => updated_object}} = object, meta @@ -429,7 +428,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do meta[:object_data] end - if orig_object_data["type"] in @updatable_object_types do + if orig_object_data["type"] in Pleroma.Constants.updatable_object_types() do %{ updated_data: updated_object_data, updated: updated, diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 5750396a4..cccee342f 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -687,6 +687,24 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> strip_internal_fields |> strip_internal_tags |> set_type + |> maybe_process_history + end + + defp maybe_process_history(%{"formerRepresentations" => %{"orderedItems" => history}} = object) do + processed_history = + Enum.map( + history, + fn + item when is_map(item) -> prepare_object(item) + item -> item + end + ) + + put_in(object, ["formerRepresentations", "orderedItems"], processed_history) + end + + defp maybe_process_history(object) do + object end # @doc @@ -711,6 +729,21 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do {:ok, data} end + def prepare_outgoing(%{"type" => "Update", "object" => %{"type" => objtype} = object} = data) + when objtype in Pleroma.Constants.updatable_object_types() do + object = + object + |> prepare_object + + data = + data + |> Map.put("object", object) + |> Map.merge(Utils.make_json_ld_header()) + |> Map.delete("bcc") + + {:ok, data} + end + def prepare_outgoing(%{"type" => "Announce", "actor" => ap_id, "object" => object_id} = data) do object = object_id @@ -902,24 +935,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def strip_internal_fields(object) do - outer = Map.drop(object, Pleroma.Constants.object_internal_fields()) - - case outer do - %{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> - update_in( - outer["formerRepresentations"]["orderedItems"], - &Enum.map( - &1, - fn - item when is_map(item) -> Map.drop(item, Pleroma.Constants.object_internal_fields()) - item -> item - end - ) - ) - - _ -> - outer - end + Map.drop(object, Pleroma.Constants.object_internal_fields()) end defp strip_internal_tags(%{"tag" => tags} = object) do diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index dae07cf21..6520eabc9 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -312,6 +312,28 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert url == "http://localhost:4001/emoji/dino%20walking.gif" end + + test "Updates of Notes are handled" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "everybody do the dinosaur :dinosaur:"}) + {:ok, update} = CommonAPI.update(user, activity, %{status: "mew mew :blank:"}) + + {:ok, prepared} = Transmogrifier.prepare_outgoing(update.data) + + assert %{ + "content" => "mew mew :blank:", + "tag" => [%{"name" => ":blank:", "type" => "Emoji"}], + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "content" => "everybody do the dinosaur :dinosaur:", + "tag" => [%{"name" => ":dinosaur:", "type" => "Emoji"}] + } + ] + } + } = prepared["object"] + end end describe "user upgrade" do @@ -576,57 +598,42 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end end - describe "strip_internal_fields/1" do - test "it strips internal fields in formerRepresentations" do + describe "prepare_object/1" do + test "it processes history" do original = %{ "formerRepresentations" => %{ "orderedItems" => [ - %{"generator" => %{}} + %{ + "generator" => %{}, + "emoji" => %{"blobcat" => "http://localhost:4001/emoji/blobcat.png"} + } ] } } - stripped = Transmogrifier.strip_internal_fields(original) + processed = Transmogrifier.prepare_object(original) - refute Map.has_key?( - Enum.at(stripped["formerRepresentations"]["orderedItems"], 0), - "generator" - ) + history_item = Enum.at(processed["formerRepresentations"]["orderedItems"], 0) + + refute Map.has_key?(history_item, "generator") + + assert [%{"name" => ":blobcat:"}] = history_item["tag"] end - test "it strips internal fields in maybe badly-formed formerRepresentations" do - original = %{ - "formerRepresentations" => %{ - "orderedItems" => [ - %{"generator" => %{}}, - "https://example.com/1" - ] - } - } - - stripped = Transmogrifier.strip_internal_fields(original) - - refute Map.has_key?( - Enum.at(stripped["formerRepresentations"]["orderedItems"], 0), - "generator" - ) - - assert Enum.at(stripped["formerRepresentations"]["orderedItems"], 1) == - "https://example.com/1" - end - - test "it ignores if formerRepresentations does not look like an OrderedCollection" do + test "it works when there is no or bad history" do original = %{ "formerRepresentations" => %{ "items" => [ - %{"generator" => %{}} + %{ + "generator" => %{}, + "emoji" => %{"blobcat" => "http://localhost:4001/emoji/blobcat.png"} + } ] } } - stripped = Transmogrifier.strip_internal_fields(original) - - assert Map.has_key?(Enum.at(stripped["formerRepresentations"]["items"], 0), "generator") + processed = Transmogrifier.prepare_object(original) + assert processed["formerRepresentations"] == original["formerRepresentations"] end end end diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 32d6562d7..842c75e21 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1584,5 +1584,26 @@ defmodule Pleroma.Web.CommonAPITest do assert updated_object.data["content"] == "updated 2 :#{emoji2}:" assert %{^emoji2 => _} = updated_object.data["emoji"] end + + test "updates a post with emoji and federate properly" do + [{emoji1, _}, {emoji2, _} | _] = Pleroma.Emoji.get_all() + + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{status: "foo1", spoiler_text: "title 1 :#{emoji1}:"}) + + clear_config([:instance, :federating], true) + + with_mock Pleroma.Web.Federator, + publish: fn p -> nil end do + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2 :#{emoji2}:"}) + + assert updated.data["object"]["content"] == "updated 2 :#{emoji2}:" + assert %{^emoji2 => _} = updated.data["object"]["emoji"] + + assert called(Pleroma.Web.Federator.publish(updated)) + end + end end end From 4367489a3e8eb8682d717014eea9092d7679c070 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 3 Jul 2022 20:02:52 -0400 Subject: [PATCH 049/123] Pass history items through ObjectValidator for updatable object types --- .../web/activity_pub/object_validator.ex | 69 +++++++++++++++++-- .../article_note_page_validator_test.exs | 44 ++++++++++++ 2 files changed, 106 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index d4bf9c31e..12278a46b 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -128,15 +128,20 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end with {:ok, object} <- - object - |> validator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object) + do_separate_with_history(object, fn object -> + with {:ok, object} <- + object + |> validator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) - # Insert copy of hashtags as strings for the non-hashtag table indexing - tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object}) - object = Map.put(object, "tag", tag) + # Insert copy of hashtags as strings for the non-hashtag table indexing + tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object}) + object = Map.put(object, "tag", tag) + {:ok, object} + end + end) do {:ok, object, meta} end end @@ -262,4 +267,54 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do Object.normalize(object["object"], fetch: true) :ok end + + defp for_each_history_item( + %{"type" => "OrderedCollection", "orderedItems" => items} = history, + object, + fun + ) do + processed_items = + Enum.map(items, fn item -> + with item <- Map.put(item, "id", object["id"]), + {:ok, item} <- fun.(item) do + item + else + _ -> nil + end + end) + + if Enum.all?(processed_items, &(not is_nil(&1))) do + {:ok, Map.put(history, "orderedItems", processed_items)} + else + {:error, :invalid_history} + end + end + + defp for_each_history_item(nil, _object, _fun) do + {:ok, nil} + end + + defp for_each_history_item(_, _object, _fun) do + {:error, :invalid_history} + end + + # fun is (object -> {:ok, validated_object_with_string_keys}) + defp do_separate_with_history(object, fun) do + with history <- object["formerRepresentations"], + object <- Map.drop(object, ["formerRepresentations"]), + {_, {:ok, object}} <- {:main_body, fun.(object)}, + {_, {:ok, history}} <- {:history_items, for_each_history_item(history, object, fun)} do + object = + if history do + Map.put(object, "formerRepresentations", history) + else + object + end + + {:ok, object} + else + {:main_body, e} -> e + {:history_items, e} -> e + end + end end diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index c87b07547..11195c40d 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest do use Pleroma.DataCase, async: true + alias Pleroma.Web.ActivityPub.ObjectValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator alias Pleroma.Web.ActivityPub.Utils @@ -38,6 +39,49 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest end end + describe "Note with history" do + setup do + user = insert(:user) + {:ok, activity} = Pleroma.Web.CommonAPI.post(user, %{status: "mew mew :dinosaur:"}) + {:ok, edit} = Pleroma.Web.CommonAPI.update(user, activity, %{status: "edited :blank:"}) + + {:ok, %{"object" => external_rep}} = + Pleroma.Web.ActivityPub.Transmogrifier.prepare_outgoing(edit.data) + + %{external_rep: external_rep} + end + + test "edited note", %{external_rep: external_rep} do + assert %{"formerRepresentations" => %{"orderedItems" => [%{"tag" => [_]}]}} = external_rep + + {:ok, validate_res, []} = ObjectValidator.validate(external_rep, []) + + assert %{"formerRepresentations" => %{"orderedItems" => [%{"emoji" => %{"dinosaur" => _}}]}} = + validate_res + end + + test "edited note, badly-formed formerRepresentations", %{external_rep: external_rep} do + external_rep = Map.put(external_rep, "formerRepresentations", %{}) + + assert {:error, _} = ObjectValidator.validate(external_rep, []) + end + + test "edited note, badly-formed history item", %{external_rep: external_rep} do + history_item = + Enum.at(external_rep["formerRepresentations"]["orderedItems"], 0) + |> Map.put("type", "Foo") + + external_rep = + put_in( + external_rep, + ["formerRepresentations", "orderedItems"], + [history_item] + ) + + assert {:error, _} = ObjectValidator.validate(external_rep, []) + end + end + test "a Note from Roadhouse validates" do insert(:user, ap_id: "https://macgirvin.com/channel/mike") From 5ce118d970d3d7a2a5dd0a3719feb1d53be6b5ae Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 3 Jul 2022 20:19:50 -0400 Subject: [PATCH 050/123] Validate object data for incoming Update activities In Create validator we do not validate the object data, but that is because the object itself will go through the pipeline again, which is not the case for Update. Thus, we added validation for objects in Update activities. --- .../web/activity_pub/object_validator.ex | 7 +++- .../update_handling_test.exs | 38 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 12278a46b..3ccb4a3d6 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -152,8 +152,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do ) when objtype in ~w[Question Answer Audio Video Event Article Note Page] do with {_, false} <- {:local, Access.get(meta, :local, false)}, - {:ok, object_data} <- cast_and_apply(object), - meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), + {_, {:ok, object_data, _}} <- {:object_validation, validate(object, meta)}, + meta = Keyword.put(meta, :object_data, object_data), {:ok, update_activity} <- update_activity |> UpdateValidator.cast_and_validate() @@ -169,6 +169,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do object = stringify_keys(object) {:ok, object, meta} end + + {:object_validation, e} -> + e end end diff --git a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs index 198c35cd3..a09dbf5c6 100644 --- a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs @@ -127,4 +127,42 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateHandlingTest do assert meta[:object_data] end end + + describe "update with history" do + setup do + user = insert(:user) + {:ok, activity} = Pleroma.Web.CommonAPI.post(user, %{status: "mew mew :dinosaur:"}) + {:ok, edit} = Pleroma.Web.CommonAPI.update(user, activity, %{status: "edited :blank:"}) + {:ok, external_rep} = Pleroma.Web.ActivityPub.Transmogrifier.prepare_outgoing(edit.data) + %{external_rep: external_rep} + end + + test "edited note", %{external_rep: external_rep} do + {:ok, _validate_res, meta} = ObjectValidator.validate(external_rep, []) + + assert %{"formerRepresentations" => %{"orderedItems" => [%{"emoji" => %{"dinosaur" => _}}]}} = + meta[:object_data] + end + + test "edited note, badly-formed formerRepresentations", %{external_rep: external_rep} do + external_rep = put_in(external_rep, ["object", "formerRepresentations"], %{}) + + assert {:error, _} = ObjectValidator.validate(external_rep, []) + end + + test "edited note, badly-formed history item", %{external_rep: external_rep} do + history_item = + Enum.at(external_rep["object"]["formerRepresentations"]["orderedItems"], 0) + |> Map.put("type", "Foo") + + external_rep = + put_in( + external_rep, + ["object", "formerRepresentations", "orderedItems"], + [history_item] + ) + + assert {:error, _} = ObjectValidator.validate(external_rep, []) + end + end end From f84ed44cea1e5793dd899c74c38336a1721889e6 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 6 Jul 2022 01:19:53 -0400 Subject: [PATCH 051/123] Fix cannot get full history on object fetch --- .../web/activity_pub/object_validator.ex | 13 ++- test/pleroma/object/fetcher_test.exs | 80 +++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 3ccb4a3d6..9f446100d 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -103,8 +103,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do meta ) when objtype in ~w[Question Answer Audio Video Event Article Note Page] do - with {:ok, object_data} <- cast_and_apply(object), - meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), + with {:ok, object_data} <- cast_and_apply_and_stringify_with_history(object), + meta = Keyword.put(meta, :object_data, object_data), {:ok, create_activity} <- create_activity |> CreateGenericValidator.cast_and_validate(meta) @@ -212,6 +212,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def validate(o, m), do: {:error, {:validator_not_set, {o, m}}} + def cast_and_apply_and_stringify_with_history(object) do + do_separate_with_history(object, fn object -> + with {:ok, object_data} <- cast_and_apply(object), + object_data <- object_data |> stringify_keys() do + {:ok, object_data} + end + end) + end + def cast_and_apply(%{"type" => "ChatMessage"} = object) do ChatMessageValidator.cast_and_apply(object) end diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index 5a79e064f..1f97f36f7 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -456,4 +456,84 @@ defmodule Pleroma.Object.FetcherTest do } = refetched.data end end + + describe "fetch with history" do + setup do + object2 = %{ + "id" => "https://mastodon.social/2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "type" => "Note", + "content" => "test 2", + "bcc" => [], + "bto" => [], + "cc" => ["https://mastodon.social/users/emelie/followers"], + "to" => [], + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "orig 2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "bcc" => [], + "bto" => [], + "cc" => ["https://mastodon.social/users/emelie/followers"], + "to" => [], + "summary" => "" + } + ], + "totalItems" => 1 + } + } + + mock(fn + %{ + method: :get, + url: "https://mastodon.social/2" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: Jason.encode!(object2) + } + + %{ + method: :get, + url: "https://mastodon.social/users/emelie/collections/featured" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: + Jason.encode!(%{ + "id" => "https://mastodon.social/users/emelie/collections/featured", + "type" => "OrderedCollection", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "orderedItems" => [], + "totalItems" => 0 + }) + } + + env -> + apply(HttpRequestMock, :request, [env]) + end) + + %{object2: object2} + end + + test "it gets history", %{object2: object2} do + {:ok, object} = Fetcher.fetch_object_from_id(object2["id"]) + + assert %{ + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [%{}] + } + } = object.data + end + end end From 069554e9253a47f99225e12cc0ee99700fb89c6e Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Thu, 7 Jul 2022 15:11:29 -0400 Subject: [PATCH 052/123] Guard against outdated Updates It is possible for an earlier Update to be received by us later. For this, we now (1) only allows Updates to poll counts if there is no updated field, or the updated field is the same as the last updated date or creation date; (2) does not allow updating anything if the updated field is older than the last updated date or creation date; (3) allows updating updatable fields otherwise (normal updates); (4) if only the updated field is changed, it does not create a new history item on its own. --- lib/pleroma/object/updater.ex | 65 ++++++++++--- .../web/activity_pub/side_effects_test.exs | 96 ++++++++++++++++++- test/pleroma/web/common_api_test.exs | 2 +- 3 files changed, 145 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index 0b21f6c99..3d34c3f27 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -10,7 +10,10 @@ defmodule Pleroma.Object.Updater do |> Enum.reduce( %{data: orig_object_data, updated: false}, fn field, %{data: data, updated: updated} -> - updated = updated or Map.get(updated_object, field) != Map.get(orig_object_data, field) + updated = + updated or + (field != "updated" and + Map.get(updated_object, field) != Map.get(orig_object_data, field)) data = if Map.has_key?(updated_object, field) do @@ -136,21 +139,57 @@ defmodule Pleroma.Object.Updater do # This calculates the data of the new Object from an Update. # new_data's formerRepresentations is considered. def make_new_object_data_from_update_object(original_data, new_data) do - %{data: updated_data, updated: updated} = - original_data - |> update_content_fields(new_data) + update_is_reasonable = + with {_, updated} when not is_nil(updated) <- {:cur_updated, new_data["updated"]}, + {_, {:ok, updated_time, _}} <- {:cur_updated, DateTime.from_iso8601(updated)}, + {_, last_updated} when not is_nil(last_updated) <- + {:last_updated, original_data["updated"] || original_data["published"]}, + {_, {:ok, last_updated_time, _}} <- + {:last_updated, DateTime.from_iso8601(last_updated)}, + :gt <- DateTime.compare(updated_time, last_updated_time) do + :update_everything + else + # only allow poll updates + {:cur_updated, _} -> :no_content_update + :eq -> :no_content_update + # allow all updates + {:last_updated, _} -> :update_everything + # allow no updates + _ -> false + end - %{updated_object: updated_data, used_history_in_new_object?: used_history_in_new_object?} = - updated_data - |> maybe_update_history(original_data, - updated: updated, - use_history_in_new_object?: true, - new_data: new_data - ) + %{ + updated_object: updated_data, + used_history_in_new_object?: used_history_in_new_object?, + updated: updated + } = + if update_is_reasonable == :update_everything do + %{data: updated_data, updated: updated} = + original_data + |> update_content_fields(new_data) + + updated_data + |> maybe_update_history(original_data, + updated: updated, + use_history_in_new_object?: true, + new_data: new_data + ) + |> Map.put(:updated, updated) + else + %{ + updated_object: original_data, + used_history_in_new_object?: false, + updated: false + } + end updated_data = - updated_data - |> maybe_update_poll(new_data) + if update_is_reasonable != false do + updated_data + |> maybe_update_poll(new_data) + else + updated_data + end %{ updated_data: updated_data, diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 8e84db774..3b313b769 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -142,14 +142,19 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do describe "update notes" do setup do + make_time = fn -> + Pleroma.Web.ActivityPub.Utils.make_date() + end + user = insert(:user) - note = insert(:note, user: user) + note = insert(:note, user: user, data: %{"published" => make_time.()}) _note_activity = insert(:note_activity, note: note) updated_note = note.data |> Map.put("summary", "edited summary") |> Map.put("content", "edited content") + |> Map.put("updated", make_time.()) {:ok, update_data, []} = Builder.update(user, updated_note) {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) @@ -170,8 +175,69 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do updated_note: updated_note } do {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + updated_time = updated_note["updated"] + new_note = Pleroma.Object.get_by_id(object_id) - assert %{"summary" => "edited summary", "content" => "edited content"} = new_note.data + + assert %{ + "summary" => "edited summary", + "content" => "edited content", + "updated" => ^updated_time + } = new_note.data + end + + test "it rejects updates with no updated attribute in object", %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + old_note = Pleroma.Object.get_by_id(object_id) + updated_note = Map.drop(updated_note, ["updated"]) + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + new_note = Pleroma.Object.get_by_id(object_id) + assert old_note.data == new_note.data + end + + test "it rejects updates with updated attribute older than what we have in the original object", + %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + old_note = Pleroma.Object.get_by_id(object_id) + {:ok, creation_time, _} = DateTime.from_iso8601(old_note.data["published"]) + + updated_note = + Map.put(updated_note, "updated", DateTime.to_iso8601(DateTime.add(creation_time, -10))) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + new_note = Pleroma.Object.get_by_id(object_id) + assert old_note.data == new_note.data + end + + test "it rejects updates with updated attribute older than the last Update", %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + old_note = Pleroma.Object.get_by_id(object_id) + {:ok, creation_time, _} = DateTime.from_iso8601(old_note.data["published"]) + + updated_note = + Map.put(updated_note, "updated", DateTime.to_iso8601(DateTime.add(creation_time, +10))) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + + old_note = Pleroma.Object.get_by_id(object_id) + {:ok, update_time, _} = DateTime.from_iso8601(old_note.data["updated"]) + + updated_note = + Map.put(updated_note, "updated", DateTime.to_iso8601(DateTime.add(update_time, -5))) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + + new_note = Pleroma.Object.get_by_id(object_id) + assert old_note.data == new_note.data end test "it updates using object_data", %{ @@ -215,6 +281,14 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do note.data |> Map.put("summary", "edited summary 2") |> Map.put("content", "edited content 2") + |> Map.put( + "updated", + first_edit["updated"] + |> DateTime.from_iso8601() + |> elem(1) + |> DateTime.add(10) + |> DateTime.to_iso8601() + ) {:ok, second_update_data, []} = Builder.update(user, second_updated_note) {:ok, update, _meta} = ActivityPub.persist(second_update_data, local: true) @@ -238,7 +312,18 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do updated_note: updated_note } do {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) - %{data: _first_edit} = Pleroma.Object.get_by_id(object_id) + %{data: first_edit} = Pleroma.Object.get_by_id(object_id) + + updated_note = + updated_note + |> Map.put( + "updated", + first_edit["updated"] + |> DateTime.from_iso8601() + |> elem(1) + |> DateTime.add(10) + |> DateTime.to_iso8601() + ) {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) %{data: new_note} = Pleroma.Object.get_by_id(object_id) @@ -270,7 +355,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do data["oneOf"] |> Enum.map(fn choice -> put_in(choice, ["replies", "totalItems"], 5) end) - updated_question = data |> Map.put("oneOf", new_choices) + updated_question = + data + |> Map.put("oneOf", new_choices) + |> Map.put("updated", Pleroma.Web.ActivityPub.Utils.make_date()) {:ok, update_data, []} = Builder.update(user, updated_question) {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 842c75e21..24f3a1a93 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1596,7 +1596,7 @@ defmodule Pleroma.Web.CommonAPITest do clear_config([:instance, :federating], true) with_mock Pleroma.Web.Federator, - publish: fn p -> nil end do + publish: fn _p -> nil end do {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2 :#{emoji2}:"}) assert updated.data["object"]["content"] == "updated 2 :#{emoji2}:" From 11a6e8842079d8f29417e0de31fda85c7b7cb1be Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Thu, 7 Jul 2022 15:22:04 -0400 Subject: [PATCH 053/123] Test that Question updates are viable --- .../web/activity_pub/side_effects_test.exs | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 3b313b769..0db3a8ea6 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -341,7 +341,12 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do describe "update questions" do setup do user = insert(:user) - question = insert(:question, user: user) + + question = + insert(:question, + user: user, + data: %{"published" => Pleroma.Web.ActivityPub.Utils.make_date()} + ) %{user: user, data: question.data, id: question.id} end @@ -372,6 +377,59 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do refute Map.has_key?(new_question, "formerRepresentations") end + + test "allows updating choice count without updated field", %{ + user: user, + data: data, + id: id + } do + new_choices = + data["oneOf"] + |> Enum.map(fn choice -> put_in(choice, ["replies", "totalItems"], 5) end) + + updated_question = + data + |> Map.put("oneOf", new_choices) + + {:ok, update_data, []} = Builder.update(user, updated_question) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_question) + + %{data: new_question} = Pleroma.Object.get_by_id(id) + + assert [%{"replies" => %{"totalItems" => 5}}, %{"replies" => %{"totalItems" => 5}}] = + new_question["oneOf"] + + refute Map.has_key?(new_question, "formerRepresentations") + end + + test "allows updating choice count with updated field same as the creation date", %{ + user: user, + data: data, + id: id + } do + new_choices = + data["oneOf"] + |> Enum.map(fn choice -> put_in(choice, ["replies", "totalItems"], 5) end) + + updated_question = + data + |> Map.put("oneOf", new_choices) + |> Map.put("updated", data["published"]) + + {:ok, update_data, []} = Builder.update(user, updated_question) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_question) + + %{data: new_question} = Pleroma.Object.get_by_id(id) + + assert [%{"replies" => %{"totalItems" => 5}}, %{"replies" => %{"totalItems" => 5}}] = + new_question["oneOf"] + + refute Map.has_key?(new_question, "formerRepresentations") + end end describe "EmojiReact objects" do From 26080b4b5c9d10887f8c52fbd76cba5b22838632 Mon Sep 17 00:00:00 2001 From: Ilja Date: Fri, 8 Jul 2022 07:32:47 +0200 Subject: [PATCH 054/123] Fix rate_limiter_test.exs test "it restricts based on config values" It used a timer to sleep. But time also goes on when doing other things, so depending on hardware, the timings could be off. I slightly changed the tests so we still test what we functionally want. Instead of waiting until the cache expires I now have a function to expire the test and use that. That means we're not testing any more if the cache really expires after a certain amount of time, but that's the responsability of the dependency imo, so shouldn't be a problem. I also changed `Pleroma.Web.Endpoint, :http, :ip` to `127.0.0.1` because that's the setting people typically have, and I see no reason to do it differently. Especially since it's an exernal ip, which may come over as weird or suspicious to people. --- test/pleroma/web/plugs/rate_limiter_test.exs | 44 +++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/test/pleroma/web/plugs/rate_limiter_test.exs b/test/pleroma/web/plugs/rate_limiter_test.exs index b1ac76120..19cee8aee 100644 --- a/test/pleroma/web/plugs/rate_limiter_test.exs +++ b/test/pleroma/web/plugs/rate_limiter_test.exs @@ -48,38 +48,42 @@ defmodule Pleroma.Web.Plugs.RateLimiterTest do refute RateLimiter.disabled?(build_conn()) end - @tag :erratic test "it restricts based on config values" do limiter_name = :test_plug_opts scale = 80 limit = 5 - clear_config([Pleroma.Web.Endpoint, :http, :ip], {8, 8, 8, 8}) + clear_config([Pleroma.Web.Endpoint, :http, :ip], {127, 0, 0, 1}) clear_config([:rate_limit, limiter_name], {scale, limit}) plug_opts = RateLimiter.init(name: limiter_name) conn = build_conn(:get, "/") - for i <- 1..5 do - conn = RateLimiter.call(conn, plug_opts) - assert {^i, _} = RateLimiter.inspect_bucket(conn, limiter_name, plug_opts) - Process.sleep(10) + for _ <- 1..5 do + conn_limited = RateLimiter.call(conn, plug_opts) + + refute conn_limited.status == Conn.Status.code(:too_many_requests) + refute conn_limited.resp_body + refute conn_limited.halted end - conn = RateLimiter.call(conn, plug_opts) - assert %{"error" => "Throttled"} = ConnTest.json_response(conn, :too_many_requests) - assert conn.halted + conn_limited = RateLimiter.call(conn, plug_opts) + assert %{"error" => "Throttled"} = ConnTest.json_response(conn_limited, :too_many_requests) + assert conn_limited.halted - Process.sleep(50) + expire_ttl(conn, limiter_name) - conn = build_conn(:get, "/") + for _ <- 1..5 do + conn_limited = RateLimiter.call(conn, plug_opts) - conn = RateLimiter.call(conn, plug_opts) - assert {1, 4} = RateLimiter.inspect_bucket(conn, limiter_name, plug_opts) + refute conn_limited.status == Conn.Status.code(:too_many_requests) + refute conn_limited.resp_body + refute conn_limited.halted + end - refute conn.status == Conn.Status.code(:too_many_requests) - refute conn.resp_body - refute conn.halted + conn_limited = RateLimiter.call(conn, plug_opts) + assert %{"error" => "Throttled"} = ConnTest.json_response(conn_limited, :too_many_requests) + assert conn_limited.halted end describe "options" do @@ -263,4 +267,12 @@ defmodule Pleroma.Web.Plugs.RateLimiterTest do refute {:err, :not_found} == RateLimiter.inspect_bucket(conn, limiter_name, opts) end + + def expire_ttl(%{remote_ip: remote_ip} = _conn, bucket_name_root) do + bucket_name = "anon:#{bucket_name_root}" |> String.to_atom() + key_name = "ip::#{remote_ip |> Tuple.to_list() |> Enum.join(".")}" + + {:ok, bucket_value} = Cachex.get(bucket_name, key_name) + Cachex.put(bucket_name, key_name, bucket_value, ttl: -1) + end end From 04ded94a50fbabb194ab9e9c5cf8f08937f85d64 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 9 Jul 2022 18:00:42 -0400 Subject: [PATCH 055/123] Fix remote emoji in subject disappearing after edits --- lib/pleroma/web/common_api.ex | 9 +++++- test/pleroma/web/common_api_test.exs | 42 ++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index e5a78c102..89f5dd606 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -415,7 +415,14 @@ defmodule Pleroma.Web.CommonAPI do defp make_update_data(user, orig_object, changes) do kept_params = %{ - visibility: Visibility.get_visibility(orig_object) + visibility: Visibility.get_visibility(orig_object), + in_reply_to_id: + with replied_id when is_binary(replied_id) <- orig_object.data["inReplyTo"], + %Activity{id: activity_id} <- Activity.get_create_by_object_ap_id(replied_id) do + activity_id + else + _ -> nil + end } params = Map.merge(changes, kept_params) diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 24f3a1a93..fcfcb9a2e 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1605,5 +1605,47 @@ defmodule Pleroma.Web.CommonAPITest do assert called(Pleroma.Web.Federator.publish(updated)) end end + + test "editing a post that copied a remote title with remote emoji should keep that emoji" do + remote_emoji_uri = "https://remote.org/emoji.png" + + note = + insert( + :note, + data: %{ + "summary" => ":remoteemoji:", + "emoji" => %{ + "remoteemoji" => remote_emoji_uri + }, + "tag" => [ + %{ + "type" => "Emoji", + "name" => "remoteemoji", + "icon" => %{"url" => remote_emoji_uri} + } + ] + } + ) + + note_activity = insert(:note_activity, note: note) + + user = insert(:user) + + {:ok, reply} = + CommonAPI.post(user, %{ + status: "reply", + spoiler_text: ":remoteemoji:", + in_reply_to_id: note_activity.id + }) + + assert reply.object.data["emoji"]["remoteemoji"] == remote_emoji_uri + + {:ok, edit} = + CommonAPI.update(user, reply, %{status: "reply mew mew", spoiler_text: ":remoteemoji:"}) + + edited_note = Pleroma.Object.normalize(edit) + + assert edited_note.data["emoji"]["remoteemoji"] == remote_emoji_uri + end end end From 28626eafc174e6707ab4020f72a5550446730da9 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Thu, 14 Jul 2022 13:35:33 +0200 Subject: [PATCH 056/123] Allow higher amount of restarts for Pleroma.Repo during testing This was done by floatingghost as part of a bigger commit in Akkoma. See . As explained in > there are so many caches that clearing them all can nuke the supervisor, which by default will become an hero if it gets more than 3 restarts in <5 seconds And further down the thread > essentially we've got like 11 caches (https://akkoma.dev/AkkomaGang/akkoma/src/commit/37ae047e1652c4089934434ec79f393c4c839122/lib/pleroma/application.ex#L165) > then in test we fetch them all (https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/test/support/data_case.ex#L50) and call clear on them > so if this clear fails on any 3 of them, the pleroma supervisor itself will die How it fails? > idk maybe cachex dies, maybe :ets does a weird thing > it doesn't log anything, it just consistently dies during cache clearing so i figured it had to be that > honestly my best bet is locksmith and queuing > https://github.com/whitfin/cachex/blob/master/lib/cachex/actions/clear.ex#L26 > clear is thrown into a locksmith transaction > locksmith says > >If the process is already in a transactional context, the provided function will be executed immediately. Otherwise the required keys will be locked until the provided function has finished executing. > so if we get 2 clears too close together, maybe it locks, then doesn't like the next clear? --- lib/pleroma/application.ex | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index d808bc732..ae3ef9738 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -112,7 +112,17 @@ defmodule Pleroma.Application do # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options - opts = [strategy: :one_for_one, name: Pleroma.Supervisor] + # If we have a lot of caches, default max_restarts can cause test + # resets to fail. + # Go for the default 3 unless we're in test + max_restarts = + if @mix_env == :test do + 100 + else + 3 + end + + opts = [strategy: :one_for_one, name: Pleroma.Supervisor, max_restarts: max_restarts] result = Supervisor.start_link(children, opts) set_postgres_server_version() From 8371fd8ca20d7aaa16e082fd7ed39d603b9731d1 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 16 Jul 2022 01:20:25 -0400 Subject: [PATCH 057/123] Implement settings api --- .../operations/pleroma_settings_operation.ex | 72 ++++++++++ .../controllers/settings_controller.ex | 79 +++++++++++ lib/pleroma/web/router.ex | 7 + .../controllers/settings_controller_test.exs | 126 ++++++++++++++++++ 4 files changed, 284 insertions(+) create mode 100644 lib/pleroma/web/api_spec/operations/pleroma_settings_operation.ex create mode 100644 lib/pleroma/web/pleroma_api/controllers/settings_controller.ex create mode 100644 test/pleroma/web/pleroma_api/controllers/settings_controller_test.exs diff --git a/lib/pleroma/web/api_spec/operations/pleroma_settings_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_settings_operation.ex new file mode 100644 index 000000000..e2cef4f67 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/pleroma_settings_operation.ex @@ -0,0 +1,72 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.PleromaSettingsOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def show_operation do + %Operation{ + tags: ["Settings"], + summary: "Get settings for an application", + description: "Get synchronized settings for an application", + operationId: "SettingsController.show", + parameters: [app_name_param()], + security: [%{"oAuth" => ["read:accounts"]}], + responses: %{ + 200 => Operation.response("object", "application/json", object()) + } + } + end + + def update_operation do + %Operation{ + tags: ["Settings"], + summary: "Update settings for an application", + description: "Update synchronized settings for an application", + operationId: "SettingsController.update", + parameters: [app_name_param()], + security: [%{"oAuth" => ["write:accounts"]}], + requestBody: request_body("Parameters", update_request(), required: true), + responses: %{ + 200 => Operation.response("object", "application/json", object()) + } + } + end + + def app_name_param do + Operation.parameter(:app, :path, %Schema{type: :string}, "Application name", + example: "pleroma-fe", + required: true + ) + end + + def object do + %Schema{ + title: "Settings object", + description: "The object that contains settings for the application.", + type: :object + } + end + + def update_request do + %Schema{ + title: "SettingsUpdateRequest", + type: :object, + description: + "The settings object to be merged with the current settings. To remove a field, set it to null.", + example: %{ + "config1" => true, + "config2_to_unset" => nil + } + } + end +end diff --git a/lib/pleroma/web/pleroma_api/controllers/settings_controller.ex b/lib/pleroma/web/pleroma_api/controllers/settings_controller.ex new file mode 100644 index 000000000..1136575b6 --- /dev/null +++ b/lib/pleroma/web/pleroma_api/controllers/settings_controller.ex @@ -0,0 +1,79 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.SettingsController do + use Pleroma.Web, :controller + + alias Pleroma.Web.Plugs.OAuthScopesPlug + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["write:accounts"]} when action in [:update] + ) + + plug( + OAuthScopesPlug, + %{scopes: ["read:accounts"]} when action in [:show] + ) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaSettingsOperation + + @doc "GET /api/v1/pleroma/settings/:app" + def show(%{assigns: %{user: user}} = conn, %{app: app} = _params) do + conn + |> json(get_settings(user, app)) + end + + @doc "PATCH /api/v1/pleroma/settings/:app" + def update(%{assigns: %{user: user}, body_params: body_params} = conn, %{app: app} = _params) do + settings = + get_settings(user, app) + |> merge_recursively(body_params) + + with changeset <- + Pleroma.User.update_changeset( + user, + %{pleroma_settings_store: %{app => settings}} + ), + {:ok, _} <- Pleroma.Repo.update(changeset) do + conn + |> json(settings) + end + end + + defp merge_recursively(old, %{} = new) do + old = ensure_object(old) + + Enum.reduce( + new, + old, + fn + {k, nil}, acc -> + Map.drop(acc, [k]) + + {k, %{} = new_child}, acc -> + Map.put(acc, k, merge_recursively(acc[k], new_child)) + + {k, v}, acc -> + Map.put(acc, k, v) + end + ) + end + + defp get_settings(user, app) do + user.pleroma_settings_store + |> Map.get(app, %{}) + |> ensure_object() + end + + defp ensure_object(%{} = object) do + object + end + + defp ensure_object(_) do + %{} + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 7bbc20275..9023b9800 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -458,6 +458,13 @@ defmodule Pleroma.Web.Router do get("/birthdays", AccountController, :birthdays) end + scope [] do + pipe_through(:authenticated_api) + + get("/settings/:app", SettingsController, :show) + patch("/settings/:app", SettingsController, :update) + end + post("/accounts/confirmation_resend", AccountController, :confirmation_resend) end diff --git a/test/pleroma/web/pleroma_api/controllers/settings_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/settings_controller_test.exs new file mode 100644 index 000000000..e3c752d53 --- /dev/null +++ b/test/pleroma/web/pleroma_api/controllers/settings_controller_test.exs @@ -0,0 +1,126 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.SettingsControllerTest do + use Pleroma.Web.ConnCase + + import Pleroma.Factory + + describe "GET /api/v1/pleroma/settings/:app" do + setup do + oauth_access(["read:accounts"]) + end + + test "it gets empty settings", %{conn: conn} do + response = + conn + |> get("/api/v1/pleroma/settings/pleroma-fe") + |> json_response_and_validate_schema(:ok) + + assert response == %{} + end + + test "it gets settings", %{conn: conn, user: user} do + response = + conn + |> assign( + :user, + struct(user, + pleroma_settings_store: %{ + "pleroma-fe" => %{ + "foo" => "bar" + } + } + ) + ) + |> get("/api/v1/pleroma/settings/pleroma-fe") + |> json_response_and_validate_schema(:ok) + + assert %{"foo" => "bar"} == response + end + end + + describe "POST /api/v1/pleroma/settings/:app" do + setup do + settings = %{ + "foo" => "bar", + "nested" => %{ + "1" => "2" + } + } + + user = + insert( + :user, + %{ + pleroma_settings_store: %{ + "pleroma-fe" => settings + } + } + ) + + %{conn: conn} = oauth_access(["write:accounts"], user: user) + + %{conn: conn, user: user, settings: settings} + end + + test "it adds keys", %{conn: conn} do + response = + conn + |> put_req_header("content-type", "application/json") + |> patch("/api/v1/pleroma/settings/pleroma-fe", %{ + "foo" => "edited", + "bar" => "new", + "nested" => %{"3" => "4"} + }) + |> json_response_and_validate_schema(:ok) + + assert response == %{ + "foo" => "edited", + "bar" => "new", + "nested" => %{ + "1" => "2", + "3" => "4" + } + } + end + + test "it removes keys", %{conn: conn} do + response = + conn + |> put_req_header("content-type", "application/json") + |> patch("/api/v1/pleroma/settings/pleroma-fe", %{ + "foo" => nil, + "bar" => nil, + "nested" => %{ + "1" => nil, + "3" => nil + } + }) + |> json_response_and_validate_schema(:ok) + + assert response == %{ + "nested" => %{} + } + end + + test "it does not override settings for other apps", %{ + conn: conn, + user: user, + settings: settings + } do + conn + |> put_req_header("content-type", "application/json") + |> patch("/api/v1/pleroma/settings/admin-fe", %{"foo" => "bar"}) + |> json_response_and_validate_schema(:ok) + + user = Pleroma.User.get_by_id(user.id) + + assert user.pleroma_settings_store == %{ + "pleroma-fe" => settings, + "admin-fe" => %{"foo" => "bar"} + } + end + end +end From 8113dd31ee5d4d50c2b864d46242e1c3b6e889be Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 16 Jul 2022 01:27:16 -0400 Subject: [PATCH 058/123] Add api docs for settings endpoint --- docs/development/API/pleroma_api.md | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/development/API/pleroma_api.md b/docs/development/API/pleroma_api.md index 0d15384b9..09ffba2e6 100644 --- a/docs/development/API/pleroma_api.md +++ b/docs/development/API/pleroma_api.md @@ -695,3 +695,42 @@ Emoji reactions work a lot like favourites do. They make it possible to react to * Authentication: required * Params: none * Response: HTTP 200 on success, 500 on error + +## `/api/v1/pleroma/settings/:app` +### Gets settings for some application +* Method `GET` +* Authentication: `read:accounts` + +* Response: JSON. The settings for that application, or empty object if there is none. +* Example response: +```json +{ + "some key": "some value" +} +``` + +### Updates settings for some application +* Method `PATCH` +* Authentication: `write:accounts` +* Request body: JSON object. The object will be merged recursively with old settings. If some field is set to null, it is removed. +* Example request: +```json +{ + "some key": "some value", + "key to remove": null, + "nested field": { + "some key": "some value", + "key to remove": null + } +} +``` +* Response: JSON. Updated (merged) settings for that application. +* Example response: +```json +{ + "some key": "some value", + "nested field": { + "some key": "some value", + } +} +``` From 9a6280cdb9a39e85ea1434dec51cb4781eacb379 Mon Sep 17 00:00:00 2001 From: Ilja Date: Wed, 20 Jul 2022 13:17:44 +0200 Subject: [PATCH 059/123] Fix warnings ":logger is used by the current application but the current application does not depend on :logger" During compilation, we had the following warning which is now fixed ``` ==> restarter Compiling 1 file (.ex) warning: Logger.__do_log__/4 defined in application :logger is used by the current application but the current application does not depend on :logger. To fix this, you must do one of: 1. If :logger is part of Erlang/Elixir, you must include it under :extra_applications inside "def application" in your mix.exs 2. If :logger is a dependency, make sure it is listed under "def deps" in your mix.exs 3. In case you don't want to add a requirement to :logger, you may optionally skip this warning by adding [xref: [exclude: [Logger]]] to your "def project" in mix.exs Invalid call found at 2 locations: lib/pleroma.ex:65: Restarter.Pleroma.handle_cast/2 lib/pleroma.ex:78: Restarter.Pleroma.handle_cast/2 warning: Logger.__should_log__/2 defined in application :logger is used by the current application but the current application does not depend on :logger. To fix this, you must do one of: 1. If :logger is part of Erlang/Elixir, you must include it under :extra_applications inside "def application" in your mix.exs 2. If :logger is a dependency, make sure it is listed under "def deps" in your mix.exs 3. In case you don't want to add a requirement to :logger, you may optionally skip this warning by adding [xref: [exclude: [Logger]]] to your "def project" in mix.exs Invalid call found at 2 locations: lib/pleroma.ex:65: Restarter.Pleroma.handle_cast/2 lib/pleroma.ex:78: Restarter.Pleroma.handle_cast/2 warning: Logger.debug/1 defined in application :logger is used by the current application but the current application does not depend on :logger. To fix this, you must do one of: 1. If :logger is part of Erlang/Elixir, you must include it under :extra_applications inside "def application" in your mix.exs 2. If :logger is a dependency, make sure it is listed under "def deps" in your mix.exs 3. In case you don't want to add a requirement to :logger, you may optionally skip this warning by adding [xref: [exclude: [Logger]]] to your "def project" in mix.exs Invalid call found at 2 locations: lib/pleroma.ex:65: Restarter.Pleroma.handle_cast/2 lib/pleroma.ex:78: Restarter.Pleroma.handle_cast/2 ``` --- restarter/mix.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/restarter/mix.exs b/restarter/mix.exs index b0908aece..9f26f5f64 100644 --- a/restarter/mix.exs +++ b/restarter/mix.exs @@ -13,7 +13,8 @@ defmodule Restarter.MixProject do def application do [ - mod: {Restarter, []} + mod: {Restarter, []}, + extra_applications: [:logger] ] end From ba31af021ca82c6eb9685e967749620d8b8a44d9 Mon Sep 17 00:00:00 2001 From: Ilja Date: Wed, 20 Jul 2022 11:53:54 +0200 Subject: [PATCH 060/123] Fix flaky/erratic tests in Pleroma.Config.TransferTaskTest There were async calls happening, so they weren't always finished when assert happened. --- restarter/lib/pleroma.ex | 12 +++++ test/pleroma/config/transfer_task_test.exs | 61 ++++++++++++++++++---- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/restarter/lib/pleroma.ex b/restarter/lib/pleroma.ex index 149a569ce..a7186cec4 100644 --- a/restarter/lib/pleroma.ex +++ b/restarter/lib/pleroma.ex @@ -61,6 +61,12 @@ defmodule Restarter.Pleroma do {:noreply, @init_state} end + # Don't actually restart during tests. + # We just check if the correct call has been done. + # If we actually restart, we get errors during the tests like + # (RuntimeError) could not lookup Ecto repo Pleroma.Repo because it was not started or + # it does not exist + # See tests in Pleroma.Config.TransferTaskTest def handle_cast({:restart, :test, _}, state) do Logger.debug("pleroma manually restarted") {:noreply, Map.put(state, :need_reboot, false)} @@ -74,6 +80,12 @@ defmodule Restarter.Pleroma do def handle_cast({:after_boot, _}, %{after_boot: true} = state), do: {:noreply, state} + # Don't actually restart during tests. + # We just check if the correct call has been done. + # If we actually restart, we get errors during the tests like + # (RuntimeError) could not lookup Ecto repo Pleroma.Repo because it was not started or + # it does not exist + # See tests in Pleroma.Config.TransferTaskTest def handle_cast({:after_boot, :test}, state) do Logger.debug("pleroma restarted after boot") state = %{state | after_boot: true, rebooted: true} diff --git a/test/pleroma/config/transfer_task_test.exs b/test/pleroma/config/transfer_task_test.exs index 927744add..3dc917362 100644 --- a/test/pleroma/config/transfer_task_test.exs +++ b/test/pleroma/config/transfer_task_test.exs @@ -79,35 +79,70 @@ defmodule Pleroma.Config.TransferTaskTest do describe "pleroma restart" do setup do - on_exit(fn -> Restarter.Pleroma.refresh() end) + on_exit(fn -> + Restarter.Pleroma.refresh() + + # Restarter.Pleroma.refresh/0 is an asynchronous call. + # A GenServer will first finish the previous call before starting a new one. + # Here we do a synchronous call. + # That way we are sure that the previous call has finished before we continue. + # See https://stackoverflow.com/questions/51361856/how-to-use-task-await-with-genserver + Restarter.Pleroma.rebooted?() + end) end - @tag :erratic test "don't restart if no reboot time settings were changed" do clear_config(:emoji) insert(:config, key: :emoji, value: [groups: [a: 1, b: 2]]) refute String.contains?( - capture_log(fn -> TransferTask.start_link([]) end), + capture_log(fn -> + TransferTask.start_link([]) + + # TransferTask.start_link/1 is an asynchronous call. + # A GenServer will first finish the previous call before starting a new one. + # Here we do a synchronous call. + # That way we are sure that the previous call has finished before we continue. + Restarter.Pleroma.rebooted?() + end), "pleroma restarted" ) end - @tag :erratic test "on reboot time key" do clear_config(:shout) insert(:config, key: :shout, value: [enabled: false]) - assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted" + + # Note that we don't actually restart Pleroma. + # See module Restarter.Pleroma + assert capture_log(fn -> + TransferTask.start_link([]) + + # TransferTask.start_link/1 is an asynchronous call. + # A GenServer will first finish the previous call before starting a new one. + # Here we do a synchronous call. + # That way we are sure that the previous call has finished before we continue. + Restarter.Pleroma.rebooted?() + end) =~ "pleroma restarted" end - @tag :erratic test "on reboot time subkey" do clear_config(Pleroma.Captcha) insert(:config, key: Pleroma.Captcha, value: [seconds_valid: 60]) - assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted" + + # Note that we don't actually restart Pleroma. + # See module Restarter.Pleroma + assert capture_log(fn -> + TransferTask.start_link([]) + + # TransferTask.start_link/1 is an asynchronous call. + # A GenServer will first finish the previous call before starting a new one. + # Here we do a synchronous call. + # That way we are sure that the previous call has finished before we continue. + Restarter.Pleroma.rebooted?() + end) =~ "pleroma restarted" end - @tag :erratic test "don't restart pleroma on reboot time key and subkey if there is false flag" do clear_config(:shout) clear_config(Pleroma.Captcha) @@ -116,7 +151,15 @@ defmodule Pleroma.Config.TransferTaskTest do insert(:config, key: Pleroma.Captcha, value: [seconds_valid: 60]) refute String.contains?( - capture_log(fn -> TransferTask.load_and_update_env([], false) end), + capture_log(fn -> + TransferTask.load_and_update_env([], false) + + # TransferTask.start_link/1 is an asynchronous call. + # A GenServer will first finish the previous call before starting a new one. + # Here we do a synchronous call. + # That way we are sure that the previous call has finished before we continue. + Restarter.Pleroma.rebooted?() + end), "pleroma restarted" ) end From eba9b0760f294482823b9bd55a430979fc2d21af Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 23 Jul 2022 15:56:36 -0400 Subject: [PATCH 061/123] Make MRF Keyword history-aware --- lib/pleroma/object/updater.ex | 40 ++++++ .../web/activity_pub/mrf/keyword_policy.ex | 55 ++++++-- .../activity_pub/mrf/keyword_policy_test.exs | 131 ++++++++++++++++++ test/pleroma/web/common_api_test.exs | 17 +++ 4 files changed, 231 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index 3d34c3f27..6381320bd 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -197,4 +197,44 @@ defmodule Pleroma.Object.Updater do used_history_in_new_object?: used_history_in_new_object? } end + + defp for_each_history_item(%{"orderedItems" => items} = history, _object, fun) do + new_items = + Enum.map(items, fun) + |> Enum.reduce_while( + {:ok, []}, + fn + {:ok, item}, {:ok, acc} -> {:cont, {:ok, acc ++ [item]}} + e, _acc -> {:halt, e} + end + ) + + case new_items do + {:ok, items} -> {:ok, Map.put(history, "orderedItems", items)} + e -> e + end + end + + defp for_each_history_item(history, _, _) do + {:ok, history} + end + + def do_with_history(object, fun) do + with history <- object["formerRepresentations"], + object <- Map.drop(object, ["formerRepresentations"]), + {_, {:ok, object}} <- {:main_body, fun.(object)}, + {_, {:ok, history}} <- {:history_items, for_each_history_item(history, object, fun)} do + object = + if history do + Map.put(object, "formerRepresentations", history) + else + object + end + + {:ok, object} + else + {:main_body, e} -> e + {:history_items, e} -> e + end + end end diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index 00b64744f..687ec6c2f 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -27,24 +27,46 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do end defp check_reject(%{"object" => %{} = object} = message) do - payload = object_payload(object) + with {:ok, _new_object} <- + Pleroma.Object.Updater.do_with_history(object, fn object -> + payload = object_payload(object) - if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern -> - string_matches?(payload, pattern) - end) do - {:reject, "[KeywordPolicy] Matches with rejected keyword"} - else + if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern -> + string_matches?(payload, pattern) + end) do + {:reject, "[KeywordPolicy] Matches with rejected keyword"} + else + {:ok, message} + end + end) do {:ok, message} + else + e -> e end end - defp check_ftl_removal(%{"to" => to, "object" => %{} = object} = message) do - payload = object_payload(object) + defp check_ftl_removal(%{"type" => "Create", "to" => to, "object" => %{} = object} = message) do + check_keyword = fn object -> + payload = object_payload(object) - if Pleroma.Constants.as_public() in to and - Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern -> + if Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern -> string_matches?(payload, pattern) end) do + {:should_delist, nil} + else + {:ok, %{}} + end + end + + should_delist? = fn object -> + with {:ok, _} <- Pleroma.Object.Updater.do_with_history(object, check_keyword) do + false + else + _ -> true + end + end + + if Pleroma.Constants.as_public() in to and should_delist?.(object) do to = List.delete(to, Pleroma.Constants.as_public()) cc = [Pleroma.Constants.as_public() | message["cc"] || []] @@ -59,8 +81,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do end end + defp check_ftl_removal(message) do + {:ok, message} + end + defp check_replace(%{"object" => %{} = object} = message) do - object = + replace_kw = fn object -> ["content", "name", "summary"] |> Enum.filter(fn field -> Map.has_key?(object, field) && object[field] end) |> Enum.reduce(object, fn field, object -> @@ -73,6 +99,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do Map.put(object, field, data) end) + |> (fn object -> {:ok, object} end).() + end + + {:ok, object} = Pleroma.Object.Updater.do_with_history(object, replace_kw) message = Map.put(message, "object", object) @@ -80,7 +110,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do end @impl true - def filter(%{"type" => "Create", "object" => %{"content" => _content}} = message) do + def filter(%{"type" => type, "object" => %{"content" => _content}} = message) + when type in ["Create", "Update"] do with {:ok, message} <- check_reject(message), {:ok, message} <- check_ftl_removal(message), {:ok, message} <- check_replace(message) do diff --git a/test/pleroma/web/activity_pub/mrf/keyword_policy_test.exs b/test/pleroma/web/activity_pub/mrf/keyword_policy_test.exs index bfa8e8f59..a0e77d7b9 100644 --- a/test/pleroma/web/activity_pub/mrf/keyword_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/keyword_policy_test.exs @@ -79,6 +79,54 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicyTest do KeywordPolicy.filter(message) end) end + + test "rejects if string matches in history" do + clear_config([:mrf_keyword, :reject], ["pun"]) + + message = %{ + "type" => "Create", + "object" => %{ + "content" => "just a daily reminder that compLAINer is a good", + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "content" => "just a daily reminder that compLAINer is a good pun", + "summary" => "" + } + ] + } + } + } + + assert {:reject, "[KeywordPolicy] Matches with rejected keyword"} = + KeywordPolicy.filter(message) + end + + test "rejects Updates" do + clear_config([:mrf_keyword, :reject], ["pun"]) + + message = %{ + "type" => "Update", + "object" => %{ + "content" => "just a daily reminder that compLAINer is a good", + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "content" => "just a daily reminder that compLAINer is a good pun", + "summary" => "" + } + ] + } + } + } + + assert {:reject, "[KeywordPolicy] Matches with rejected keyword"} = + KeywordPolicy.filter(message) + end end describe "delisting from ftl based on keywords" do @@ -157,6 +205,31 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicyTest do not (["https://www.w3.org/ns/activitystreams#Public"] == result["to"]) end) end + + test "delists if string matches in history" do + clear_config([:mrf_keyword, :federated_timeline_removal], ["pun"]) + + message = %{ + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Create", + "object" => %{ + "content" => "just a daily reminder that compLAINer is a good", + "summary" => "", + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "content" => "just a daily reminder that compLAINer is a good pun", + "summary" => "" + } + ] + } + } + } + + {:ok, result} = KeywordPolicy.filter(message) + assert ["https://www.w3.org/ns/activitystreams#Public"] == result["cc"] + refute ["https://www.w3.org/ns/activitystreams#Public"] == result["to"] + end end describe "replacing keywords" do @@ -221,5 +294,63 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicyTest do result == "ZFS is free software" end) end + + test "replaces keyword if string matches in history" do + clear_config([:mrf_keyword, :replace], [{"opensource", "free software"}]) + + message = %{ + "type" => "Create", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "object" => %{ + "content" => "ZFS is opensource", + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"content" => "ZFS is opensource mew mew", "summary" => ""} + ] + } + } + } + + {:ok, + %{ + "object" => %{ + "content" => "ZFS is free software", + "formerRepresentations" => %{ + "orderedItems" => [%{"content" => "ZFS is free software mew mew"}] + } + } + }} = KeywordPolicy.filter(message) + end + + test "replaces keyword in Updates" do + clear_config([:mrf_keyword, :replace], [{"opensource", "free software"}]) + + message = %{ + "type" => "Update", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "object" => %{ + "content" => "ZFS is opensource", + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"content" => "ZFS is opensource mew mew", "summary" => ""} + ] + } + } + } + + {:ok, + %{ + "object" => %{ + "content" => "ZFS is free software", + "formerRepresentations" => %{ + "orderedItems" => [%{"content" => "ZFS is free software mew mew"}] + } + } + }} = KeywordPolicy.filter(message) + end end end diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index fcfcb9a2e..c87e64d9e 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1647,5 +1647,22 @@ defmodule Pleroma.Web.CommonAPITest do assert edited_note.data["emoji"]["remoteemoji"] == remote_emoji_uri end + + test "respects MRF" do + user = insert(:user) + + clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy]) + clear_config([:mrf_keyword, :replace], [{"updated", "mewmew"}]) + + {:ok, activity} = CommonAPI.post(user, %{status: "foo1", spoiler_text: "updated 1"}) + assert Object.normalize(activity).data["summary"] == "mewmew 1" + + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2"}) + + updated_object = Object.normalize(updated) + assert updated_object.data["content"] == "mewmew 2" + assert Map.get(updated_object.data, "summary", "") == "" + assert Map.has_key?(updated_object.data, "updated") + end end end From cd19537f391b792ee67c728320801d5a247ceb2c Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 23 Jul 2022 17:48:39 -0400 Subject: [PATCH 062/123] Make EnsureRePrepended history-aware --- lib/pleroma/object/updater.ex | 4 +- lib/pleroma/web/activity_pub/mrf.ex | 45 +++++++++++++++- .../activity_pub/mrf/ensure_re_prepended.ex | 6 ++- lib/pleroma/web/activity_pub/mrf/policy.ex | 3 +- .../mrf/ensure_re_prepended_test.exs | 51 ++++++++++++++++++- 5 files changed, 102 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index 6381320bd..ab38d3ed2 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -198,7 +198,7 @@ defmodule Pleroma.Object.Updater do } end - defp for_each_history_item(%{"orderedItems" => items} = history, _object, fun) do + def for_each_history_item(%{"orderedItems" => items} = history, _object, fun) do new_items = Enum.map(items, fun) |> Enum.reduce_while( @@ -215,7 +215,7 @@ defmodule Pleroma.Object.Updater do end end - defp for_each_history_item(history, _, _) do + def for_each_history_item(history, _, _) do {:ok, history} end diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 323ecdbf1..ff9f84497 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -53,10 +53,53 @@ defmodule Pleroma.Web.ActivityPub.MRF do @required_description_keys [:key, :related_policy] + def filter_one(policy, message) do + should_plug_history? = + if function_exported?(policy, :history_awareness, 0) do + policy.history_awareness() + else + :manual + end + |> Kernel.==(:auto) + + if not should_plug_history? do + policy.filter(message) + else + main_result = policy.filter(message) + + with {_, {:ok, main_message}} <- {:main, main_result}, + {_, + %{ + "formerRepresentations" => %{ + "orderedItems" => [_ | _] + } + }} = {_, object} <- {:object, message["object"]}, + {_, {:ok, new_history}} <- + {:history, + Pleroma.Object.Updater.for_each_history_item( + object["formerRepresentations"], + object, + fn item -> + with {:ok, filtered} <- policy.filter(Map.put(message, "object", item)) do + {:ok, filtered["object"]} + else + e -> e + end + end + )} do + {:ok, put_in(main_message, ["object", "formerRepresentations"], new_history)} + else + {:main, _} -> main_result + {:object, _} -> main_result + {:history, e} -> e + end + end + end + def filter(policies, %{} = message) do policies |> Enum.reduce({:ok, message}, fn - policy, {:ok, message} -> policy.filter(message) + policy, {:ok, message} -> filter_one(policy, message) _, error -> error end) end diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index 51596c09f..a148cc1e7 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do @reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless]) + def history_awareness, do: :auto + def filter_by_summary( %{data: %{"summary" => parent_summary}} = _in_reply_to, %{"summary" => child_summary} = child @@ -27,8 +29,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do def filter_by_summary(_in_reply_to, child), do: child - def filter(%{"type" => "Create", "object" => child_object} = object) - when is_map(child_object) do + def filter(%{"type" => type, "object" => child_object} = object) + when type in ["Create", "Update"] and is_map(child_object) do child = child_object["inReplyTo"] |> Object.normalize(fetch: false) diff --git a/lib/pleroma/web/activity_pub/mrf/policy.ex b/lib/pleroma/web/activity_pub/mrf/policy.ex index 0ac250c3d..0234de4d5 100644 --- a/lib/pleroma/web/activity_pub/mrf/policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/policy.ex @@ -12,5 +12,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.Policy do label: String.t(), description: String.t() } - @optional_callbacks config_description: 0 + @callback history_awareness() :: :auto | :manual + @optional_callbacks config_description: 0, history_awareness: 0 end diff --git a/test/pleroma/web/activity_pub/mrf/ensure_re_prepended_test.exs b/test/pleroma/web/activity_pub/mrf/ensure_re_prepended_test.exs index bc2f09e29..859e6f1e9 100644 --- a/test/pleroma/web/activity_pub/mrf/ensure_re_prepended_test.exs +++ b/test/pleroma/web/activity_pub/mrf/ensure_re_prepended_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrependedTest do alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.EnsureRePrepended describe "rewrites summary" do @@ -35,10 +36,58 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrependedTest do assert {:ok, res} = EnsureRePrepended.filter(message) assert res["object"]["summary"] == "re: object-summary" end + + test "it adds `re:` to history" do + message = %{ + "type" => "Create", + "object" => %{ + "summary" => "object-summary", + "inReplyTo" => %Activity{object: %Object{data: %{"summary" => "object-summary"}}}, + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "summary" => "object-summary", + "inReplyTo" => %Activity{object: %Object{data: %{"summary" => "object-summary"}}} + } + ] + } + } + } + + assert {:ok, res} = MRF.filter_one(EnsureRePrepended, message) + assert res["object"]["summary"] == "re: object-summary" + + assert Enum.at(res["object"]["formerRepresentations"]["orderedItems"], 0)["summary"] == + "re: object-summary" + end + + test "it accepts Updates" do + message = %{ + "type" => "Update", + "object" => %{ + "summary" => "object-summary", + "inReplyTo" => %Activity{object: %Object{data: %{"summary" => "object-summary"}}}, + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "summary" => "object-summary", + "inReplyTo" => %Activity{object: %Object{data: %{"summary" => "object-summary"}}} + } + ] + } + } + } + + assert {:ok, res} = MRF.filter_one(EnsureRePrepended, message) + assert res["object"]["summary"] == "re: object-summary" + + assert Enum.at(res["object"]["formerRepresentations"]["orderedItems"], 0)["summary"] == + "re: object-summary" + end end describe "skip filter" do - test "it skip if type isn't 'Create'" do + test "it skip if type isn't 'Create' or 'Update'" do message = %{ "type" => "Annotation", "object" => %{"summary" => "object-summary"} From 0a337063e14a63b3ed80776b493e3c9c56dd95d1 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 23 Jul 2022 22:23:57 -0400 Subject: [PATCH 063/123] Make ForceMentionsInContent history-aware --- .../mrf/force_mentions_in_content.ex | 7 +- .../mrf/force_mentions_in_content_test.exs | 95 +++++++++++++++++++ 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex index 255910b2f..70224561c 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex @@ -11,6 +11,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do @behaviour Pleroma.Web.ActivityPub.MRF.Policy + @impl true + def history_awareness, do: :auto + defp do_extract({:a, attrs, _}, acc) do if Enum.find(attrs, fn {name, value} -> name == "class" && value in ["mention", "u-url mention", "mention u-url"] @@ -74,11 +77,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do @impl true def filter( %{ - "type" => "Create", + "type" => type, "object" => %{"type" => "Note", "to" => to, "inReplyTo" => in_reply_to} } = object ) - when is_list(to) and is_binary(in_reply_to) do + when type in ["Create", "Update"] and is_list(to) and is_binary(in_reply_to) do # image-only posts from pleroma apparently reach this MRF without the content field content = object["object"]["content"] || "" diff --git a/test/pleroma/web/activity_pub/mrf/force_mentions_in_content_test.exs b/test/pleroma/web/activity_pub/mrf/force_mentions_in_content_test.exs index 125b14a59..b349a4bb7 100644 --- a/test/pleroma/web/activity_pub/mrf/force_mentions_in_content_test.exs +++ b/test/pleroma/web/activity_pub/mrf/force_mentions_in_content_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContentTest do alias Pleroma.Constants alias Pleroma.Object + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent alias Pleroma.Web.CommonAPI @@ -161,4 +162,98 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContentTest do assert filtered == "

@luigi I'ma tired...

" end + + test "aware of history" do + mario = insert(:user, nickname: "mario") + wario = insert(:user, nickname: "wario") + + {:ok, post1} = CommonAPI.post(mario, %{status: "Letsa go!"}) + + activity = %{ + "type" => "Create", + "actor" => wario.ap_id, + "object" => %{ + "type" => "Note", + "actor" => wario.ap_id, + "content" => "WHA-HA!", + "to" => [ + mario.ap_id, + Constants.as_public() + ], + "inReplyTo" => post1.object.data["id"], + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "type" => "Note", + "actor" => wario.ap_id, + "content" => "WHA-HA!", + "to" => [ + mario.ap_id, + Constants.as_public() + ], + "inReplyTo" => post1.object.data["id"] + } + ] + } + } + } + + expected = + "@mario WHA-HA!" + + assert {:ok, + %{ + "object" => %{ + "content" => ^expected, + "formerRepresentations" => %{"orderedItems" => [%{"content" => ^expected}]} + } + }} = MRF.filter_one(ForceMentionsInContent, activity) + end + + test "works with Updates" do + mario = insert(:user, nickname: "mario") + wario = insert(:user, nickname: "wario") + + {:ok, post1} = CommonAPI.post(mario, %{status: "Letsa go!"}) + + activity = %{ + "type" => "Update", + "actor" => wario.ap_id, + "object" => %{ + "type" => "Note", + "actor" => wario.ap_id, + "content" => "WHA-HA!", + "to" => [ + mario.ap_id, + Constants.as_public() + ], + "inReplyTo" => post1.object.data["id"], + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "type" => "Note", + "actor" => wario.ap_id, + "content" => "WHA-HA!", + "to" => [ + mario.ap_id, + Constants.as_public() + ], + "inReplyTo" => post1.object.data["id"] + } + ] + } + } + } + + expected = + "@mario WHA-HA!" + + assert {:ok, + %{ + "object" => %{ + "content" => ^expected, + "formerRepresentations" => %{"orderedItems" => [%{"content" => ^expected}]} + } + }} = MRF.filter_one(ForceMentionsInContent, activity) + end end From dce7e429286dfe8cb44a27c50713a03f0e696357 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 23 Jul 2022 22:34:03 -0400 Subject: [PATCH 064/123] Make MediaProxyWarmingPolicy history-aware --- .../mrf/media_proxy_warming_policy.ex | 9 ++-- .../mrf/media_proxy_warming_policy_test.exs | 44 +++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index 0eac8f021..c95d35bb9 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -16,6 +16,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do recv_timeout: 10_000 ] + @impl true + def history_awareness, do: :auto + defp prefetch(url) do # Fetching only proxiable resources if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do @@ -54,10 +57,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do end @impl true - def filter( - %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message - ) - when is_list(attachments) and length(attachments) > 0 do + def filter(%{"type" => type, "object" => %{"attachment" => attachments} = _object} = message) + when type in ["Create", "Update"] and is_list(attachments) and length(attachments) > 0 do preload(message) {:ok, message} diff --git a/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs b/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs index 09301c6ca..6557c3a98 100644 --- a/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do use Pleroma.Tests.Helpers alias Pleroma.HTTP + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy import Mock @@ -22,6 +23,25 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do } } + @message_with_history %{ + "type" => "Create", + "object" => %{ + "type" => "Note", + "content" => "content", + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "content", + "attachment" => [ + %{"url" => [%{"href" => "http://example.com/image.jpg"}]} + ] + } + ] + } + } + } + setup do: clear_config([:media_proxy, :enabled], true) test "it prefetches media proxy URIs" do @@ -50,4 +70,28 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do refute called(HTTP.get(:_, :_, :_)) end end + + test "history-aware" do + Tesla.Mock.mock(fn %{method: :get, url: "http://example.com/image.jpg"} -> + {:ok, %Tesla.Env{status: 200, body: ""}} + end) + + with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do + MRF.filter_one(MediaProxyWarmingPolicy, @message_with_history) + + assert called(HTTP.get(:_, :_, :_)) + end + end + + test "works with Updates" do + Tesla.Mock.mock(fn %{method: :get, url: "http://example.com/image.jpg"} -> + {:ok, %Tesla.Env{status: 200, body: ""}} + end) + + with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do + MRF.filter_one(MediaProxyWarmingPolicy, @message_with_history |> Map.put("type", "Update")) + + assert called(HTTP.get(:_, :_, :_)) + end + end end From fc7ce5f93c4031863cbaf62b72dce55b5b6b0390 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 23 Jul 2022 22:41:04 -0400 Subject: [PATCH 065/123] Make NoPlaceholderTextPolicy history-aware --- .../mrf/no_placeholder_text_policy.ex | 7 +++- .../mrf/no_placeholder_text_policy_test.exs | 41 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index aab647d8e..f81e9e52a 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -6,14 +6,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do @moduledoc "Ensure no content placeholder is present (such as the dot from mastodon)" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + @impl true + def history_awareness, do: :auto + @impl true def filter( %{ - "type" => "Create", + "type" => type, "object" => %{"content" => content, "attachment" => _} = _child_object } = object ) - when content in [".", "

.

"] do + when type in ["Create", "Update"] and content in [".", "

.

"] do {:ok, put_in(object, ["object", "content"], "")} end diff --git a/test/pleroma/web/activity_pub/mrf/no_placeholder_text_policy_test.exs b/test/pleroma/web/activity_pub/mrf/no_placeholder_text_policy_test.exs index acc7c3566..3533c2bc8 100644 --- a/test/pleroma/web/activity_pub/mrf/no_placeholder_text_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/no_placeholder_text_policy_test.exs @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicyTest do use Pleroma.DataCase, async: true + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy test "it clears content object" do @@ -20,6 +21,46 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicyTest do assert res["object"]["content"] == "" end + test "history-aware" do + message = %{ + "type" => "Create", + "object" => %{ + "content" => ".", + "attachment" => "image", + "formerRepresentations" => %{ + "orderedItems" => [%{"content" => ".", "attachment" => "image"}] + } + } + } + + assert {:ok, res} = MRF.filter_one(NoPlaceholderTextPolicy, message) + + assert %{ + "content" => "", + "formerRepresentations" => %{"orderedItems" => [%{"content" => ""}]} + } = res["object"] + end + + test "works with Updates" do + message = %{ + "type" => "Update", + "object" => %{ + "content" => ".", + "attachment" => "image", + "formerRepresentations" => %{ + "orderedItems" => [%{"content" => ".", "attachment" => "image"}] + } + } + } + + assert {:ok, res} = MRF.filter_one(NoPlaceholderTextPolicy, message) + + assert %{ + "content" => "", + "formerRepresentations" => %{"orderedItems" => [%{"content" => ""}]} + } = res["object"] + end + @messages [ %{ "type" => "Create", From 46a5c06853c21e720b41a4b38a4d88a38a218ad4 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 23 Jul 2022 22:50:38 -0400 Subject: [PATCH 066/123] Make NormalizeMarkup history-aware --- .../web/activity_pub/mrf/normalize_markup.ex | 6 +- .../mrf/normalize_markup_test.exs | 59 +++++++++++++++---- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex index dc2c19d49..2dfc9a901 100644 --- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex +++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex @@ -9,7 +9,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do @behaviour Pleroma.Web.ActivityPub.MRF.Policy @impl true - def filter(%{"type" => "Create", "object" => child_object} = object) do + def history_awareness, do: :auto + + @impl true + def filter(%{"type" => type, "object" => child_object} = object) + when type in ["Create", "Update"] do scrub_policy = Pleroma.Config.get([:mrf_normalize_markup, :scrub_policy]) content = diff --git a/test/pleroma/web/activity_pub/mrf/normalize_markup_test.exs b/test/pleroma/web/activity_pub/mrf/normalize_markup_test.exs index 20176b63b..66a8f4e44 100644 --- a/test/pleroma/web/activity_pub/mrf/normalize_markup_test.exs +++ b/test/pleroma/web/activity_pub/mrf/normalize_markup_test.exs @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkupTest do use Pleroma.DataCase, async: true + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.NormalizeMarkup @html_sample """ @@ -16,24 +17,58 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkupTest do """ - test "it filter html tags" do - expected = """ - this is in bold -

this is a paragraph

- this is a linebreak
- this is a link with allowed "rel" attribute: - this is a link with not allowed "rel" attribute: example.com - this is an image:
- alert('hacked') - """ + @expected """ + this is in bold +

this is a paragraph

+ this is a linebreak
+ this is a link with allowed "rel" attribute: + this is a link with not allowed "rel" attribute: example.com + this is an image:
+ alert('hacked') + """ + test "it filter html tags" do message = %{"type" => "Create", "object" => %{"content" => @html_sample}} assert {:ok, res} = NormalizeMarkup.filter(message) - assert res["object"]["content"] == expected + assert res["object"]["content"] == @expected end - test "it skips filter if type isn't `Create`" do + test "history-aware" do + message = %{ + "type" => "Create", + "object" => %{ + "content" => @html_sample, + "formerRepresentations" => %{"orderedItems" => [%{"content" => @html_sample}]} + } + } + + assert {:ok, res} = MRF.filter_one(NormalizeMarkup, message) + + assert %{ + "content" => @expected, + "formerRepresentations" => %{"orderedItems" => [%{"content" => @expected}]} + } = res["object"] + end + + test "works with Updates" do + message = %{ + "type" => "Update", + "object" => %{ + "content" => @html_sample, + "formerRepresentations" => %{"orderedItems" => [%{"content" => @html_sample}]} + } + } + + assert {:ok, res} = MRF.filter_one(NormalizeMarkup, message) + + assert %{ + "content" => @expected, + "formerRepresentations" => %{"orderedItems" => [%{"content" => @expected}]} + } = res["object"] + end + + test "it skips filter if type isn't `Create` or `Update`" do message = %{"type" => "Note", "object" => %{}} assert {:ok, res} = NormalizeMarkup.filter(message) From 82c8fc1ede26837d024ecc2fd1231c6d2a3c2c3e Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 23 Jul 2022 23:24:25 -0400 Subject: [PATCH 067/123] Make NoEmptyPolicy work with Update --- .../web/activity_pub/mrf/no_empty_policy.ex | 15 +++++++++--- .../activity_pub/mrf/no_empty_policy_test.exs | 23 +++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex index 4dc96e068..855cda3b9 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do @impl true def filter(%{"actor" => actor} = object) do with true <- is_local?(actor), + true <- is_eligible_type?(object), true <- is_note?(object), false <- has_attachment?(object), true <- only_mentions?(object) do @@ -32,7 +33,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do end defp has_attachment?(%{ - "type" => "Create", "object" => %{"type" => "Note", "attachment" => attachments} }) when length(attachments) > 0, @@ -40,7 +40,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do defp has_attachment?(_), do: false - defp only_mentions?(%{"type" => "Create", "object" => %{"type" => "Note", "source" => source}}) do + defp only_mentions?(%{"object" => %{"type" => "Note", "source" => source}}) do + source = + case source do + %{"content" => text} -> text + _ -> source + end + non_mentions = source |> String.split() |> Enum.filter(&(not String.starts_with?(&1, "@"))) |> length @@ -53,9 +59,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do defp only_mentions?(_), do: false - defp is_note?(%{"type" => "Create", "object" => %{"type" => "Note"}}), do: true + defp is_note?(%{"object" => %{"type" => "Note"}}), do: true defp is_note?(_), do: false + defp is_eligible_type?(%{"type" => type}) when type in ["Create", "Update"], do: true + defp is_eligible_type?(_), do: false + @impl true def describe, do: {:ok, %{}} end diff --git a/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs b/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs index fe4bb8f0a..386ed395f 100644 --- a/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs @@ -151,4 +151,27 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicyTest do assert NoEmptyPolicy.filter(message) == {:reject, "[NoEmptyPolicy]"} end + + test "works with Update" do + message = %{ + "actor" => "http://localhost:4001/users/testuser", + "cc" => ["http://localhost:4001/users/testuser/followers"], + "object" => %{ + "actor" => "http://localhost:4001/users/testuser", + "attachment" => [], + "cc" => ["http://localhost:4001/users/testuser/followers"], + "source" => "", + "to" => [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "type" => "Note" + }, + "to" => [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "type" => "Update" + } + + assert NoEmptyPolicy.filter(message) == {:reject, "[NoEmptyPolicy]"} + end end From d877d2a4e7449e942b4d192f283824eebcade563 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 24 Jul 2022 00:02:39 -0400 Subject: [PATCH 068/123] Make HashtagPolicy history-aware --- .../web/activity_pub/mrf/hashtag_policy.ex | 47 ++++++++++--- .../activity_pub/mrf/hashtag_policy_test.exs | 70 +++++++++++++++++++ 2 files changed, 107 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex index 2142b7add..b73fd974c 100644 --- a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex @@ -16,6 +16,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do @behaviour Pleroma.Web.ActivityPub.MRF.Policy + @impl true + def history_awareness, do: :manual + defp check_reject(message, hashtags) do if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do {:reject, "[HashtagPolicy] Matches with rejected keyword"} @@ -47,22 +50,46 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do defp check_ftl_removal(message, _hashtags), do: {:ok, message} - defp check_sensitive(message, hashtags) do - if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do - {:ok, Kernel.put_in(message, ["object", "sensitive"], true)} - else - {:ok, message} - end + defp check_sensitive(message) do + {:ok, new_object} = + Object.Updater.do_with_history(message["object"], fn object -> + hashtags = Object.hashtags(%Object{data: object}) + + if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do + {:ok, Map.put(object, "sensitive", true)} + else + {:ok, object} + end + end) + + {:ok, Map.put(message, "object", new_object)} end @impl true - def filter(%{"type" => "Create", "object" => object} = message) do - hashtags = Object.hashtags(%Object{data: object}) + def filter(%{"type" => type, "object" => object} = message) when type in ["Create", "Update"] do + history_items = + with %{"formerRepresentations" => %{"orderedItems" => items}} <- object do + items + else + _ -> [] + end + + historical_hashtags = + Enum.reduce(history_items, [], fn item, acc -> + acc ++ Object.hashtags(%Object{data: item}) + end) + + hashtags = Object.hashtags(%Object{data: object}) ++ historical_hashtags if hashtags != [] do with {:ok, message} <- check_reject(message, hashtags), - {:ok, message} <- check_ftl_removal(message, hashtags), - {:ok, message} <- check_sensitive(message, hashtags) do + {:ok, message} <- + (if "type" == "Create" do + check_ftl_removal(message, hashtags) + else + {:ok, message} + end), + {:ok, message} <- check_sensitive(message) do {:ok, message} end else diff --git a/test/pleroma/web/activity_pub/mrf/hashtag_policy_test.exs b/test/pleroma/web/activity_pub/mrf/hashtag_policy_test.exs index 7f2d78a4c..32991c966 100644 --- a/test/pleroma/web/activity_pub/mrf/hashtag_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/hashtag_policy_test.exs @@ -20,6 +20,76 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicyTest do assert modified["object"]["sensitive"] end + test "it is history-aware" do + activity = %{ + "type" => "Create", + "object" => %{ + "content" => "hey", + "tag" => [] + } + } + + activity_data = + activity + |> put_in( + ["object", "formerRepresentations"], + %{ + "type" => "OrderedCollection", + "orderedItems" => [ + Map.put( + activity["object"], + "tag", + [%{"type" => "Hashtag", "name" => "#nsfw"}] + ) + ] + } + ) + + {:ok, modified} = + Pleroma.Web.ActivityPub.MRF.filter_one( + Pleroma.Web.ActivityPub.MRF.HashtagPolicy, + activity_data + ) + + refute modified["object"]["sensitive"] + assert Enum.at(modified["object"]["formerRepresentations"]["orderedItems"], 0)["sensitive"] + end + + test "it works with Update" do + activity = %{ + "type" => "Update", + "object" => %{ + "content" => "hey", + "tag" => [] + } + } + + activity_data = + activity + |> put_in( + ["object", "formerRepresentations"], + %{ + "type" => "OrderedCollection", + "orderedItems" => [ + Map.put( + activity["object"], + "tag", + [%{"type" => "Hashtag", "name" => "#nsfw"}] + ) + ] + } + ) + + {:ok, modified} = + Pleroma.Web.ActivityPub.MRF.filter_one( + Pleroma.Web.ActivityPub.MRF.HashtagPolicy, + activity_data + ) + + refute modified["object"]["sensitive"] + assert Enum.at(modified["object"]["formerRepresentations"]["orderedItems"], 0)["sensitive"] + end + test "it doesn't sets the sensitive property with irrelevant hashtags" do user = insert(:user) From 997f08b3500a983e8b27db9a6e4745582bb4763c Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 24 Jul 2022 00:18:09 -0400 Subject: [PATCH 069/123] Make AntiLinkSpamPolicy history-aware --- .../activity_pub/mrf/anti_link_spam_policy.ex | 3 ++ .../mrf/anti_link_spam_policy_test.exs | 33 ++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index f0504ead4..3ec9c52ee 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -9,6 +9,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do require Logger + @impl true + def history_awareness, do: :auto + # has the user successfully posted before? defp old_user?(%User{} = u) do u.note_count > 0 || u.follower_count > 0 diff --git a/test/pleroma/web/activity_pub/mrf/anti_link_spam_policy_test.exs b/test/pleroma/web/activity_pub/mrf/anti_link_spam_policy_test.exs index 8c7d4de5c..303d7ca1e 100644 --- a/test/pleroma/web/activity_pub/mrf/anti_link_spam_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/anti_link_spam_policy_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicyTest do import Pleroma.Factory import ExUnit.CaptureLog + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy @linkless_message %{ @@ -49,11 +50,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicyTest do assert user.note_count == 0 - message = - @linkful_message - |> Map.put("actor", user.ap_id) + message = %{ + "type" => "Create", + "actor" => user.ap_id, + "object" => %{ + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "content" => "hi world!" + } + ] + }, + "content" => "mew" + } + } - {:reject, _} = AntiLinkSpamPolicy.filter(message) + {:reject, _} = MRF.filter_one(AntiLinkSpamPolicy, message) end test "it allows posts with links for local users" do @@ -67,6 +80,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicyTest do {:ok, _message} = AntiLinkSpamPolicy.filter(message) end + + test "it disallows posts with links in history" do + user = insert(:user, local: false) + + assert user.note_count == 0 + + message = + @linkful_message + |> Map.put("actor", user.ap_id) + + {:reject, _} = AntiLinkSpamPolicy.filter(message) + end end describe "with old user" do From a4fa286d200b4f0c0ac9f453eb3e0a0526560a20 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 2 Aug 2022 10:15:56 -0400 Subject: [PATCH 070/123] Use actor_types() to determine whether the Update is for user --- lib/pleroma/web/activity_pub/side_effects.ex | 5 +++-- test/pleroma/web/activity_pub/side_effects_test.exs | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index f56e357bf..5eefd2824 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -163,8 +163,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do updated_object_id = updated_object["id"] with {_, true} <- {:has_id, is_binary(updated_object_id)}, - {_, user} <- {:user, Pleroma.User.get_by_ap_id(updated_object_id)} do - if user do + %{"type" => type} <- updated_object, + {_, is_user} <- {:is_user, type in Pleroma.Constants.actor_types()} do + if is_user do handle_update_user(object, meta) else handle_update_object(object, meta) diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 0db3a8ea6..9d3490ecd 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -118,7 +118,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do describe "update users" do setup do user = insert(:user, local: false) - {:ok, update_data, []} = Builder.update(user, %{"id" => user.ap_id, "name" => "new name!"}) + + {:ok, update_data, []} = + Builder.update(user, %{"id" => user.ap_id, "type" => "Person", "name" => "new name!"}) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) %{user: user, update_data: update_data, update: update} From f2a9285ff089fbae043091898fb016f4aa16f689 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Sat, 23 Jul 2022 18:58:45 +0000 Subject: [PATCH 071/123] bugfix/follow-state (#104) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/104 --- lib/mix/tasks/pleroma/user.ex | 32 ++++++++++++++++++++++++++++ lib/pleroma/user.ex | 8 ++++++- test/pleroma/web/common_api_test.exs | 16 +++++++++++--- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 96d4eb90b..50ffb7f27 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -421,6 +421,38 @@ defmodule Mix.Tasks.Pleroma.User do |> Stream.run() end + def run(["fix_follow_state", local_user, remote_user]) do + start_pleroma() + + with {:local, %User{} = local} <- {:local, User.get_by_nickname(local_user)}, + {:remote, %User{} = remote} <- {:remote, User.get_by_nickname(remote_user)}, + {:follow_data, %{data: %{"state" => request_state}}} <- + {:follow_data, Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(local, remote)} do + calculated_state = User.following?(local, remote) + + shell_info( + "Request state is #{request_state}, vs calculated state of following=#{calculated_state}" + ) + + if calculated_state == false && request_state == "accept" do + shell_info("Discrepancy found, fixing") + Pleroma.Web.CommonAPI.reject_follow_request(local, remote) + shell_info("Relationship fixed") + else + shell_info("No discrepancy found") + end + else + {:local, _} -> + shell_error("No local user #{local_user}") + + {:remote, _} -> + shell_error("No remote user #{remote_user}") + + {:follow_data, _} -> + shell_error("No follow data for #{local_user} and #{remote_user}") + end + end + defp set_moderator(user, value) do {:ok, user} = user diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index eeea240fb..a57295891 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1574,13 +1574,19 @@ defmodule Pleroma.User do blocker end - # clear any requested follows as well + # clear any requested follows from both sides as well blocked = case CommonAPI.reject_follow_request(blocked, blocker) do {:ok, %User{} = updated_blocked} -> updated_blocked nil -> blocked end + blocker = + case CommonAPI.reject_follow_request(blocker, blocked) do + {:ok, %User{} = updated_blocker} -> updated_blocker + nil -> blocker + end + unsubscribe(blocked, blocker) unfollowing_blocked = Config.get([:activitypub, :unfollow_blocked], true) diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index b502aaa03..288175ddb 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -61,9 +61,11 @@ defmodule Pleroma.Web.CommonAPITest do describe "blocking" do setup do blocker = insert(:user) - blocked = insert(:user) - User.follow(blocker, blocked) - User.follow(blocked, blocker) + blocked = insert(:user, local: false) + CommonAPI.follow(blocker, blocked) + CommonAPI.follow(blocked, blocker) + CommonAPI.accept_follow_request(blocker, blocked) + CommonAPI.accept_follow_request(blocked, blocked) %{blocker: blocker, blocked: blocked} end @@ -72,6 +74,9 @@ defmodule Pleroma.Web.CommonAPITest do with_mock Pleroma.Web.Federator, publish: fn _ -> nil end do + assert User.get_follow_state(blocker, blocked) == :follow_accept + refute is_nil(Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(blocker, blocked)) + assert {:ok, block} = CommonAPI.block(blocker, blocked) assert block.local @@ -79,6 +84,11 @@ defmodule Pleroma.Web.CommonAPITest do refute User.following?(blocker, blocked) refute User.following?(blocked, blocker) + refute User.get_follow_state(blocker, blocked) + + assert %{data: %{"state" => "reject"}} = + Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(blocker, blocked) + assert called(Pleroma.Web.Federator.publish(block)) end end From 5d900a5cd141db06da474dbf691e815a0e97ef7a Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 22 Jun 2022 15:22:40 -0400 Subject: [PATCH 072/123] Use latest alpine version for docker image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c51ebbab0..e68b7ea7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN apk add git gcc g++ musl-dev make cmake file-dev &&\ mkdir release &&\ mix release --path release -FROM alpine:3.14 +FROM alpine ARG BUILD_DATE ARG VCS_REF From a0166e92fac596651ecaad78659a0f6907ccb6bd Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 6 Aug 2022 00:31:36 -0400 Subject: [PATCH 073/123] Treat MRF rejects as success in Oban worker --- lib/pleroma/workers/receiver_worker.ex | 7 +++++- test/pleroma/workers/receiver_worker_test.exs | 25 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 test/pleroma/workers/receiver_worker_test.exs diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 268b5f30f..309e197dc 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -9,6 +9,11 @@ defmodule Pleroma.Workers.ReceiverWorker do @impl Oban.Worker def perform(%Job{args: %{"op" => "incoming_ap_doc", "params" => params}}) do - Federator.perform(:incoming_ap_doc, params) + with {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do + {:ok, res} + else + {:error, {:reject, reason}} -> {:cancel, reason} + e -> e + end end end diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs new file mode 100644 index 000000000..283beee4d --- /dev/null +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -0,0 +1,25 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.ReceiverWorkerTest do + use Pleroma.DataCase, async: true + use Oban.Testing, repo: Pleroma.Repo + + import Mock + import Pleroma.Factory + + alias Pleroma.Workers.ReceiverWorker + + test "it ignores MRF reject" do + params = insert(:note).data + + with_mock Pleroma.Web.ActivityPub.Transmogrifier, + handle_incoming: fn _ -> {:reject, "MRF"} end do + assert {:cancel, "MRF"} = + ReceiverWorker.perform(%Oban.Job{ + args: %{"op" => "incoming_ap_doc", "params" => params} + }) + end + end +end From 88e0e6acd5cc5f84256c531093a8cebee8b79786 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 6 Aug 2022 00:42:10 -0400 Subject: [PATCH 074/123] Fix FederatorTest --- test/pleroma/web/federator_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/web/federator_test.exs b/test/pleroma/web/federator_test.exs index 5120bf57c..20ac631f7 100644 --- a/test/pleroma/web/federator_test.exs +++ b/test/pleroma/web/federator_test.exs @@ -169,7 +169,7 @@ defmodule Pleroma.Web.FederatorTest do |> Jason.decode!() assert {:ok, job} = Federator.incoming_ap_doc(params) - assert {:error, _} = ObanHelpers.perform(job) + assert {:cancel, _} = ObanHelpers.perform(job) end end end From d487e0160cdc4cdf84c45e4c64f6589b317479cc Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 8 Aug 2022 08:41:33 -0400 Subject: [PATCH 075/123] Treat containment failure as cancel in ReceiverWorker --- lib/pleroma/workers/receiver_worker.ex | 1 + test/pleroma/web/federator_test.exs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 309e197dc..c41b44e14 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -12,6 +12,7 @@ defmodule Pleroma.Workers.ReceiverWorker do with {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do {:ok, res} else + {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed} {:error, {:reject, reason}} -> {:cancel, reason} e -> e end diff --git a/test/pleroma/web/federator_test.exs b/test/pleroma/web/federator_test.exs index 20ac631f7..41d1c5d5e 100644 --- a/test/pleroma/web/federator_test.exs +++ b/test/pleroma/web/federator_test.exs @@ -153,7 +153,7 @@ defmodule Pleroma.Web.FederatorTest do } assert {:ok, job} = Federator.incoming_ap_doc(params) - assert {:error, :origin_containment_failed} = ObanHelpers.perform(job) + assert {:cancel, :origin_containment_failed} = ObanHelpers.perform(job) end test "it does not crash if MRF rejects the post" do From f3e061c9645571997a01b1091d0e8a3f68c6bb21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Sat, 6 Aug 2022 03:24:31 +0200 Subject: [PATCH 076/123] Object: remove context_id field 30 to 70% of the objects in the object table are simple JSON objects containing a single field, 'id', being the context's ID. The reason for the creation of an object per context seems to be an old relic from the StatusNet era, and has only been used nowadays as an helper for threads in Pleroma-FE via the `pleroma.conversation_id` field in status views. An object per context was created, and its numerical ID (table column) was used and stored as 'context_id' in the object and activity along with the full 'context' URI/string. This commit removes this field and stops creation of objects for each context, which will also allow incoming activities to use activity IDs as contexts, something which was not possible before, or would have been very broken under most circumstances. The `pleroma.conversation_id` field has been reimplemented in a way to maintain backwards-compatibility by calculating a CRC32 of the full context URI/string in the object, instead of relying on the row ID for the created context object. --- lib/pleroma/object.ex | 4 --- .../article_note_page_validator.ex | 2 +- .../object_validators/common_fixes.ex | 4 +-- .../object_validators/event_validator.ex | 2 +- .../object_validators/question_validator.ex | 2 +- lib/pleroma/web/activity_pub/utils.ex | 23 ++------------- lib/pleroma/web/common_api/utils.ex | 29 ------------------- .../web/mastodon_api/views/status_view.ex | 2 +- .../web/activity_pub/activity_pub_test.exs | 5 +--- .../transmogrifier/question_handling_test.exs | 3 -- .../web/activity_pub/transmogrifier_test.exs | 2 -- test/pleroma/web/activity_pub/utils_test.exs | 4 --- test/pleroma/web/common_api/utils_test.exs | 27 ----------------- .../mastodon_api/views/status_view_test.exs | 2 +- 14 files changed, 9 insertions(+), 102 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index fe264b5e0..c214a79c5 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -208,10 +208,6 @@ defmodule Pleroma.Object do end end - def context_mapping(context) do - Object.change(%Object{}, %{data: %{"id" => context}}) - end - def make_tombstone(%Object{data: %{"id" => id, "type" => type}}, deleted \\ DateTime.utc_now()) do %ObjectTombstone{ id: id, diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index 57c8d1dc0..c5fb94034 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -94,7 +94,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Article", "Note", "Page"]) - |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) + |> validate_required([:id, :actor, :attributedTo, :type, :context]) |> CommonValidations.validate_any_presence([:cc, :to]) |> CommonValidations.validate_fields_match([:actor, :attributedTo]) |> CommonValidations.validate_actor_presence() diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 4f8c083eb..c7e292bec 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -22,14 +22,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do end def fix_object_defaults(data) do - %{data: %{"id" => context}, id: context_id} = - Utils.create_context(data["context"] || data["conversation"]) + context = Utils.maybe_create_context(data["context"] || data["conversation"]) %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"]) data |> Map.put("context", context) - |> Map.put("context_id", context_id) |> cast_and_filter_recipients("to", follower_collection) |> cast_and_filter_recipients("cc", follower_collection) |> cast_and_filter_recipients("bto", follower_collection) diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex index 0e99f2037..ab204f69a 100644 --- a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex @@ -62,7 +62,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Event"]) - |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) + |> validate_required([:id, :actor, :attributedTo, :type, :context]) |> CommonValidations.validate_any_presence([:cc, :to]) |> CommonValidations.validate_fields_match([:actor, :attributedTo]) |> CommonValidations.validate_actor_presence() diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 9412be4bc..ce3305142 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -80,7 +80,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Question"]) - |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) + |> validate_required([:id, :actor, :attributedTo, :type, :context]) |> CommonValidations.validate_any_presence([:cc, :to]) |> CommonValidations.validate_fields_match([:actor, :attributedTo]) |> CommonValidations.validate_actor_presence() diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 9cde7805c..d3b7d804f 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -154,22 +154,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do Notification.get_notified_from_activity(%Activity{data: object}, false) end - def create_context(context) do - context = context || generate_id("contexts") - - # Ecto has problems accessing the constraint inside the jsonb, - # so we explicitly check for the existed object before insert - object = Object.get_cached_by_ap_id(context) - - with true <- is_nil(object), - changeset <- Object.context_mapping(context), - {:ok, inserted_object} <- Repo.insert(changeset) do - inserted_object - else - _ -> - object - end - end + def maybe_create_context(context), do: context || generate_id("contexts") @doc """ Enqueues an activity for federation if it's local @@ -201,18 +186,16 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> Map.put_new("id", "pleroma:fakeid") |> Map.put_new_lazy("published", &make_date/0) |> Map.put_new("context", "pleroma:fakecontext") - |> Map.put_new("context_id", -1) |> lazy_put_object_defaults(true) end def lazy_put_activity_defaults(map, _fake?) do - %{data: %{"id" => context}, id: context_id} = create_context(map["context"]) + context = maybe_create_context(map["context"]) map |> Map.put_new_lazy("id", &generate_activity_id/0) |> Map.put_new_lazy("published", &make_date/0) |> Map.put_new("context", context) - |> Map.put_new("context_id", context_id) |> lazy_put_object_defaults(false) end @@ -226,7 +209,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> Map.put_new("id", "pleroma:fake_object_id") |> Map.put_new_lazy("published", &make_date/0) |> Map.put_new("context", activity["context"]) - |> Map.put_new("context_id", activity["context_id"]) |> Map.put_new("fake", true) %{activity | "object" => object} @@ -239,7 +221,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> Map.put_new_lazy("id", &generate_object_id/0) |> Map.put_new_lazy("published", &make_date/0) |> Map.put_new("context", activity["context"]) - |> Map.put_new("context_id", activity["context_id"]) %{activity | "object" => object} end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index ce850b038..052bd7770 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -449,35 +449,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do def get_report_statuses(_, _), do: {:ok, nil} - # DEPRECATED mostly, context objects are now created at insertion time. - def context_to_conversation_id(context) do - with %Object{id: id} <- Object.get_cached_by_ap_id(context) do - id - else - _e -> - changeset = Object.context_mapping(context) - - case Repo.insert(changeset) do - {:ok, %{id: id}} -> - id - - # This should be solved by an upsert, but it seems ecto - # has problems accessing the constraint inside the jsonb. - {:error, _} -> - Object.get_cached_by_ap_id(context).id - end - end - end - - def conversation_id_to_context(id) do - with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do - context - else - _e -> - {:error, dgettext("errors", "No such conversation")} - end - end - def validate_character_limit("" = _full_payload, [] = _attachments) do {:error, dgettext("errors", "Cannot post an empty status without attachments")} end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 1ebfd6740..31a0c420f 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -61,7 +61,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do do: context_id defp get_context_id(%{data: %{"context" => context}}) when is_binary(context), - do: Utils.context_to_conversation_id(context) + do: :erlang.crc32(context) defp get_context_id(_), do: nil diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index 4d7e76266..b8d73ea10 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -554,7 +554,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert activity.data["ok"] == data["ok"] assert activity.data["id"] == given_id assert activity.data["context"] == "blabla" - assert activity.data["context_id"] end test "adds a context when none is there" do @@ -576,8 +575,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert is_binary(activity.data["context"]) assert is_binary(object.data["context"]) - assert activity.data["context_id"] - assert object.data["context_id"] end test "adds an id to a given object if it lacks one and is a note and inserts it to the object database" do @@ -1612,7 +1609,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do }) assert Repo.aggregate(Activity, :count, :id) == 1 - assert Repo.aggregate(Object, :count, :id) == 2 + assert Repo.aggregate(Object, :count, :id) == 1 assert Repo.aggregate(Notification, :count, :id) == 0 end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/question_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/question_handling_test.exs index d22ec400d..d31070546 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/question_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/question_handling_test.exs @@ -33,8 +33,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do assert object.data["context"] == "tag:mastodon.sdf.org,2019-05-10:objectId=15095122:objectType=Conversation" - assert object.data["context_id"] - assert object.data["anyOf"] == [] assert Enum.sort(object.data["oneOf"]) == @@ -68,7 +66,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do reply_object = Object.normalize(reply_activity, fetch: false) assert reply_object.data["context"] == object.data["context"] - assert reply_object.data["context_id"] == object.data["context_id"] end test "Mastodon Question activity with HTML tags in plaintext" do diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index 335fe1a30..b254e5591 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -227,7 +227,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert is_nil(modified["object"]["like_count"]) assert is_nil(modified["object"]["announcements"]) assert is_nil(modified["object"]["announcement_count"]) - assert is_nil(modified["object"]["context_id"]) assert is_nil(modified["object"]["generator"]) end @@ -242,7 +241,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert is_nil(modified["object"]["like_count"]) assert is_nil(modified["object"]["announcements"]) assert is_nil(modified["object"]["announcement_count"]) - assert is_nil(modified["object"]["context_id"]) assert is_nil(modified["object"]["likes"]) end diff --git a/test/pleroma/web/activity_pub/utils_test.exs b/test/pleroma/web/activity_pub/utils_test.exs index 447621718..e42893849 100644 --- a/test/pleroma/web/activity_pub/utils_test.exs +++ b/test/pleroma/web/activity_pub/utils_test.exs @@ -429,7 +429,6 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do object = Object.normalize(note_activity, fetch: false) res = Utils.lazy_put_activity_defaults(%{"context" => object.data["id"]}) assert res["context"] == object.data["id"] - assert res["context_id"] == object.id assert res["id"] assert res["published"] end @@ -437,7 +436,6 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do test "returns map with fake id and published data" do assert %{ "context" => "pleroma:fakecontext", - "context_id" => -1, "id" => "pleroma:fakeid", "published" => _ } = Utils.lazy_put_activity_defaults(%{}, true) @@ -454,13 +452,11 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do }) assert res["context"] == object.data["id"] - assert res["context_id"] == object.id assert res["id"] assert res["published"] assert res["object"]["id"] assert res["object"]["published"] assert res["object"]["context"] == object.data["id"] - assert res["object"]["context_id"] == object.id end end diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs index 5b2019969..33ede0f04 100644 --- a/test/pleroma/web/common_api/utils_test.exs +++ b/test/pleroma/web/common_api/utils_test.exs @@ -273,22 +273,6 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do end end - describe "context_to_conversation_id" do - test "creates a mapping object" do - conversation_id = Utils.context_to_conversation_id("random context") - object = Object.get_by_ap_id("random context") - - assert conversation_id == object.id - end - - test "returns an existing mapping for an existing object" do - {:ok, object} = Object.context_mapping("random context") |> Repo.insert() - conversation_id = Utils.context_to_conversation_id("random context") - - assert conversation_id == object.id - end - end - describe "formats date to asctime" do test "when date is in ISO 8601 format" do date = DateTime.utc_now() |> DateTime.to_iso8601() @@ -517,17 +501,6 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do end end - describe "conversation_id_to_context/1" do - test "returns id" do - object = insert(:note) - assert Utils.conversation_id_to_context(object.id) == object.data["id"] - end - - test "returns error if object not found" do - assert Utils.conversation_id_to_context("123") == {:error, "No such conversation"} - end - end - describe "maybe_notify_mentioned_recipients/2" do test "returns recipients when activity is not `Create`" do activity = insert(:like_activity) diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index 5d81c92b9..fa71f86f2 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -226,7 +226,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do object_data = Object.normalize(note, fetch: false).data user = User.get_cached_by_ap_id(note.data["actor"]) - convo_id = Utils.context_to_conversation_id(object_data["context"]) + convo_id = :erlang.crc32(object_data["context"]) status = StatusView.render("show.json", %{activity: note}) From 7f71e3d0fe347920834be8c8e28d9c7f5b169e9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Sat, 6 Aug 2022 03:32:58 +0200 Subject: [PATCH 077/123] CommonFields: remove context_id --- .../web/activity_pub/object_validators/common_fields.ex | 2 -- lib/pleroma/web/mastodon_api/views/status_view.ex | 3 --- 2 files changed, 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex index 8e768ffbf..095bd0da2 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex @@ -51,8 +51,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do field(:summary, :string) field(:context, :string) - # short identifier for PleromaFE to group statuses by context - field(:context_id, :integer) field(:sensitive, :boolean, default: false) field(:replies_count, :integer, default: 0) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 31a0c420f..3ffe55c5d 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -57,9 +57,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end) end - defp get_context_id(%{data: %{"context_id" => context_id}}) when not is_nil(context_id), - do: context_id - defp get_context_id(%{data: %{"context" => context}}) when is_binary(context), do: :erlang.crc32(context) From a9111bcaf2ba2371a1021dc171dab50615a4c040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Sun, 7 Aug 2022 20:37:17 +0200 Subject: [PATCH 078/123] StatusView: clear MSB on calculated conversation_id This field seems to be a left-over from the StatusNet era. If your application uses `pleroma.conversation_id`: this field is deprecated. It is currently stubbed instead by doing a CRC32 of the context, and clearing the MSB to avoid overflow exceptions with signed integers on the different clients using this field (Java/Kotlin code, mostly; see Husky and probably other mobile clients.) This should be removed in a future version of Pleroma. Pleroma-FE currently depends on this field, as well. --- lib/pleroma/web/mastodon_api/views/status_view.ex | 15 +++++++++++++-- test/pleroma/web/common_api/utils_test.exs | 1 - .../web/mastodon_api/views/status_view_test.exs | 3 +-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 3ffe55c5d..5cb524f56 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -57,8 +57,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end) end - defp get_context_id(%{data: %{"context" => context}}) when is_binary(context), - do: :erlang.crc32(context) + # DEPRECATED This field seems to be a left-over from the StatusNet era. + # If your application uses `pleroma.conversation_id`: this field is deprecated. + # It is currently stubbed instead by doing a CRC32 of the context, and + # clearing the MSB to avoid overflow exceptions with signed integers on the + # different clients using this field (Java/Kotlin code, mostly; see Husky.) + # This should be removed in a future version of Pleroma. Pleroma-FE currently + # depends on this field, as well. + defp get_context_id(%{data: %{"context" => context}}) when is_binary(context) do + use Bitwise + + :erlang.crc32(context) + |> band(bnot(0x8000_0000)) + end defp get_context_id(_), do: nil diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs index 33ede0f04..b538c5979 100644 --- a/test/pleroma/web/common_api/utils_test.exs +++ b/test/pleroma/web/common_api/utils_test.exs @@ -4,7 +4,6 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do alias Pleroma.Builders.UserBuilder - alias Pleroma.Object alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.ActivityDraft alias Pleroma.Web.CommonAPI.Utils diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index fa71f86f2..4429f73c4 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -14,7 +14,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do alias Pleroma.User alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI - alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.StatusView @@ -226,7 +225,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do object_data = Object.normalize(note, fetch: false).data user = User.get_cached_by_ap_id(note.data["actor"]) - convo_id = :erlang.crc32(object_data["context"]) + convo_id = :erlang.crc32(object_data["context"]) |> Bitwise.band(Bitwise.bnot(0x8000_0000)) status = StatusView.render("show.json", %{activity: note}) From def0f5dc2e76b7c4ac22b393abf7f5de5e197659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Sun, 7 Aug 2022 20:39:35 +0200 Subject: [PATCH 079/123] StatusView: implement pleroma.context field This field replaces the now deprecated conversation_id field, and now exposes the ActivityPub object `context` directly via the MastoAPI instead of relying on StatusNet-era data concepts. --- lib/pleroma/web/api_spec/schemas/status.ex | 9 ++++++++- lib/pleroma/web/mastodon_api/views/status_view.ex | 1 + .../mastodon_api/controllers/status_controller_test.exs | 2 ++ test/pleroma/web/mastodon_api/views/status_view_test.exs | 3 +++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 6e6e30315..8c19a9d9f 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -142,9 +142,15 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do description: "A map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`" }, + context: %Schema{ + type: :string, + description: "The thread identifier the status is associated with" + }, conversation_id: %Schema{ type: :integer, - description: "The ID of the AP context the status is associated with (if any)" + deprecated: true, + description: + "The ID of the AP context the status is associated with (if any); deprecated, please use `context` instead" }, direct_conversation_id: %Schema{ type: :integer, @@ -319,6 +325,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do "pinned" => false, "pleroma" => %{ "content" => %{"text/plain" => "foobar"}, + "context" => "http://localhost:4001/objects/8b4c0c80-6a37-4d2a-b1b9-05a19e3875aa", "conversation_id" => 345_972, "direct_conversation_id" => nil, "emoji_reactions" => [], diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 5cb524f56..a4d6cd807 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -375,6 +375,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do pleroma: %{ local: activity.local, conversation_id: get_context_id(activity), + context: object.data["context"], in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, content: %{"text/plain" => content_plaintext}, spoiler_text: %{"text/plain" => summary}, diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index dc6912b7b..b85b7da9a 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -262,6 +262,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do |> Map.put("url", nil) |> Map.put("uri", nil) |> Map.put("created_at", nil) + |> Kernel.put_in(["pleroma", "context"], nil) |> Kernel.put_in(["pleroma", "conversation_id"], nil) fake_conn = @@ -285,6 +286,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do |> Map.put("url", nil) |> Map.put("uri", nil) |> Map.put("created_at", nil) + |> Kernel.put_in(["pleroma", "context"], nil) |> Kernel.put_in(["pleroma", "conversation_id"], nil) assert real_status == fake_status diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index 4429f73c4..f9cdfee4e 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -17,6 +17,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.StatusView + require Bitwise + import Pleroma.Factory import Tesla.Mock import OpenApiSpex.TestAssertions @@ -278,6 +280,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do pleroma: %{ local: true, conversation_id: convo_id, + context: object_data["context"], in_reply_to_account_acct: nil, content: %{"text/plain" => HTML.strip_tags(object_data["content"])}, spoiler_text: %{"text/plain" => HTML.strip_tags(object_data["summary"])}, From c559c240d1a56f05fc70f69ae6b8c0809026fa2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Sun, 7 Aug 2022 20:41:24 +0200 Subject: [PATCH 080/123] Migrations: delete context objects These objects represent from 30 to 70% of the rows on the objects table, based on numbers from a few live instances (single-user, small, large.) As those pseudo-objects prevent creating objects with those actual IDs, deleting them is a better solution. This could have happened if an object used another object's ID as its context. --- ...5023_data_migration_delete_context_objects.exs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs diff --git a/priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs b/priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs new file mode 100644 index 000000000..debb474b2 --- /dev/null +++ b/priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs @@ -0,0 +1,15 @@ +defmodule Pleroma.Repo.Migrations.DataMigrationDeleteContextObjects do + use Ecto.Migration + + require Logger + + @doc "This migration removes objects created exclusively for contexts, containing only an `id` field." + + def change do + Logger.warn( + "This migration can take a very long time to execute, depending on your database size. Please be patient, Pleroma-tan is doing her best!\n" + ) + + execute("DELETE FROM objects WHERE (data->>'type') IS NULL;") + end +end From 3b6784b1de8454ab8c009ac688f6c62039117742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Tue, 2 Aug 2022 17:30:36 +0200 Subject: [PATCH 081/123] CreateGenericValidator: fix reply context fixing Incoming Pleroma replies to a Misskey thread were rejected due to a broken context fix, which caused them to not be visible until a non-Pleroma user interacted with the replies. This fix properly sets the post-fix object context to its parent Create activity as well, if it was changed. --- .../create_generic_validator.ex | 2 +- ...reate-pleroma-reply-to-misskey-thread.json | 61 +++++++++++++++++ .../tesla_mock/helene@p.helene.moe.json | 50 ++++++++++++++ .../mametsuko@mk.absturztau.be.json | 65 +++++++++++++++++++ .../mk.absturztau.be-93e7nm8wqg.json | 44 +++++++++++++ .../p.helene.moe-AM7S6vZQmL6pI9TgPY.json | 36 ++++++++++ .../create_generic_validator_test.exs | 9 ++- .../web/activity_pub/transmogrifier_test.exs | 15 +++-- test/support/http_request_mock.ex | 36 ++++++++++ 9 files changed, 309 insertions(+), 9 deletions(-) create mode 100644 test/fixtures/create-pleroma-reply-to-misskey-thread.json create mode 100644 test/fixtures/tesla_mock/helene@p.helene.moe.json create mode 100644 test/fixtures/tesla_mock/mametsuko@mk.absturztau.be.json create mode 100644 test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg.json create mode 100644 test/fixtures/tesla_mock/p.helene.moe-AM7S6vZQmL6pI9TgPY.json diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index c9a621cb1..2395abfd4 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -75,7 +75,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do data |> CommonFixes.fix_actor() - |> Map.put_new("context", object["context"]) + |> Map.put("context", object["context"]) |> fix_addressing(object) end diff --git a/test/fixtures/create-pleroma-reply-to-misskey-thread.json b/test/fixtures/create-pleroma-reply-to-misskey-thread.json new file mode 100644 index 000000000..0c31efa76 --- /dev/null +++ b/test/fixtures/create-pleroma-reply-to-misskey-thread.json @@ -0,0 +1,61 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://p.helene.moe/schemas/litepub-0.1.jsonld", + { + "@language": "und" + } + ], + "actor": "https://p.helene.moe/users/helene", + "attachment": [], + "attributedTo": "https://p.helene.moe/users/helene", + "cc": [ + "https://p.helene.moe/users/helene/followers" + ], + "context": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d", + "conversation": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d", + "directMessage": false, + "id": "https://p.helene.moe/activities/5f80db86-a9bb-4883-9845-fbdbd1478f3a", + "object": { + "actor": "https://p.helene.moe/users/helene", + "attachment": [], + "attributedTo": "https://p.helene.moe/users/helene", + "cc": [ + "https://p.helene.moe/users/helene/followers" + ], + "content": "@mametsuko meow", + "context": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d", + "conversation": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d", + "id": "https://p.helene.moe/objects/fd5910ac-d9dc-412e-8d1d-914b203296c4", + "inReplyTo": "https://mk.absturztau.be/notes/93e7nm8wqg", + "published": "2022-08-02T13:46:58.403996Z", + "sensitive": null, + "source": "@mametsuko@mk.absturztau.be meow", + "summary": "", + "tag": [ + { + "href": "https://mk.absturztau.be/users/8ozbzjs3o8", + "name": "@mametsuko@mk.absturztau.be", + "type": "Mention" + } + ], + "to": [ + "https://mk.absturztau.be/users/8ozbzjs3o8", + "https://www.w3.org/ns/activitystreams#Public" + ], + "type": "Note" + }, + "published": "2022-08-02T13:46:58.403883Z", + "tag": [ + { + "href": "https://mk.absturztau.be/users/8ozbzjs3o8", + "name": "@mametsuko@mk.absturztau.be", + "type": "Mention" + } + ], + "to": [ + "https://mk.absturztau.be/users/8ozbzjs3o8", + "https://www.w3.org/ns/activitystreams#Public" + ], + "type": "Create" +} \ No newline at end of file diff --git a/test/fixtures/tesla_mock/helene@p.helene.moe.json b/test/fixtures/tesla_mock/helene@p.helene.moe.json new file mode 100644 index 000000000..d7444817f --- /dev/null +++ b/test/fixtures/tesla_mock/helene@p.helene.moe.json @@ -0,0 +1,50 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://p.helene.moe/schemas/litepub-0.1.jsonld", + { + "@language": "und" + } + ], + "alsoKnownAs": [], + "attachment": [ + { + "name": "Timezone", + "type": "PropertyValue", + "value": "UTC+2 (Paris/Berlin)" + } + ], + "capabilities": { + "acceptsChatMessages": true + }, + "discoverable": true, + "endpoints": { + "oauthAuthorizationEndpoint": "https://p.helene.moe/oauth/authorize", + "oauthRegistrationEndpoint": "https://p.helene.moe/api/v1/apps", + "oauthTokenEndpoint": "https://p.helene.moe/oauth/token", + "sharedInbox": "https://p.helene.moe/inbox", + "uploadMedia": "https://p.helene.moe/api/ap/upload_media" + }, + "featured": "https://p.helene.moe/users/helene/collections/featured", + "followers": "https://p.helene.moe/users/helene/followers", + "following": "https://p.helene.moe/users/helene/following", + "icon": { + "type": "Image", + "url": "https://p.helene.moe/media/9a39209daa5a66b7ebb0547b08bf8360aa9d8d65a4ffba2603c6ffbe6aecb432.jpg" + }, + "id": "https://p.helene.moe/users/helene", + "inbox": "https://p.helene.moe/users/helene/inbox", + "manuallyApprovesFollowers": false, + "name": "Hélène", + "outbox": "https://p.helene.moe/users/helene/outbox", + "preferredUsername": "helene", + "publicKey": { + "id": "https://p.helene.moe/users/helene#main-key", + "owner": "https://p.helene.moe/users/helene", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtoSBPU/VS2Kx3f6ap3zv\nZVacJsgUfaoFb3c2ii/FRh9RmRVlarq8sJXcjsQt1e0oxWaWJaIDDwyKZPt6hXae\nrY/AiGGeNu+NA+BtY7l7+9Yu67HUyT62+1qAwYHKBXX3fLOPs/YmQI0Tt0c4wKAG\nKEkiYsRizghgpzUC6jqdKV71DJkUZ8yhckCGb2fLko1ajbWEssdaP51aLsyRMyC2\nuzeWrxtD4O/HG0ea4S6y5X6hnsAHIK4Y3nnyIQ6pn4tOsl3HgqkjXE9MmZSvMCFx\nBq89TfZrVXNa2gSZdZLdbbJstzEScQWNt1p6tA6rM+e4JXYGr+rMdF3G+jV7afI2\nFQIDAQAB\n-----END PUBLIC KEY-----\n\n" + }, + "summary": "I can speak: Français, English, Deutsch (nicht sehr gut), 日本語 (not very well)", + "tag": [], + "type": "Person", + "url": "https://p.helene.moe/users/helene" +} \ No newline at end of file diff --git a/test/fixtures/tesla_mock/mametsuko@mk.absturztau.be.json b/test/fixtures/tesla_mock/mametsuko@mk.absturztau.be.json new file mode 100644 index 000000000..d8c13f775 --- /dev/null +++ b/test/fixtures/tesla_mock/mametsuko@mk.absturztau.be.json @@ -0,0 +1,65 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "sensitive": "as:sensitive", + "Hashtag": "as:Hashtag", + "quoteUrl": "as:quoteUrl", + "toot": "http://joinmastodon.org/ns#", + "Emoji": "toot:Emoji", + "featured": "toot:featured", + "discoverable": "toot:discoverable", + "schema": "http://schema.org#", + "PropertyValue": "schema:PropertyValue", + "value": "schema:value", + "misskey": "https://misskey-hub.net/ns#", + "_misskey_content": "misskey:_misskey_content", + "_misskey_quote": "misskey:_misskey_quote", + "_misskey_reaction": "misskey:_misskey_reaction", + "_misskey_votes": "misskey:_misskey_votes", + "_misskey_talk": "misskey:_misskey_talk", + "isCat": "misskey:isCat", + "vcard": "http://www.w3.org/2006/vcard/ns#" + } + ], + "type": "Person", + "id": "https://mk.absturztau.be/users/8ozbzjs3o8", + "inbox": "https://mk.absturztau.be/users/8ozbzjs3o8/inbox", + "outbox": "https://mk.absturztau.be/users/8ozbzjs3o8/outbox", + "followers": "https://mk.absturztau.be/users/8ozbzjs3o8/followers", + "following": "https://mk.absturztau.be/users/8ozbzjs3o8/following", + "featured": "https://mk.absturztau.be/users/8ozbzjs3o8/collections/featured", + "sharedInbox": "https://mk.absturztau.be/inbox", + "endpoints": { + "sharedInbox": "https://mk.absturztau.be/inbox" + }, + "url": "https://mk.absturztau.be/@mametsuko", + "preferredUsername": "mametsuko", + "name": "mametschko", + "summary": "

nya, ich bin eine Brotperson

", + "icon": { + "type": "Image", + "url": "https://mk.absturztau.be/files/webpublic-3b5594f4-fa52-4548-b4e3-c379ae2143ed", + "sensitive": false, + "name": null + }, + "image": { + "type": "Image", + "url": "https://mk.absturztau.be/files/webpublic-0d03b03d-b14b-4916-ac3d-8a137118ec84", + "sensitive": false, + "name": null + }, + "tag": [], + "manuallyApprovesFollowers": true, + "discoverable": false, + "publicKey": { + "id": "https://mk.absturztau.be/users/8ozbzjs3o8#main-key", + "type": "Key", + "owner": "https://mk.absturztau.be/users/8ozbzjs3o8", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAuN/S1spBGmh8FXI1Bt16\nXB7Cc0QutBp7UPgmDNHjOfsq0zrF4g3L1UBxvrpU0XX77XPMCd9yPvGwAYURH2mv\ntIcYuE+R90VLDmBu5MTVthcG2D874eCZ2rD2YsEYmN5AjTX7QBIqCck+qDhVWkkM\nEZ6S5Ht6IJ5Of74eKffXElQI/C6QB+9uEDOmPk0jCzgI5gw7xvJqFj/DIF4kUUAu\nA89JqaFZzZlkrSrj4cr48bLN/YOmpdaHu0BKHaDSHct4+MqlixqovgdB6RboCEDw\ne4Aeav7+Q0Y9oGIvuggg0Q+nCubnVNnaPyzd817tpPVzyZmTts+DKyDuv90SX3nR\nsPaNa5Ty60eqplUk4b7X1gSvuzBJUFBxTVV84WnjwoeoydaS6rSyjCDPGLBjaByc\nFyWMMEb/zlQyhLZfBlvT7k96wRSsMszh2hDALWmgYIhq/jNwINvALJ1GKLNHHKZ4\nyz2LnxVpRm2rWrZzbvtcnSQOt3LaPSZn8Wgwv4buyHF02iuVuIamZVtKexsE1Ixl\nIi9qa3AKEc5gOzYXhRhvHaruzoCehUbb/UHC5c8Tto8L5G1xYzjLP3qj3PT9w/wM\n+k1Ra/4JhuAnVFROOoOmx9rIELLHH7juY2nhM7plGhyt1M5gysgqEloij8QzyQU2\nZK1YlAERG2XFO6br8omhcmECAwEAAQ==\n-----END PUBLIC KEY-----\n" + }, + "isCat": true, + "vcard:Address": "Vienna, Austria" +} \ No newline at end of file diff --git a/test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg.json b/test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg.json new file mode 100644 index 000000000..1b931a9a4 --- /dev/null +++ b/test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg.json @@ -0,0 +1,44 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "sensitive": "as:sensitive", + "Hashtag": "as:Hashtag", + "quoteUrl": "as:quoteUrl", + "toot": "http://joinmastodon.org/ns#", + "Emoji": "toot:Emoji", + "featured": "toot:featured", + "discoverable": "toot:discoverable", + "schema": "http://schema.org#", + "PropertyValue": "schema:PropertyValue", + "value": "schema:value", + "misskey": "https://misskey-hub.net/ns#", + "_misskey_content": "misskey:_misskey_content", + "_misskey_quote": "misskey:_misskey_quote", + "_misskey_reaction": "misskey:_misskey_reaction", + "_misskey_votes": "misskey:_misskey_votes", + "_misskey_talk": "misskey:_misskey_talk", + "isCat": "misskey:isCat", + "vcard": "http://www.w3.org/2006/vcard/ns#" + } + ], + "id": "https://mk.absturztau.be/notes/93e7nm8wqg", + "type": "Note", + "attributedTo": "https://mk.absturztau.be/users/8ozbzjs3o8", + "summary": null, + "content": "

meow

", + "_misskey_content": "meow", + "published": "2022-08-01T11:06:49.568Z", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://mk.absturztau.be/users/8ozbzjs3o8/followers" + ], + "inReplyTo": null, + "attachment": [], + "sensitive": false, + "tag": [] +} \ No newline at end of file diff --git a/test/fixtures/tesla_mock/p.helene.moe-AM7S6vZQmL6pI9TgPY.json b/test/fixtures/tesla_mock/p.helene.moe-AM7S6vZQmL6pI9TgPY.json new file mode 100644 index 000000000..a1ef5e20b --- /dev/null +++ b/test/fixtures/tesla_mock/p.helene.moe-AM7S6vZQmL6pI9TgPY.json @@ -0,0 +1,36 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://p.helene.moe/schemas/litepub-0.1.jsonld", + { + "@language": "und" + } + ], + "actor": "https://p.helene.moe/users/helene", + "attachment": [], + "attributedTo": "https://p.helene.moe/users/helene", + "cc": [ + "https://p.helene.moe/users/helene/followers" + ], + "content": "@mametsuko meow", + "context": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d", + "conversation": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d", + "id": "https://p.helene.moe/objects/fd5910ac-d9dc-412e-8d1d-914b203296c4", + "inReplyTo": "https://mk.absturztau.be/notes/93e7nm8wqg", + "published": "2022-08-02T13:46:58.403996Z", + "sensitive": null, + "source": "@mametsuko@mk.absturztau.be meow", + "summary": "", + "tag": [ + { + "href": "https://mk.absturztau.be/users/8ozbzjs3o8", + "name": "@mametsuko@mk.absturztau.be", + "type": "Mention" + } + ], + "to": [ + "https://mk.absturztau.be/users/8ozbzjs3o8", + "https://www.w3.org/ns/activitystreams#Public" + ], + "type": "Note" +} \ No newline at end of file diff --git a/test/pleroma/web/activity_pub/object_validators/create_generic_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/create_generic_validator_test.exs index 0a5b44beb..e771260c9 100644 --- a/test/pleroma/web/activity_pub/object_validators/create_generic_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/create_generic_validator_test.exs @@ -23,10 +23,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidatorTest do {:ok, object_data} = ObjectValidator.cast_and_apply(note_activity["object"]) meta = [object_data: ObjectValidator.stringify_keys(object_data)] - %{valid?: true} = CreateGenericValidator.cast_and_validate(note_activity, meta) + assert %{valid?: true} = CreateGenericValidator.cast_and_validate(note_activity, meta) end - test "a Create/Note with mismatched context is invalid" do + test "a Create/Note with mismatched context uses the Note's context" do user = insert(:user) note = %{ @@ -54,6 +54,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidatorTest do {:ok, object_data} = ObjectValidator.cast_and_apply(note_activity["object"]) meta = [object_data: ObjectValidator.stringify_keys(object_data)] - %{valid?: false} = CreateGenericValidator.cast_and_validate(note_activity, meta) + validated = CreateGenericValidator.cast_and_validate(note_activity, meta) + + assert validated.valid? + assert {:context, note["context"]} in validated.changes end end diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index b254e5591..a07972895 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -108,15 +108,20 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert activity.data["type"] == "Move" end - test "a reply with mismatched context is rejected" do - insert(:user, ap_id: "https://macgirvin.com/channel/mike") + test "it fixes both the Create and object contexts in a reply" do + insert(:user, ap_id: "https://mk.absturztau.be/users/8ozbzjs3o8") + insert(:user, ap_id: "https://p.helene.moe/users/helene") - note_activity = - "test/fixtures/roadhouse-create-activity.json" + create_activity = + "test/fixtures/create-pleroma-reply-to-misskey-thread.json" |> File.read!() |> Jason.decode!() - assert {:error, _} = Transmogrifier.handle_incoming(note_activity) + assert {:ok, %Activity{} = activity} = Transmogrifier.handle_incoming(create_activity) + + object = Object.normalize(activity, fetch: false) + + assert activity.data["context"] == object.data["context"] end end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index eb844e469..c1f0f7af1 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1401,6 +1401,42 @@ defmodule HttpRequestMock do }} end + def get("https://mk.absturztau.be/users/8ozbzjs3o8", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/mametsuko@mk.absturztau.be.json"), + headers: activitypub_object_headers() + }} + end + + def get("https://p.helene.moe/users/helene", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/helene@p.helene.moe.json"), + headers: activitypub_object_headers() + }} + end + + def get("https://mk.absturztau.be/notes/93e7nm8wqg", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg.json"), + headers: activitypub_object_headers() + }} + end + + def get("https://p.helene.moe/objects/fd5910ac-d9dc-412e-8d1d-914b203296c4", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/p.helene.moe-AM7S6vZQmL6pI9TgPY.json"), + headers: activitypub_object_headers() + }} + end + def get(url, query, body, headers) do {:error, "Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"} From cbdc13b76710e854c96f504526aff9da83b90ce5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 10 Aug 2022 17:09:58 -0400 Subject: [PATCH 082/123] Fix Varnish 7 support by ensuring Media Preview Proxy fetches headers with a capitalized HEAD verb --- lib/pleroma/web/media_proxy/media_proxy_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 3d6716d43..d2ad62c13 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -54,7 +54,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do media_proxy_url = MediaProxy.url(url) with {:ok, %{status: status} = head_response} when status in 200..299 <- - Pleroma.HTTP.request("head", media_proxy_url, [], [], pool: :media) do + Pleroma.HTTP.request("HEAD", media_proxy_url, [], [], pool: :media) do content_type = Tesla.get_header(head_response, "content-type") content_length = Tesla.get_header(head_response, "content-length") content_length = content_length && String.to_integer(content_length) From 243ed7d60f4b649a9dfd57a278077c084df9e309 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 10 Aug 2022 17:18:06 -0400 Subject: [PATCH 083/123] Update the recommended VCL configuration --- installation/pleroma.vcl | 41 +++++----------------------------------- 1 file changed, 5 insertions(+), 36 deletions(-) diff --git a/installation/pleroma.vcl b/installation/pleroma.vcl index 4752510ea..4eb2f3cfa 100644 --- a/installation/pleroma.vcl +++ b/installation/pleroma.vcl @@ -1,4 +1,5 @@ # Recommended varnishncsa logging format: '%h %l %u %t "%m %{X-Forwarded-Proto}i://%{Host}i%U%q %H" %s %b "%{Referer}i" "%{User-agent}i"' +# Please use Varnish 7.0+ for proper Range Requests / Chunked encoding support vcl 4.1; import std; @@ -22,11 +23,6 @@ sub vcl_recv { set req.http.X-Forwarded-Proto = "https"; } - # CHUNKED SUPPORT - if (req.http.Range ~ "bytes=") { - set req.http.x-range = req.http.Range; - } - # Pipe if WebSockets request is coming through if (req.http.upgrade ~ "(?i)websocket") { return (pipe); @@ -35,9 +31,9 @@ sub vcl_recv { # Allow purging of the cache if (req.method == "PURGE") { if (!client.ip ~ purge) { - return(synth(405,"Not allowed.")); + return (synth(405,"Not allowed.")); } - return(purge); + return (purge); } } @@ -53,17 +49,11 @@ sub vcl_backend_response { return (retry); } - # CHUNKED SUPPORT - if (bereq.http.x-range ~ "bytes=" && beresp.status == 206) { - set beresp.ttl = 10m; - set beresp.http.CR = beresp.http.content-range; - } - # Bypass cache for large files # 50000000 ~ 50MB if (std.integer(beresp.http.content-length, 0) > 50000000) { set beresp.uncacheable = true; - return(deliver); + return (deliver); } # Don't cache objects that require authentication @@ -94,7 +84,7 @@ sub vcl_synth { if (resp.status == 750) { set resp.status = 301; set resp.http.Location = req.http.x-redir; - return(deliver); + return (deliver); } } @@ -106,25 +96,12 @@ sub vcl_pipe { } } -sub vcl_hash { - # CHUNKED SUPPORT - if (req.http.x-range ~ "bytes=") { - hash_data(req.http.x-range); - unset req.http.Range; - } -} - sub vcl_backend_fetch { # Be more lenient for slow servers on the fediverse if (bereq.url ~ "^/proxy/") { set bereq.first_byte_timeout = 300s; } - # CHUNKED SUPPORT - if (bereq.http.x-range) { - set bereq.http.Range = bereq.http.x-range; - } - if (bereq.retries == 0) { # Clean up the X-Varnish-Backend-503 flag that is used internally # to mark broken backend responses that should be retried. @@ -143,14 +120,6 @@ sub vcl_backend_fetch { } } -sub vcl_deliver { - # CHUNKED SUPPORT - if (resp.http.CR) { - set resp.http.Content-Range = resp.http.CR; - unset resp.http.CR; - } -} - sub vcl_backend_error { # Retry broken backend responses. set bereq.http.X-Varnish-Backend-503 = "1"; From 73b4d0d9a74348bbd5729a0ce8db41f8595948f1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 10 Aug 2022 21:46:56 +0000 Subject: [PATCH 084/123] Fix the mocks to use uppercase as well --- .../media_proxy_controller_test.exs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs index 5ace2eee9..5246bf0c4 100644 --- a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs +++ b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs @@ -158,7 +158,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do media_proxy_url: media_proxy_url } do Tesla.Mock.mock(fn - %{method: "head", url: ^media_proxy_url} -> + %{method: "HEAD", url: ^media_proxy_url} -> %Tesla.Env{status: 500, body: ""} end) @@ -173,7 +173,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do media_proxy_url: media_proxy_url } do Tesla.Mock.mock(fn - %{method: "head", url: ^media_proxy_url} -> + %{method: "HEAD", url: ^media_proxy_url} -> %Tesla.Env{status: 200, body: "", headers: [{"content-type", "application/pdf"}]} end) @@ -193,7 +193,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do clear_config([:media_preview_proxy, :min_content_length], 1_000_000_000) Tesla.Mock.mock(fn - %{method: "head", url: ^media_proxy_url} -> + %{method: "HEAD", url: ^media_proxy_url} -> %Tesla.Env{ status: 200, body: "", @@ -218,7 +218,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do media_proxy_url: media_proxy_url } do Tesla.Mock.mock(fn - %{method: "head", url: ^media_proxy_url} -> + %{method: "HEAD", url: ^media_proxy_url} -> %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/gif"}]} end) @@ -236,7 +236,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do media_proxy_url: media_proxy_url } do Tesla.Mock.mock(fn - %{method: "head", url: ^media_proxy_url} -> + %{method: "HEAD", url: ^media_proxy_url} -> %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]} end) @@ -256,7 +256,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do clear_config([:media_preview_proxy, :min_content_length], 100_000) Tesla.Mock.mock(fn - %{method: "head", url: ^media_proxy_url} -> + %{method: "HEAD", url: ^media_proxy_url} -> %Tesla.Env{ status: 200, body: "", @@ -278,7 +278,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do assert_dependencies_installed() Tesla.Mock.mock(fn - %{method: "head", url: ^media_proxy_url} -> + %{method: "HEAD", url: ^media_proxy_url} -> %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/png"}]} %{method: :get, url: ^media_proxy_url} -> @@ -300,7 +300,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do assert_dependencies_installed() Tesla.Mock.mock(fn - %{method: "head", url: ^media_proxy_url} -> + %{method: "HEAD", url: ^media_proxy_url} -> %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]} %{method: :get, url: ^media_proxy_url} -> @@ -320,7 +320,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do media_proxy_url: media_proxy_url } do Tesla.Mock.mock(fn - %{method: "head", url: ^media_proxy_url} -> + %{method: "HEAD", url: ^media_proxy_url} -> %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]} %{method: :get, url: ^media_proxy_url} -> From 80c32ae00b0a67a19fcddb5ca04651468dfe246d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 12 Aug 2022 15:06:45 -0400 Subject: [PATCH 085/123] Document the changes for Varnish 7.0+ compatibility and RFC compliance --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2ed9bbad..8d0ef4e11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - Allow users to remove their emails if instance does not need email to register - Uploadfilter `Pleroma.Upload.Filter.Exiftool` has been renamed to `Pleroma.Upload.Filter.Exiftool.StripLocation` +- Updated the recommended pleroma.vcl configuration for Varnish to target Varnish 7.0+ ### Added - `activeMonth` and `activeHalfyear` fields in NodeInfo usage.users object @@ -48,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Fixed crash when pinned_objects is nil - Fixed slow timelines when there are a lot of deactivated users - Fixed account deletion API +- Fixed lowercase HTTP HEAD method in the Media Proxy Preview code ### Removed From bb02ee99f58e378e33162211f41fe5979d5da8ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Wed, 10 Aug 2022 04:21:28 +0200 Subject: [PATCH 086/123] CommonFixes: more predictable context generation `context` fields for objects and activities can now be generated based on the object/activity `inReplyTo` field or its ActivityPub ID, as a fallback method in cases where `context` fields are missing for incoming activities and objects. --- .../object_validators/common_fixes.ex | 5 ++- .../mk.absturztau.be-93e7nm8wqg-activity.json | 1 + .../transmogrifier/note_handling_test.exs | 38 +++++++++++++++++++ test/support/http_request_mock.ex | 17 +++++++++ 4 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index c7e292bec..add46d561 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -22,7 +22,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do end def fix_object_defaults(data) do - context = Utils.maybe_create_context(data["context"] || data["conversation"]) + context = + Utils.maybe_create_context( + data["context"] || data["conversation"] || data["inReplyTo"] || data["id"] + ) %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"]) diff --git a/test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json b/test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json new file mode 100644 index 000000000..b45ab78e4 --- /dev/null +++ b/test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json @@ -0,0 +1 @@ +{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","sensitive":"as:sensitive","Hashtag":"as:Hashtag","quoteUrl":"as:quoteUrl","toot":"http://joinmastodon.org/ns#","Emoji":"toot:Emoji","featured":"toot:featured","discoverable":"toot:discoverable","schema":"http://schema.org#","PropertyValue":"schema:PropertyValue","value":"schema:value","misskey":"https://misskey-hub.net/ns#","_misskey_content":"misskey:_misskey_content","_misskey_quote":"misskey:_misskey_quote","_misskey_reaction":"misskey:_misskey_reaction","_misskey_votes":"misskey:_misskey_votes","_misskey_talk":"misskey:_misskey_talk","isCat":"misskey:isCat","vcard":"http://www.w3.org/2006/vcard/ns#"}],"id":"https://mk.absturztau.be/notes/93e7nm8wqg/activity","actor":"https://mk.absturztau.be/users/8ozbzjs3o8","type":"Create","published":"2022-08-01T11:06:49.568Z","object":{"id":"https://mk.absturztau.be/notes/93e7nm8wqg","type":"Note","attributedTo":"https://mk.absturztau.be/users/8ozbzjs3o8","summary":null,"content":"

meow

","_misskey_content":"meow","published":"2022-08-01T11:06:49.568Z","to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://mk.absturztau.be/users/8ozbzjs3o8/followers"],"inReplyTo":null,"attachment":[],"sensitive":false,"tag":[]},"to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://mk.absturztau.be/users/8ozbzjs3o8/followers"]} \ No newline at end of file diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index b00fd919b..7c406fbd0 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -707,4 +707,42 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do } ] end + + test "the standalone note uses its own ID when context is missing" do + insert(:user, ap_id: "https://mk.absturztau.be/users/8ozbzjs3o8") + + activity = + "test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json" + |> File.read!() + |> Jason.decode!() + + {:ok, %Activity{} = modified} = Transmogrifier.handle_incoming(activity) + object = Object.normalize(modified, fetch: false) + + assert object.data["context"] == object.data["id"] + assert modified.data["context"] == object.data["id"] + end + + test "the reply note uses its parent's ID when context is missing and reply is unreachable" do + insert(:user, ap_id: "https://mk.absturztau.be/users/8ozbzjs3o8") + + activity = + "test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json" + |> File.read!() + |> Jason.decode!() + + object = + activity["object"] + |> Map.put("inReplyTo", "https://404.site/object/went-to-buy-milk") + + activity = + activity + |> Map.put("object", object) + + {:ok, %Activity{} = modified} = Transmogrifier.handle_incoming(activity) + object = Object.normalize(modified, fetch: false) + + assert object.data["context"] == object.data["inReplyTo"] + assert modified.data["context"] == object.data["inReplyTo"] + end end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index c1f0f7af1..7f6065579 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1084,6 +1084,14 @@ defmodule HttpRequestMock do }} end + def get("https://404.site" <> _, _, _, _) do + {:ok, + %Tesla.Env{ + status: 404, + body: "" + }} + end + def get( "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=acct:lain@zetsubou.xn--q9jyb4c", _, @@ -1428,6 +1436,15 @@ defmodule HttpRequestMock do }} end + def get("https://mk.absturztau.be/notes/93e7nm8wqg/activity", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json"), + headers: activitypub_object_headers() + }} + end + def get("https://p.helene.moe/objects/fd5910ac-d9dc-412e-8d1d-914b203296c4", _, _, _) do {:ok, %Tesla.Env{ From 88c1c76d3eca3412d1e02008f1b8d96fe8fe0b96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Mon, 15 Aug 2022 01:15:23 +0200 Subject: [PATCH 087/123] Migrations: delete contexts with BaseMigrator Due to the lengthiness of this task, the migration has been adapted into a BaseMigrator migration, running in the background instead. --- config/config.exs | 2 + config/description.exs | 21 +++ lib/pleroma/application.ex | 3 +- lib/pleroma/data_migration.ex | 1 + .../context_objects_deletion_migrator.ex | 139 ++++++++++++++++++ ..._data_migration_delete_context_objects.exs | 13 +- 6 files changed, 173 insertions(+), 6 deletions(-) create mode 100644 lib/pleroma/migrators/context_objects_deletion_migrator.ex diff --git a/config/config.exs b/config/config.exs index 0fc959807..eadc255cc 100644 --- a/config/config.exs +++ b/config/config.exs @@ -673,6 +673,8 @@ config :pleroma, :features, improved_hashtag_timeline: :auto config :pleroma, :populate_hashtags_table, fault_rate_allowance: 0.01 +config :pleroma, :delete_context_objects, fault_rate_allowance: 0.01 + config :pleroma, :env, Mix.env() config :http_signatures, diff --git a/config/description.exs b/config/description.exs index c6c6b1b5d..c28447b37 100644 --- a/config/description.exs +++ b/config/description.exs @@ -495,6 +495,27 @@ config :pleroma, :config_description, [ } ] }, + %{ + group: :pleroma, + key: :delete_context_objects, + type: :group, + description: "`delete_context_objects` background migration settings", + children: [ + %{ + key: :fault_rate_allowance, + type: :float, + description: + "Max accepted rate of objects that failed in the migration. Any value from 0.0 which tolerates no errors to 1.0 which will enable the feature even if context object deletion failed for all records.", + suggestions: [0.01] + }, + %{ + key: :sleep_interval_ms, + type: :integer, + description: + "Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances)." + } + ] + }, %{ group: :pleroma, key: :instance, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index d808bc732..c546713ca 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -238,7 +238,8 @@ defmodule Pleroma.Application do defp background_migrators do [ - Pleroma.Migrators.HashtagsTableMigrator + Pleroma.Migrators.HashtagsTableMigrator, + Pleroma.Migrators.ContextObjectsDeletionMigrator ] end diff --git a/lib/pleroma/data_migration.ex b/lib/pleroma/data_migration.ex index 59d891d8d..8451678fc 100644 --- a/lib/pleroma/data_migration.ex +++ b/lib/pleroma/data_migration.ex @@ -42,4 +42,5 @@ defmodule Pleroma.DataMigration do end def populate_hashtags_table, do: get_by_name("populate_hashtags_table") + def delete_context_objects, do: get_by_name("delete_context_objects") end diff --git a/lib/pleroma/migrators/context_objects_deletion_migrator.ex b/lib/pleroma/migrators/context_objects_deletion_migrator.ex new file mode 100644 index 000000000..fb224795a --- /dev/null +++ b/lib/pleroma/migrators/context_objects_deletion_migrator.ex @@ -0,0 +1,139 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Migrators.ContextObjectsDeletionMigrator do + defmodule State do + use Pleroma.Migrators.Support.BaseMigratorState + + @impl Pleroma.Migrators.Support.BaseMigratorState + defdelegate data_migration(), to: Pleroma.DataMigration, as: :delete_context_objects + end + + use Pleroma.Migrators.Support.BaseMigrator + + alias Pleroma.Migrators.Support.BaseMigrator + alias Pleroma.Object + + @doc "This migration removes objects created exclusively for contexts, containing only an `id` field." + + @impl BaseMigrator + def feature_config_path, do: [:features, :delete_context_objects] + + @impl BaseMigrator + def fault_rate_allowance, do: Config.get([:delete_context_objects, :fault_rate_allowance], 0) + + @impl BaseMigrator + def perform do + data_migration_id = data_migration_id() + max_processed_id = get_stat(:max_processed_id, 0) + + Logger.info("Deleting context objects from `objects` (from oid: #{max_processed_id})...") + + query() + |> where([object], object.id > ^max_processed_id) + |> Repo.chunk_stream(100, :batches, timeout: :infinity) + |> Stream.each(fn objects -> + object_ids = Enum.map(objects, & &1.id) + + results = Enum.map(object_ids, &delete_context_object(&1)) + + failed_ids = + results + |> Enum.filter(&(elem(&1, 0) == :error)) + |> Enum.map(&elem(&1, 1)) + + chunk_affected_count = + results + |> Enum.filter(&(elem(&1, 0) == :ok)) + |> length() + + for failed_id <- failed_ids do + _ = + Repo.query( + "INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <> + "VALUES ($1, $2) ON CONFLICT DO NOTHING;", + [data_migration_id, failed_id] + ) + end + + _ = + Repo.query( + "DELETE FROM data_migration_failed_ids " <> + "WHERE data_migration_id = $1 AND record_id = ANY($2)", + [data_migration_id, object_ids -- failed_ids] + ) + + max_object_id = Enum.at(object_ids, -1) + + put_stat(:max_processed_id, max_object_id) + increment_stat(:iteration_processed_count, length(object_ids)) + increment_stat(:processed_count, length(object_ids)) + increment_stat(:failed_count, length(failed_ids)) + increment_stat(:affected_count, chunk_affected_count) + put_stat(:records_per_second, records_per_second()) + persist_state() + + # A quick and dirty approach to controlling the load this background migration imposes + sleep_interval = Config.get([:delete_context_objects, :sleep_interval_ms], 0) + Process.sleep(sleep_interval) + end) + |> Stream.run() + end + + @impl BaseMigrator + def query do + # Context objects have no activity type, and only one field, `id`. + # Only those context objects are without types. + from( + object in Object, + where: fragment("(?)->'type' IS NULL", object.data), + select: %{ + id: object.id + } + ) + end + + @spec delete_context_object(integer()) :: {:ok | :error, integer()} + defp delete_context_object(id) do + result = + %Object{id: id} + |> Repo.delete() + |> elem(0) + + {result, id} + end + + @impl BaseMigrator + def retry_failed do + data_migration_id = data_migration_id() + + failed_objects_query() + |> Repo.chunk_stream(100, :one) + |> Stream.each(fn object -> + with {res, _} when res != :error <- delete_context_object(object.id) do + _ = + Repo.query( + "DELETE FROM data_migration_failed_ids " <> + "WHERE data_migration_id = $1 AND record_id = $2", + [data_migration_id, object.id] + ) + end + end) + |> Stream.run() + + put_stat(:failed_count, failures_count()) + persist_state() + + force_continue() + end + + defp failed_objects_query do + from(o in Object) + |> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"), + on: dmf.record_id == o.id + ) + |> where([_o, dmf], dmf.data_migration_id == ^data_migration_id()) + |> order_by([o], asc: o.id) + end +end diff --git a/priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs b/priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs index debb474b2..84365dbe3 100644 --- a/priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs +++ b/priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs @@ -3,13 +3,16 @@ defmodule Pleroma.Repo.Migrations.DataMigrationDeleteContextObjects do require Logger - @doc "This migration removes objects created exclusively for contexts, containing only an `id` field." + def up do + dt = NaiveDateTime.utc_now() - def change do - Logger.warn( - "This migration can take a very long time to execute, depending on your database size. Please be patient, Pleroma-tan is doing her best!\n" + execute( + "INSERT INTO data_migrations(name, inserted_at, updated_at) " <> + "VALUES ('delete_context_objects', '#{dt}', '#{dt}') ON CONFLICT DO NOTHING;" ) + end - execute("DELETE FROM objects WHERE (data->>'type') IS NULL;") + def down do + execute("DELETE FROM data_migrations WHERE name = 'delete_context_objects';") end end From f41d970a592568956aa97959f28cb89cadf5f2bc Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 18 Jul 2022 15:21:27 +0100 Subject: [PATCH 088/123] fix resolution of GTS user keys --- lib/pleroma/signature.ex | 23 +++++++++++++++-------- test/pleroma/signature_test.exs | 5 +++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index dbe6fd209..ff0c56856 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -10,17 +10,14 @@ defmodule Pleroma.Signature do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + @known_suffixes ["/publickey", "/main-key"] + def key_id_to_actor_id(key_id) do uri = - URI.parse(key_id) + key_id + |> URI.parse() |> Map.put(:fragment, nil) - - uri = - if not is_nil(uri.path) and String.ends_with?(uri.path, "/publickey") do - Map.put(uri, :path, String.replace(uri.path, "/publickey", "")) - else - uri - end + |> remove_suffix(@known_suffixes) maybe_ap_id = URI.to_string(uri) @@ -36,6 +33,16 @@ defmodule Pleroma.Signature do end end + defp remove_suffix(uri, [test | rest]) do + if not is_nil(uri.path) and String.ends_with?(uri.path, test) do + Map.put(uri, :path, String.replace(uri.path, test, "")) + else + remove_suffix(uri, rest) + end + end + + defp remove_suffix(uri, []), do: uri + def fetch_public_key(conn) do with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), {:ok, actor_id} <- key_id_to_actor_id(kid), diff --git a/test/pleroma/signature_test.exs b/test/pleroma/signature_test.exs index 92d05f26c..b849cbee7 100644 --- a/test/pleroma/signature_test.exs +++ b/test/pleroma/signature_test.exs @@ -109,6 +109,11 @@ defmodule Pleroma.SignatureTest do {:ok, "https://example.com/users/1234"} end + test "it deduces the actor id for gotoSocial" do + assert Signature.key_id_to_actor_id("https://example.com/users/1234/main-key") == + {:ok, "https://example.com/users/1234"} + end + test "it calls webfinger for 'acct:' accounts" do with_mock(Pleroma.Web.WebFinger, finger: fn _ -> %{"ap_id" => "https://gensokyo.2hu/users/raymoo"} end From 61254111e59f02118cad15de49d1e0704c07030e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Wed, 17 Aug 2022 03:30:02 +0200 Subject: [PATCH 089/123] HttpSignaturePlug: accept standard (request-target) The (request-target) used by Pleroma is non-standard, but many HTTP signature implementations do it this way due to a misinterpretation of the draft 06 of HTTP signatures: "path" was interpreted as not having the query, though later examples show that it must be the absolute path with the query part of the URL as well. This behavior is kept to make sure most software (Pleroma itself, Mastodon, and probably others) do not break, but Pleroma now accepts signatures for a (request-target) containing the query, as expected by many HTTP signature libraries, and clarified in the draft 11 of HTTP signatures. Additionally, the new draft renamed (request-target) to @request-target. We now support both for incoming requests' signatures. --- lib/pleroma/web/plugs/http_signature_plug.ex | 53 +++++++++++++++++--- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex index d023754a6..4bf325218 100644 --- a/lib/pleroma/web/plugs/http_signature_plug.ex +++ b/lib/pleroma/web/plugs/http_signature_plug.ex @@ -25,21 +25,58 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do end end + defp validate_signature(conn, request_target) do + # Newer drafts for HTTP signatures now use @request-target instead of the + # old (request-target). We'll now support both for incoming signatures. + conn = + conn + |> put_req_header("(request-target)", request_target) + |> put_req_header("@request-target", request_target) + + HTTPSignatures.validate_conn(conn) + end + + defp validate_signature(conn) do + # This (request-target) is non-standard, but many implementations do it + # this way due to a misinterpretation of + # https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures-06 + # "path" was interpreted as not having the query, though later examples + # show that it must be the absolute path + query. This behavior is kept to + # make sure most software (Pleroma itself, Mastodon, and probably others) + # do not break. + request_target = String.downcase("#{conn.method}") <> " #{conn.request_path}" + + # This is the proper way to build the @request-target, as expected by + # many HTTP signature libraries, clarified in the following draft: + # https://www.ietf.org/archive/id/draft-ietf-httpbis-message-signatures-11.html#section-2.2.6 + # It is the same as before, but containing the query part as well. + proper_target = request_target <> "?#{conn.query_string}" + + cond do + # Normal, non-standard behavior but expected by Pleroma and more. + validate_signature(conn, request_target) -> + true + + # Has query string and the previous one failed: let's try the standard. + conn.query_string != "" -> + validate_signature(conn, proper_target) + + # If there's no query string and signature fails, it's rotten. + true -> + false + end + end + defp maybe_assign_valid_signature(conn) do if has_signature_header?(conn) do - # set (request-target) header to the appropriate value - # we also replace the digest header with the one we computed - request_target = String.downcase("#{conn.method}") <> " #{conn.request_path}" - + # we replace the digest header with the one we computed in DigestPlug conn = - conn - |> put_req_header("(request-target)", request_target) - |> case do + case conn do %{assigns: %{digest: digest}} = conn -> put_req_header(conn, "digest", digest) conn -> conn end - assign(conn, :valid_signature, HTTPSignatures.validate_conn(conn)) + assign(conn, :valid_signature, validate_signature(conn)) else Logger.debug("No signature header!") conn From 4661b56720b4f70eb6996bf975c4d88db9828006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Fri, 19 Aug 2022 02:45:49 +0200 Subject: [PATCH 090/123] ArticleNotePageValidator: fix replies fixing Some software, like GoToSocial, expose replies as ActivityPub Collections, but do not expose any item array directly in the object, causing validation to fail via the ObjectID validator. Now, Pleroma will drop that field in this situation too. --- .../article_note_page_validator.ex | 5 ++++- .../article_note_page_validator_test.exs | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index 57c8d1dc0..4243e0fbf 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -60,7 +60,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies), do: Map.put(data, "replies", replies) - defp fix_replies(%{"replies" => replies} = data) when is_bitstring(replies), + # TODO: Pleroma does not have any support for Collections at the moment. + # If the `replies` field is not something the ObjectID validator can handle, + # the activity/object would be rejected, which is bad behavior. + defp fix_replies(%{"replies" => replies} = data) when not is_list(replies), do: Map.drop(data, ["replies"]) defp fix_replies(data), do: data diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index e59bf6787..2dd1361ea 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -54,4 +54,17 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note) end + + test "a Note without replies/first/items validates" do + insert(:user, ap_id: "https://mastodon.social/users/emelie") + + note = + "test/fixtures/tesla_mock/status.emelie.json" + |> File.read!() + |> Jason.decode!() + |> pop_in(["replies", "first", "items"]) + |> elem(1) + + %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note) + end end From 0cee3c6e937ce7b15392a7abc5bbc30bfc80e7f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Sat, 20 Aug 2022 00:21:07 +0200 Subject: [PATCH 091/123] emoji-test: update to latest 15.0 draft --- lib/pleroma/emoji-test.txt | 125 +++++++++++++++++++++++-------------- 1 file changed, 79 insertions(+), 46 deletions(-) diff --git a/lib/pleroma/emoji-test.txt b/lib/pleroma/emoji-test.txt index dd5493366..87d093d64 100644 --- a/lib/pleroma/emoji-test.txt +++ b/lib/pleroma/emoji-test.txt @@ -1,13 +1,13 @@ # emoji-test.txt -# Date: 2021-08-26, 17:22:23 GMT -# © 2021 Unicode®, Inc. +# Date: 2022-08-12, 20:24:39 GMT +# © 2022 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. -# For terms of use, see http://www.unicode.org/terms_of_use.html +# For terms of use, see https://www.unicode.org/terms_of_use.html # # Emoji Keyboard/Display Test Data for UTS #51 -# Version: 14.0 +# Version: 15.0 # -# For documentation and usage, see http://www.unicode.org/reports/tr51 +# For documentation and usage, see https://www.unicode.org/reports/tr51 # # This file provides data for testing which emoji forms should be in keyboards and which should also be displayed/processed. # Format: code points; status # emoji name @@ -92,6 +92,7 @@ 1F62C ; fully-qualified # 😬 E1.0 grimacing face 1F62E 200D 1F4A8 ; fully-qualified # 😮‍💨 E13.1 face exhaling 1F925 ; fully-qualified # 🤥 E3.0 lying face +1FAE8 ; fully-qualified # 🫨 E15.0 shaking face # subgroup: face-sleepy 1F60C ; fully-qualified # 😌 E0.6 relieved face @@ -155,7 +156,7 @@ # subgroup: face-negative 1F624 ; fully-qualified # 😤 E0.6 face with steam from nose -1F621 ; fully-qualified # 😡 E0.6 pouting face +1F621 ; fully-qualified # 😡 E0.6 enraged face 1F620 ; fully-qualified # 😠 E0.6 angry face 1F92C ; fully-qualified # 🤬 E5.0 face with symbols on mouth 1F608 ; fully-qualified # 😈 E1.0 smiling face with horns @@ -190,8 +191,7 @@ 1F649 ; fully-qualified # 🙉 E0.6 hear-no-evil monkey 1F64A ; fully-qualified # 🙊 E0.6 speak-no-evil monkey -# subgroup: emotion -1F48B ; fully-qualified # 💋 E0.6 kiss mark +# subgroup: heart 1F48C ; fully-qualified # 💌 E0.6 love letter 1F498 ; fully-qualified # 💘 E0.6 heart with arrow 1F49D ; fully-qualified # 💝 E0.6 heart with ribbon @@ -210,14 +210,20 @@ 2764 200D 1FA79 ; unqualified # ❤‍🩹 E13.1 mending heart 2764 FE0F ; fully-qualified # ❤️ E0.6 red heart 2764 ; unqualified # ❤ E0.6 red heart +1FA77 ; fully-qualified # 🩷 E15.0 pink heart 1F9E1 ; fully-qualified # 🧡 E5.0 orange heart 1F49B ; fully-qualified # 💛 E0.6 yellow heart 1F49A ; fully-qualified # 💚 E0.6 green heart 1F499 ; fully-qualified # 💙 E0.6 blue heart +1FA75 ; fully-qualified # 🩵 E15.0 light blue heart 1F49C ; fully-qualified # 💜 E0.6 purple heart 1F90E ; fully-qualified # 🤎 E12.0 brown heart 1F5A4 ; fully-qualified # 🖤 E3.0 black heart +1FA76 ; fully-qualified # 🩶 E15.0 grey heart 1F90D ; fully-qualified # 🤍 E12.0 white heart + +# subgroup: emotion +1F48B ; fully-qualified # 💋 E0.6 kiss mark 1F4AF ; fully-qualified # 💯 E0.6 hundred points 1F4A2 ; fully-qualified # 💢 E0.6 anger symbol 1F4A5 ; fully-qualified # 💥 E0.6 collision @@ -226,21 +232,20 @@ 1F4A8 ; fully-qualified # 💨 E0.6 dashing away 1F573 FE0F ; fully-qualified # 🕳️ E0.7 hole 1F573 ; unqualified # 🕳 E0.7 hole -1F4A3 ; fully-qualified # 💣 E0.6 bomb 1F4AC ; fully-qualified # 💬 E0.6 speech balloon 1F441 FE0F 200D 1F5E8 FE0F ; fully-qualified # 👁️‍🗨️ E2.0 eye in speech bubble 1F441 200D 1F5E8 FE0F ; unqualified # 👁‍🗨️ E2.0 eye in speech bubble -1F441 FE0F 200D 1F5E8 ; unqualified # 👁️‍🗨 E2.0 eye in speech bubble +1F441 FE0F 200D 1F5E8 ; minimally-qualified # 👁️‍🗨 E2.0 eye in speech bubble 1F441 200D 1F5E8 ; unqualified # 👁‍🗨 E2.0 eye in speech bubble 1F5E8 FE0F ; fully-qualified # 🗨️ E2.0 left speech bubble 1F5E8 ; unqualified # 🗨 E2.0 left speech bubble 1F5EF FE0F ; fully-qualified # 🗯️ E0.7 right anger bubble 1F5EF ; unqualified # 🗯 E0.7 right anger bubble 1F4AD ; fully-qualified # 💭 E1.0 thought balloon -1F4A4 ; fully-qualified # 💤 E0.6 zzz +1F4A4 ; fully-qualified # 💤 E0.6 ZZZ -# Smileys & Emotion subtotal: 177 -# Smileys & Emotion subtotal: 177 w/o modifiers +# Smileys & Emotion subtotal: 180 +# Smileys & Emotion subtotal: 180 w/o modifiers # group: People & Body @@ -300,6 +305,18 @@ 1FAF4 1F3FD ; fully-qualified # 🫴🏽 E14.0 palm up hand: medium skin tone 1FAF4 1F3FE ; fully-qualified # 🫴🏾 E14.0 palm up hand: medium-dark skin tone 1FAF4 1F3FF ; fully-qualified # 🫴🏿 E14.0 palm up hand: dark skin tone +1FAF7 ; fully-qualified # 🫷 E15.0 leftwards pushing hand +1FAF7 1F3FB ; fully-qualified # 🫷🏻 E15.0 leftwards pushing hand: light skin tone +1FAF7 1F3FC ; fully-qualified # 🫷🏼 E15.0 leftwards pushing hand: medium-light skin tone +1FAF7 1F3FD ; fully-qualified # 🫷🏽 E15.0 leftwards pushing hand: medium skin tone +1FAF7 1F3FE ; fully-qualified # 🫷🏾 E15.0 leftwards pushing hand: medium-dark skin tone +1FAF7 1F3FF ; fully-qualified # 🫷🏿 E15.0 leftwards pushing hand: dark skin tone +1FAF8 ; fully-qualified # 🫸 E15.0 rightwards pushing hand +1FAF8 1F3FB ; fully-qualified # 🫸🏻 E15.0 rightwards pushing hand: light skin tone +1FAF8 1F3FC ; fully-qualified # 🫸🏼 E15.0 rightwards pushing hand: medium-light skin tone +1FAF8 1F3FD ; fully-qualified # 🫸🏽 E15.0 rightwards pushing hand: medium skin tone +1FAF8 1F3FE ; fully-qualified # 🫸🏾 E15.0 rightwards pushing hand: medium-dark skin tone +1FAF8 1F3FF ; fully-qualified # 🫸🏿 E15.0 rightwards pushing hand: dark skin tone # subgroup: hand-fingers-partial 1F44C ; fully-qualified # 👌 E0.6 OK hand @@ -473,11 +490,11 @@ 1F932 1F3FE ; fully-qualified # 🤲🏾 E5.0 palms up together: medium-dark skin tone 1F932 1F3FF ; fully-qualified # 🤲🏿 E5.0 palms up together: dark skin tone 1F91D ; fully-qualified # 🤝 E3.0 handshake -1F91D 1F3FB ; fully-qualified # 🤝🏻 E3.0 handshake: light skin tone -1F91D 1F3FC ; fully-qualified # 🤝🏼 E3.0 handshake: medium-light skin tone -1F91D 1F3FD ; fully-qualified # 🤝🏽 E3.0 handshake: medium skin tone -1F91D 1F3FE ; fully-qualified # 🤝🏾 E3.0 handshake: medium-dark skin tone -1F91D 1F3FF ; fully-qualified # 🤝🏿 E3.0 handshake: dark skin tone +1F91D 1F3FB ; fully-qualified # 🤝🏻 E14.0 handshake: light skin tone +1F91D 1F3FC ; fully-qualified # 🤝🏼 E14.0 handshake: medium-light skin tone +1F91D 1F3FD ; fully-qualified # 🤝🏽 E14.0 handshake: medium skin tone +1F91D 1F3FE ; fully-qualified # 🤝🏾 E14.0 handshake: medium-dark skin tone +1F91D 1F3FF ; fully-qualified # 🤝🏿 E14.0 handshake: dark skin tone 1FAF1 1F3FB 200D 1FAF2 1F3FC ; fully-qualified # 🫱🏻‍🫲🏼 E14.0 handshake: light skin tone, medium-light skin tone 1FAF1 1F3FB 200D 1FAF2 1F3FD ; fully-qualified # 🫱🏻‍🫲🏽 E14.0 handshake: light skin tone, medium skin tone 1FAF1 1F3FB 200D 1FAF2 1F3FE ; fully-qualified # 🫱🏻‍🫲🏾 E14.0 handshake: light skin tone, medium-dark skin tone @@ -1455,7 +1472,7 @@ 1F575 1F3FF ; fully-qualified # 🕵🏿 E2.0 detective: dark skin tone 1F575 FE0F 200D 2642 FE0F ; fully-qualified # 🕵️‍♂️ E4.0 man detective 1F575 200D 2642 FE0F ; unqualified # 🕵‍♂️ E4.0 man detective -1F575 FE0F 200D 2642 ; unqualified # 🕵️‍♂ E4.0 man detective +1F575 FE0F 200D 2642 ; minimally-qualified # 🕵️‍♂ E4.0 man detective 1F575 200D 2642 ; unqualified # 🕵‍♂ E4.0 man detective 1F575 1F3FB 200D 2642 FE0F ; fully-qualified # 🕵🏻‍♂️ E4.0 man detective: light skin tone 1F575 1F3FB 200D 2642 ; minimally-qualified # 🕵🏻‍♂ E4.0 man detective: light skin tone @@ -1469,7 +1486,7 @@ 1F575 1F3FF 200D 2642 ; minimally-qualified # 🕵🏿‍♂ E4.0 man detective: dark skin tone 1F575 FE0F 200D 2640 FE0F ; fully-qualified # 🕵️‍♀️ E4.0 woman detective 1F575 200D 2640 FE0F ; unqualified # 🕵‍♀️ E4.0 woman detective -1F575 FE0F 200D 2640 ; unqualified # 🕵️‍♀ E4.0 woman detective +1F575 FE0F 200D 2640 ; minimally-qualified # 🕵️‍♀ E4.0 woman detective 1F575 200D 2640 ; unqualified # 🕵‍♀ E4.0 woman detective 1F575 1F3FB 200D 2640 FE0F ; fully-qualified # 🕵🏻‍♀️ E4.0 woman detective: light skin tone 1F575 1F3FB 200D 2640 ; minimally-qualified # 🕵🏻‍♀ E4.0 woman detective: light skin tone @@ -2302,7 +2319,7 @@ 1F3CC 1F3FF ; fully-qualified # 🏌🏿 E4.0 person golfing: dark skin tone 1F3CC FE0F 200D 2642 FE0F ; fully-qualified # 🏌️‍♂️ E4.0 man golfing 1F3CC 200D 2642 FE0F ; unqualified # 🏌‍♂️ E4.0 man golfing -1F3CC FE0F 200D 2642 ; unqualified # 🏌️‍♂ E4.0 man golfing +1F3CC FE0F 200D 2642 ; minimally-qualified # 🏌️‍♂ E4.0 man golfing 1F3CC 200D 2642 ; unqualified # 🏌‍♂ E4.0 man golfing 1F3CC 1F3FB 200D 2642 FE0F ; fully-qualified # 🏌🏻‍♂️ E4.0 man golfing: light skin tone 1F3CC 1F3FB 200D 2642 ; minimally-qualified # 🏌🏻‍♂ E4.0 man golfing: light skin tone @@ -2316,7 +2333,7 @@ 1F3CC 1F3FF 200D 2642 ; minimally-qualified # 🏌🏿‍♂ E4.0 man golfing: dark skin tone 1F3CC FE0F 200D 2640 FE0F ; fully-qualified # 🏌️‍♀️ E4.0 woman golfing 1F3CC 200D 2640 FE0F ; unqualified # 🏌‍♀️ E4.0 woman golfing -1F3CC FE0F 200D 2640 ; unqualified # 🏌️‍♀ E4.0 woman golfing +1F3CC FE0F 200D 2640 ; minimally-qualified # 🏌️‍♀ E4.0 woman golfing 1F3CC 200D 2640 ; unqualified # 🏌‍♀ E4.0 woman golfing 1F3CC 1F3FB 200D 2640 FE0F ; fully-qualified # 🏌🏻‍♀️ E4.0 woman golfing: light skin tone 1F3CC 1F3FB 200D 2640 ; minimally-qualified # 🏌🏻‍♀ E4.0 woman golfing: light skin tone @@ -2427,7 +2444,7 @@ 26F9 1F3FF ; fully-qualified # ⛹🏿 E2.0 person bouncing ball: dark skin tone 26F9 FE0F 200D 2642 FE0F ; fully-qualified # ⛹️‍♂️ E4.0 man bouncing ball 26F9 200D 2642 FE0F ; unqualified # ⛹‍♂️ E4.0 man bouncing ball -26F9 FE0F 200D 2642 ; unqualified # ⛹️‍♂ E4.0 man bouncing ball +26F9 FE0F 200D 2642 ; minimally-qualified # ⛹️‍♂ E4.0 man bouncing ball 26F9 200D 2642 ; unqualified # ⛹‍♂ E4.0 man bouncing ball 26F9 1F3FB 200D 2642 FE0F ; fully-qualified # ⛹🏻‍♂️ E4.0 man bouncing ball: light skin tone 26F9 1F3FB 200D 2642 ; minimally-qualified # ⛹🏻‍♂ E4.0 man bouncing ball: light skin tone @@ -2441,7 +2458,7 @@ 26F9 1F3FF 200D 2642 ; minimally-qualified # ⛹🏿‍♂ E4.0 man bouncing ball: dark skin tone 26F9 FE0F 200D 2640 FE0F ; fully-qualified # ⛹️‍♀️ E4.0 woman bouncing ball 26F9 200D 2640 FE0F ; unqualified # ⛹‍♀️ E4.0 woman bouncing ball -26F9 FE0F 200D 2640 ; unqualified # ⛹️‍♀ E4.0 woman bouncing ball +26F9 FE0F 200D 2640 ; minimally-qualified # ⛹️‍♀ E4.0 woman bouncing ball 26F9 200D 2640 ; unqualified # ⛹‍♀ E4.0 woman bouncing ball 26F9 1F3FB 200D 2640 FE0F ; fully-qualified # ⛹🏻‍♀️ E4.0 woman bouncing ball: light skin tone 26F9 1F3FB 200D 2640 ; minimally-qualified # ⛹🏻‍♀ E4.0 woman bouncing ball: light skin tone @@ -2462,7 +2479,7 @@ 1F3CB 1F3FF ; fully-qualified # 🏋🏿 E2.0 person lifting weights: dark skin tone 1F3CB FE0F 200D 2642 FE0F ; fully-qualified # 🏋️‍♂️ E4.0 man lifting weights 1F3CB 200D 2642 FE0F ; unqualified # 🏋‍♂️ E4.0 man lifting weights -1F3CB FE0F 200D 2642 ; unqualified # 🏋️‍♂ E4.0 man lifting weights +1F3CB FE0F 200D 2642 ; minimally-qualified # 🏋️‍♂ E4.0 man lifting weights 1F3CB 200D 2642 ; unqualified # 🏋‍♂ E4.0 man lifting weights 1F3CB 1F3FB 200D 2642 FE0F ; fully-qualified # 🏋🏻‍♂️ E4.0 man lifting weights: light skin tone 1F3CB 1F3FB 200D 2642 ; minimally-qualified # 🏋🏻‍♂ E4.0 man lifting weights: light skin tone @@ -2476,7 +2493,7 @@ 1F3CB 1F3FF 200D 2642 ; minimally-qualified # 🏋🏿‍♂ E4.0 man lifting weights: dark skin tone 1F3CB FE0F 200D 2640 FE0F ; fully-qualified # 🏋️‍♀️ E4.0 woman lifting weights 1F3CB 200D 2640 FE0F ; unqualified # 🏋‍♀️ E4.0 woman lifting weights -1F3CB FE0F 200D 2640 ; unqualified # 🏋️‍♀ E4.0 woman lifting weights +1F3CB FE0F 200D 2640 ; minimally-qualified # 🏋️‍♀ E4.0 woman lifting weights 1F3CB 200D 2640 ; unqualified # 🏋‍♀ E4.0 woman lifting weights 1F3CB 1F3FB 200D 2640 FE0F ; fully-qualified # 🏋🏻‍♀️ E4.0 woman lifting weights: light skin tone 1F3CB 1F3FB 200D 2640 ; minimally-qualified # 🏋🏻‍♀ E4.0 woman lifting weights: light skin tone @@ -3262,8 +3279,8 @@ 1FAC2 ; fully-qualified # 🫂 E13.0 people hugging 1F463 ; fully-qualified # 👣 E0.6 footprints -# People & Body subtotal: 2986 -# People & Body subtotal: 506 w/o modifiers +# People & Body subtotal: 2998 +# People & Body subtotal: 508 w/o modifiers # group: Component @@ -3306,6 +3323,8 @@ 1F405 ; fully-qualified # 🐅 E1.0 tiger 1F406 ; fully-qualified # 🐆 E1.0 leopard 1F434 ; fully-qualified # 🐴 E0.6 horse face +1FACE ; fully-qualified # 🫎 E15.0 moose +1FACF ; fully-qualified # 🫏 E15.0 donkey 1F40E ; fully-qualified # 🐎 E0.6 horse 1F984 ; fully-qualified # 🦄 E1.0 unicorn 1F993 ; fully-qualified # 🦓 E5.0 zebra @@ -3373,6 +3392,9 @@ 1F9A9 ; fully-qualified # 🦩 E12.0 flamingo 1F99A ; fully-qualified # 🦚 E11.0 peacock 1F99C ; fully-qualified # 🦜 E11.0 parrot +1FABD ; fully-qualified # 🪽 E15.0 wing +1F426 200D 2B1B ; fully-qualified # 🐦‍⬛ E15.0 black bird +1FABF ; fully-qualified # 🪿 E15.0 goose # subgroup: animal-amphibian 1F438 ; fully-qualified # 🐸 E0.6 frog @@ -3399,6 +3421,7 @@ 1F419 ; fully-qualified # 🐙 E0.6 octopus 1F41A ; fully-qualified # 🐚 E0.6 spiral shell 1FAB8 ; fully-qualified # 🪸 E14.0 coral +1FABC ; fully-qualified # 🪼 E15.0 jellyfish # subgroup: animal-bug 1F40C ; fully-qualified # 🐌 E0.6 snail @@ -3433,6 +3456,7 @@ 1F33B ; fully-qualified # 🌻 E0.6 sunflower 1F33C ; fully-qualified # 🌼 E0.6 blossom 1F337 ; fully-qualified # 🌷 E0.6 tulip +1FABB ; fully-qualified # 🪻 E15.0 hyacinth # subgroup: plant-other 1F331 ; fully-qualified # 🌱 E0.6 seedling @@ -3451,9 +3475,10 @@ 1F343 ; fully-qualified # 🍃 E0.6 leaf fluttering in wind 1FAB9 ; fully-qualified # 🪹 E14.0 empty nest 1FABA ; fully-qualified # 🪺 E14.0 nest with eggs +1F344 ; fully-qualified # 🍄 E0.6 mushroom -# Animals & Nature subtotal: 151 -# Animals & Nature subtotal: 151 w/o modifiers +# Animals & Nature subtotal: 159 +# Animals & Nature subtotal: 159 w/o modifiers # group: Food & Drink @@ -3492,10 +3517,11 @@ 1F966 ; fully-qualified # 🥦 E5.0 broccoli 1F9C4 ; fully-qualified # 🧄 E12.0 garlic 1F9C5 ; fully-qualified # 🧅 E12.0 onion -1F344 ; fully-qualified # 🍄 E0.6 mushroom 1F95C ; fully-qualified # 🥜 E3.0 peanuts 1FAD8 ; fully-qualified # 🫘 E14.0 beans 1F330 ; fully-qualified # 🌰 E0.6 chestnut +1FADA ; fully-qualified # 🫚 E15.0 ginger root +1FADB ; fully-qualified # 🫛 E15.0 pea pod # subgroup: food-prepared 1F35E ; fully-qualified # 🍞 E0.6 bread @@ -3607,8 +3633,8 @@ 1FAD9 ; fully-qualified # 🫙 E14.0 jar 1F3FA ; fully-qualified # 🏺 E1.0 amphora -# Food & Drink subtotal: 134 -# Food & Drink subtotal: 134 w/o modifiers +# Food & Drink subtotal: 135 +# Food & Drink subtotal: 135 w/o modifiers # group: Travel & Places @@ -3974,11 +4000,10 @@ 1F3AF ; fully-qualified # 🎯 E0.6 bullseye 1FA80 ; fully-qualified # 🪀 E12.0 yo-yo 1FA81 ; fully-qualified # 🪁 E12.0 kite +1F52B ; fully-qualified # 🔫 E0.6 water pistol 1F3B1 ; fully-qualified # 🎱 E0.6 pool 8 ball 1F52E ; fully-qualified # 🔮 E0.6 crystal ball 1FA84 ; fully-qualified # 🪄 E13.0 magic wand -1F9FF ; fully-qualified # 🧿 E11.0 nazar amulet -1FAAC ; fully-qualified # 🪬 E14.0 hamsa 1F3AE ; fully-qualified # 🎮 E0.6 video game 1F579 FE0F ; fully-qualified # 🕹️ E0.7 joystick 1F579 ; unqualified # 🕹 E0.7 joystick @@ -4013,8 +4038,8 @@ 1F9F6 ; fully-qualified # 🧶 E11.0 yarn 1FAA2 ; fully-qualified # 🪢 E13.0 knot -# Activities subtotal: 97 -# Activities subtotal: 97 w/o modifiers +# Activities subtotal: 96 +# Activities subtotal: 96 w/o modifiers # group: Objects @@ -4040,6 +4065,7 @@ 1FA73 ; fully-qualified # 🩳 E12.0 shorts 1F459 ; fully-qualified # 👙 E0.6 bikini 1F45A ; fully-qualified # 👚 E0.6 woman’s clothes +1FAAD ; fully-qualified # 🪭 E15.0 folding hand fan 1F45B ; fully-qualified # 👛 E0.6 purse 1F45C ; fully-qualified # 👜 E0.6 handbag 1F45D ; fully-qualified # 👝 E0.6 clutch bag @@ -4055,6 +4081,7 @@ 1F461 ; fully-qualified # 👡 E0.6 woman’s sandal 1FA70 ; fully-qualified # 🩰 E12.0 ballet shoes 1F462 ; fully-qualified # 👢 E0.6 woman’s boot +1FAAE ; fully-qualified # 🪮 E15.0 hair pick 1F451 ; fully-qualified # 👑 E0.6 crown 1F452 ; fully-qualified # 👒 E0.6 woman’s hat 1F3A9 ; fully-qualified # 🎩 E0.6 top hat @@ -4103,6 +4130,8 @@ 1FA95 ; fully-qualified # 🪕 E12.0 banjo 1F941 ; fully-qualified # 🥁 E3.0 drum 1FA98 ; fully-qualified # 🪘 E13.0 long drum +1FA87 ; fully-qualified # 🪇 E15.0 maracas +1FA88 ; fully-qualified # 🪈 E15.0 flute # subgroup: phone 1F4F1 ; fully-qualified # 📱 E0.6 mobile phone @@ -4275,7 +4304,7 @@ 1F5E1 ; unqualified # 🗡 E0.7 dagger 2694 FE0F ; fully-qualified # ⚔️ E1.0 crossed swords 2694 ; unqualified # ⚔ E1.0 crossed swords -1F52B ; fully-qualified # 🔫 E0.6 water pistol +1F4A3 ; fully-qualified # 💣 E0.6 bomb 1FA83 ; fully-qualified # 🪃 E13.0 boomerang 1F3F9 ; fully-qualified # 🏹 E1.0 bow and arrow 1F6E1 FE0F ; fully-qualified # 🛡️ E0.7 shield @@ -4354,12 +4383,14 @@ 1FAA6 ; fully-qualified # 🪦 E13.0 headstone 26B1 FE0F ; fully-qualified # ⚱️ E1.0 funeral urn 26B1 ; unqualified # ⚱ E1.0 funeral urn +1F9FF ; fully-qualified # 🧿 E11.0 nazar amulet +1FAAC ; fully-qualified # 🪬 E14.0 hamsa 1F5FF ; fully-qualified # 🗿 E0.6 moai 1FAA7 ; fully-qualified # 🪧 E13.0 placard 1FAAA ; fully-qualified # 🪪 E14.0 identification card -# Objects subtotal: 304 -# Objects subtotal: 304 w/o modifiers +# Objects subtotal: 310 +# Objects subtotal: 310 w/o modifiers # group: Symbols @@ -4455,6 +4486,7 @@ 262E ; unqualified # ☮ E1.0 peace symbol 1F54E ; fully-qualified # 🕎 E1.0 menorah 1F52F ; fully-qualified # 🔯 E0.6 dotted six-pointed star +1FAAF ; fully-qualified # 🪯 E15.0 khanda # subgroup: zodiac 2648 ; fully-qualified # ♈ E0.6 Aries @@ -4503,6 +4535,7 @@ 1F505 ; fully-qualified # 🔅 E1.0 dim button 1F506 ; fully-qualified # 🔆 E1.0 bright button 1F4F6 ; fully-qualified # 📶 E0.6 antenna bars +1F6DC ; fully-qualified # 🛜 E15.0 wireless 1F4F3 ; fully-qualified # 📳 E0.6 vibration mode 1F4F4 ; fully-qualified # 📴 E0.6 mobile phone off @@ -4693,8 +4726,8 @@ 1F533 ; fully-qualified # 🔳 E0.6 white square button 1F532 ; fully-qualified # 🔲 E0.6 black square button -# Symbols subtotal: 302 -# Symbols subtotal: 302 w/o modifiers +# Symbols subtotal: 304 +# Symbols subtotal: 304 w/o modifiers # group: Flags @@ -4709,7 +4742,7 @@ 1F3F3 200D 1F308 ; unqualified # 🏳‍🌈 E4.0 rainbow flag 1F3F3 FE0F 200D 26A7 FE0F ; fully-qualified # 🏳️‍⚧️ E13.0 transgender flag 1F3F3 200D 26A7 FE0F ; unqualified # 🏳‍⚧️ E13.0 transgender flag -1F3F3 FE0F 200D 26A7 ; unqualified # 🏳️‍⚧ E13.0 transgender flag +1F3F3 FE0F 200D 26A7 ; minimally-qualified # 🏳️‍⚧ E13.0 transgender flag 1F3F3 200D 26A7 ; unqualified # 🏳‍⚧ E13.0 transgender flag 1F3F4 200D 2620 FE0F ; fully-qualified # 🏴‍☠️ E11.0 pirate flag 1F3F4 200D 2620 ; minimally-qualified # 🏴‍☠ E11.0 pirate flag @@ -4983,9 +5016,9 @@ # Flags subtotal: 275 w/o modifiers # Status Counts -# fully-qualified : 3624 -# minimally-qualified : 817 -# unqualified : 252 +# fully-qualified : 3655 +# minimally-qualified : 827 +# unqualified : 242 # component : 9 #EOF From cc0f32c25339c534bb462773cfc2df33b7536edc Mon Sep 17 00:00:00 2001 From: Sean King Date: Fri, 19 Aug 2022 22:54:56 -0600 Subject: [PATCH 092/123] Add glitch-lily as an installable frontend --- config/config.exs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config/config.exs b/config/config.exs index 0fc959807..666268a0a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -761,6 +761,14 @@ config :pleroma, :frontends, "https://gitlab.com/soapbox-pub/soapbox-fe/-/jobs/artifacts/${ref}/download?job=build-production", "ref" => "v1.0.0", "build_dir" => "static" + }, + "glitch-lily" => %{ + "name" => "glitch-lily", + "git" => "https://lily-is.land/infra/glitch-lily", + "build_url" => + "https://lily-is.land/infra/glitch-lily/-/jobs/artifacts/${ref}/download?job=build", + "ref" => "servant", + "build_dir" => "public" } } From 06678fb4ad42fcaecb99eccc2237c3b863a2b9a5 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 11 Jul 2022 14:58:38 -0400 Subject: [PATCH 093/123] Add function to calculate associated object id --- ...2322_add_associated_object_id_function.exs | 37 ++++++++++ test/pleroma/activity_test.exs | 74 +++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 priv/repo/migrations/20220711182322_add_associated_object_id_function.exs diff --git a/priv/repo/migrations/20220711182322_add_associated_object_id_function.exs b/priv/repo/migrations/20220711182322_add_associated_object_id_function.exs new file mode 100644 index 000000000..76348f31a --- /dev/null +++ b/priv/repo/migrations/20220711182322_add_associated_object_id_function.exs @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.AddAssociatedObjectIdFunction do + use Ecto.Migration + + def up do + statement = """ + CREATE OR REPLACE FUNCTION associated_object_id(data jsonb) RETURNS varchar AS $$ + DECLARE + object_data jsonb; + BEGIN + IF jsonb_typeof(data->'object') = 'array' THEN + object_data := data->'object'->0; + ELSE + object_data := data->'object'; + END IF; + + IF jsonb_typeof(object_data->'id') = 'string' THEN + RETURN object_data->>'id'; + ELSIF jsonb_typeof(object_data) = 'string' THEN + RETURN object_data#>>'{}'; + ELSE + RETURN NULL; + END IF; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """ + + execute(statement) + end + + def down do + execute("DROP FUNCTION IF EXISTS associated_object_id(data jsonb)") + end +end diff --git a/test/pleroma/activity_test.exs b/test/pleroma/activity_test.exs index b5bb4bafe..e38384c9c 100644 --- a/test/pleroma/activity_test.exs +++ b/test/pleroma/activity_test.exs @@ -278,4 +278,78 @@ defmodule Pleroma.ActivityTest do assert Repo.aggregate(Activity, :count, :id) == 2 end + + describe "associated_object_id() sql function" do + test "with json object" do + %{rows: [[object_id]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + """ + select associated_object_id('{"object": {"id":"foobar"}}'::jsonb); + """ + ) + + assert object_id == "foobar" + end + + test "with string object" do + %{rows: [[object_id]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + """ + select associated_object_id('{"object": "foobar"}'::jsonb); + """ + ) + + assert object_id == "foobar" + end + + test "with array object" do + %{rows: [[object_id]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + """ + select associated_object_id('{"object": ["foobar", {}]}'::jsonb); + """ + ) + + assert object_id == "foobar" + end + + test "invalid" do + %{rows: [[object_id]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + """ + select associated_object_id('{"object": {}}'::jsonb); + """ + ) + + assert is_nil(object_id) + end + + test "invalid object id" do + %{rows: [[object_id]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + """ + select associated_object_id('{"object": {"id": 123}}'::jsonb); + """ + ) + + assert is_nil(object_id) + end + + test "no object field" do + %{rows: [[object_id]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + """ + select associated_object_id('{}'::jsonb); + """ + ) + + assert is_nil(object_id) + end + end end From 3885ee182a572a10b326ae553703ee0d38f3b66d Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 11 Jul 2022 15:49:58 -0400 Subject: [PATCH 094/123] Switch to associated_object_id index --- lib/mix/tasks/pleroma/database.ex | 3 +- lib/pleroma/activity.ex | 5 +-- lib/pleroma/activity/queries.ex | 6 +-- .../migrators/hashtags_table_migrator.ex | 2 +- lib/pleroma/notification.ex | 9 ++--- lib/pleroma/object.ex | 3 +- lib/pleroma/web/activity_pub/activity_pub.ex | 3 +- ...0_switch_to_associated_object_id_index.exs | 39 +++++++++++++++++++ 8 files changed, 50 insertions(+), 20 deletions(-) create mode 100644 priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 6b8f0ef68..ed560c177 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -154,9 +154,8 @@ defmodule Mix.Tasks.Pleroma.Database do |> join(:inner, [a], o in Object, on: fragment( - "(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')", + "(?->>'id') = associated_object_id((?))", o.data, - a.data, a.data ) ) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 12c1a3b2e..ebfd4ed45 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -53,7 +53,7 @@ defmodule Pleroma.Activity do # # ``` # |> join(:inner, [activity], o in Object, - # on: fragment("(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')", + # on: fragment("(?->>'id') = associated_object_id((?))", # o.data, activity.data, activity.data)) # |> preload([activity, object], [object: object]) # ``` @@ -69,9 +69,8 @@ defmodule Pleroma.Activity do join(query, join_type, [activity], o in Object, on: fragment( - "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", + "(?->>'id') = associated_object_id(?)", o.data, - activity.data, activity.data ), as: :object diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex index a898b2ea7..81c44ac05 100644 --- a/lib/pleroma/activity/queries.ex +++ b/lib/pleroma/activity/queries.ex @@ -52,8 +52,7 @@ defmodule Pleroma.Activity.Queries do activity in query, where: fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ANY(?)", - activity.data, + "associated_object_id((?)) = ANY(?)", activity.data, ^object_ids ) @@ -64,8 +63,7 @@ defmodule Pleroma.Activity.Queries do from(activity in query, where: fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ?", - activity.data, + "associated_object_id((?)) = ?", activity.data, ^object_id ) diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index fa1190b7d..dca4bfa6f 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -183,7 +183,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do DELETE FROM hashtags_objects WHERE object_id IN (SELECT DISTINCT objects.id FROM objects JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities - ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = + ON associated_object_id(activities) = (objects.data->>'id') AND activities.data->>'type' = 'Create' WHERE activities.id IS NULL); diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 52fd2656b..76d2d5ece 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -117,9 +117,8 @@ defmodule Pleroma.Notification do |> join(:left, [n, a], object in Object, on: fragment( - "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", + "(?->>'id') = associated_object_id(?)", object.data, - a.data, a.data ) ) @@ -193,13 +192,11 @@ defmodule Pleroma.Notification do |> join(:left, [n, a], mutated_activity in Pleroma.Activity, on: fragment( - "COALESCE((?->'object')->>'id', ?->>'object')", - a.data, + "associated_object_id(?)", a.data ) == fragment( - "COALESCE((?->'object')->>'id', ?->>'object')", - mutated_activity.data, + "associated_object_id(?)", mutated_activity.data ) and fragment("(?->>'type' = 'Like' or ?->>'type' = 'Announce')", a.data, a.data) and diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index fe264b5e0..e7d0d52b0 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -40,8 +40,7 @@ defmodule Pleroma.Object do join(query, join_type, [{object, object_position}], a in Activity, on: fragment( - "COALESCE(?->'object'->>'id', ?->>'object') = (? ->> 'id') AND (?->>'type' = ?) ", - a.data, + "associated_object_id(?) = (? ->> 'id') AND (?->>'type' = ?) ", a.data, object.data, a.data, diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index bded254c6..07b0a92a4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1150,8 +1150,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do [activity, object: o] in query, where: fragment( - "(?)->>'type' = 'Create' and coalesce((?)->'object'->>'id', (?)->>'object') = any (?)", - activity.data, + "(?)->>'type' = 'Create' and associated_object_id((?)) = any (?)", activity.data, activity.data, ^ids diff --git a/priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs b/priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs new file mode 100644 index 000000000..c0b89731b --- /dev/null +++ b/priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs @@ -0,0 +1,39 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.SwitchToAssociatedObjectIdIndex do + use Ecto.Migration + @disable_ddl_transaction true + @disable_migration_lock true + + def up do + drop_if_exists( + index(:activities, ["(coalesce(data->'object'->>'id', data->>'object'))"], + name: :activities_create_objects_index + ) + ) + + create( + index(:activities, ["associated_object_id(data)"], + name: :activities_create_objects_index, + concurrently: true + ) + ) + end + + def down do + drop_if_exists( + index(:activities, ["associated_object_id(data)"], + name: :activities_create_objects_index + ) + ) + + create( + index(:activities, ["(coalesce(data->'object'->>'id', data->>'object'))"], + name: :activities_create_objects_index, + concurrently: true + ) + ) + end +end From 4e7ed563c050e3781990e6c62ea5996d61b63d37 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 11 Jul 2022 16:24:38 -0400 Subject: [PATCH 095/123] Lint --- .../20220711192750_switch_to_associated_object_id_index.exs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs b/priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs index c0b89731b..75c1cd40b 100644 --- a/priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs +++ b/priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs @@ -24,9 +24,7 @@ defmodule Pleroma.Repo.Migrations.SwitchToAssociatedObjectIdIndex do def down do drop_if_exists( - index(:activities, ["associated_object_id(data)"], - name: :activities_create_objects_index - ) + index(:activities, ["associated_object_id(data)"], name: :activities_create_objects_index) ) create( From f047088a937ddf95d5fd7f84ad69fd97decbffc0 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 20 Aug 2022 21:06:12 -0400 Subject: [PATCH 096/123] Update thread visibility function --- ..._visibility_to_use_new_object_id_index.exs | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 priv/repo/migrations/20220821004840_change_thread_visibility_to_use_new_object_id_index.exs diff --git a/priv/repo/migrations/20220821004840_change_thread_visibility_to_use_new_object_id_index.exs b/priv/repo/migrations/20220821004840_change_thread_visibility_to_use_new_object_id_index.exs new file mode 100644 index 000000000..bb56843cb --- /dev/null +++ b/priv/repo/migrations/20220821004840_change_thread_visibility_to_use_new_object_id_index.exs @@ -0,0 +1,156 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.ChangeThreadVisibilityToUseNewObjectIdIndex do + use Ecto.Migration + + def up do + execute(update_thread_visibility()) + end + + def down do + execute(restore_thread_visibility()) + end + + def update_thread_visibility do + """ + CREATE OR REPLACE FUNCTION thread_visibility(actor varchar, activity_id varchar, local_public varchar default '') RETURNS boolean AS $$ + DECLARE + public varchar := 'https://www.w3.org/ns/activitystreams#Public'; + child objects%ROWTYPE; + activity activities%ROWTYPE; + author_fa varchar; + valid_recipients varchar[]; + actor_user_following varchar[]; + BEGIN + --- Fetch actor following + SELECT array_agg(following.follower_address) INTO actor_user_following FROM following_relationships + JOIN users ON users.id = following_relationships.follower_id + JOIN users AS following ON following.id = following_relationships.following_id + WHERE users.ap_id = actor; + + --- Fetch our initial activity. + SELECT * INTO activity FROM activities WHERE activities.data->>'id' = activity_id; + + LOOP + --- Ensure that we have an activity before continuing. + --- If we don't, the thread is not satisfiable. + IF activity IS NULL THEN + RETURN false; + END IF; + + --- We only care about Create activities. + IF activity.data->>'type' != 'Create' THEN + RETURN true; + END IF; + + --- Normalize the child object into child. + SELECT * INTO child FROM objects + INNER JOIN activities ON associated_object_id(activities.data) = objects.data->>'id' + WHERE associated_object_id(activity.data) = objects.data->>'id'; + + --- Fetch the author's AS2 following collection. + SELECT COALESCE(users.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; + + --- Prepare valid recipients array. + valid_recipients := ARRAY[actor, public]; + --- If we specified local public, add it. + IF local_public <> '' THEN + valid_recipients := valid_recipients || local_public; + END IF; + IF ARRAY[author_fa] && actor_user_following THEN + valid_recipients := valid_recipients || author_fa; + END IF; + + --- Check visibility. + IF NOT valid_recipients && activity.recipients THEN + --- activity not visible, break out of the loop + RETURN false; + END IF; + + --- If there's a parent, load it and do this all over again. + IF (child.data->'inReplyTo' IS NOT NULL) AND (child.data->'inReplyTo' != 'null'::jsonb) THEN + SELECT * INTO activity FROM activities + INNER JOIN objects ON associated_object_id(activities.data) = objects.data->>'id' + WHERE child.data->>'inReplyTo' = objects.data->>'id'; + ELSE + RETURN true; + END IF; + END LOOP; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """ + end + + # priv/repo/migrations/20220509180452_change_thread_visibility_to_be_local_only_aware.exs + def restore_thread_visibility do + """ + CREATE OR REPLACE FUNCTION thread_visibility(actor varchar, activity_id varchar, local_public varchar default '') RETURNS boolean AS $$ + DECLARE + public varchar := 'https://www.w3.org/ns/activitystreams#Public'; + child objects%ROWTYPE; + activity activities%ROWTYPE; + author_fa varchar; + valid_recipients varchar[]; + actor_user_following varchar[]; + BEGIN + --- Fetch actor following + SELECT array_agg(following.follower_address) INTO actor_user_following FROM following_relationships + JOIN users ON users.id = following_relationships.follower_id + JOIN users AS following ON following.id = following_relationships.following_id + WHERE users.ap_id = actor; + + --- Fetch our initial activity. + SELECT * INTO activity FROM activities WHERE activities.data->>'id' = activity_id; + + LOOP + --- Ensure that we have an activity before continuing. + --- If we don't, the thread is not satisfiable. + IF activity IS NULL THEN + RETURN false; + END IF; + + --- We only care about Create activities. + IF activity.data->>'type' != 'Create' THEN + RETURN true; + END IF; + + --- Normalize the child object into child. + SELECT * INTO child FROM objects + INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' + WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id'; + + --- Fetch the author's AS2 following collection. + SELECT COALESCE(users.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; + + --- Prepare valid recipients array. + valid_recipients := ARRAY[actor, public]; + --- If we specified local public, add it. + IF local_public <> '' THEN + valid_recipients := valid_recipients || local_public; + END IF; + IF ARRAY[author_fa] && actor_user_following THEN + valid_recipients := valid_recipients || author_fa; + END IF; + + --- Check visibility. + IF NOT valid_recipients && activity.recipients THEN + --- activity not visible, break out of the loop + RETURN false; + END IF; + + --- If there's a parent, load it and do this all over again. + IF (child.data->'inReplyTo' IS NOT NULL) AND (child.data->'inReplyTo' != 'null'::jsonb) THEN + SELECT * INTO activity FROM activities + INNER JOIN objects ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' + WHERE child.data->>'inReplyTo' = objects.data->>'id'; + ELSE + RETURN true; + END IF; + END LOOP; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """ + end +end From 27016287862a93b1fb4a4bebda3199e32c46d962 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 28 Dec 2021 15:01:37 -0500 Subject: [PATCH 097/123] Add remote interaction ui for posts --- .../twitter_api/util/status_interact.html.eex | 13 ++++ .../controllers/util_controller.ex | 47 ++++++++++++++ .../web/twitter_api/util_controller_test.exs | 64 +++++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex diff --git a/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex b/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex new file mode 100644 index 000000000..bb3d0a0af --- /dev/null +++ b/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex @@ -0,0 +1,13 @@ +<%= if @error do %> +

Error: <%= @error %>

+<% else %> +

Interacting with <%= @nickname %>

+
+ <%= @status_id %> +
+ <%= form_for @conn, Routes.util_path(@conn, :remote_subscribe), [as: "status"], fn f -> %> + <%= hidden_input f, :status, value: @status_id %> + <%= text_input f, :profile, placeholder: "Your account ID, e.g. lain@quitter.se" %> + <%= submit "Interact" %> + <% end %> +<% end %> diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 5731c78a8..ee99aab3e 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do require Logger + alias Pleroma.Activity alias Pleroma.Config alias Pleroma.Emoji alias Pleroma.Healthcheck @@ -59,6 +60,27 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end + def remote_subscribe(conn, %{"status_id" => id, "profile" => _}) do + with %Activity{} = activity <- Activity.get_by_id(id), + %User{} = user <- User.get_cached_by_ap_id(activity.actor), + avatar = User.avatar_url(user) do + conn + |> render("status_interact.html", %{ + status_id: id, + nickname: user.nickname, + avatar: avatar, + error: false + }) + else + _e -> + render(conn, "status_interact.html", %{ + status_id: id, + avatar: nil, + error: "Could not find status" + }) + end + end + def remote_subscribe(conn, %{"user" => %{"nickname" => nick, "profile" => profile}}) do with {:ok, %{"subscribe_address" => template}} <- WebFinger.finger(profile), %User{ap_id: ap_id} <- User.get_cached_by_nickname(nick) do @@ -74,6 +96,31 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end + def remote_subscribe(conn, %{"status" => %{"status_id" => id, "profile" => profile}}) do + get_ap_id = fn activity -> + object = Pleroma.Object.normalize(activity, fetch: false) + + case object do + %{data: %{"id" => ap_id}} -> {:ok, ap_id} + _ -> {:no_ap_id, nil} + end + end + + with {:ok, %{"subscribe_address" => template}} <- WebFinger.finger(profile), + %Activity{} = activity <- Activity.get_by_id(id), + {:ok, ap_id} <- get_ap_id.(activity) do + conn + |> Phoenix.Controller.redirect(external: String.replace(template, "{uri}", ap_id)) + else + _e -> + render(conn, "status_interact.html", %{ + status_id: id, + avatar: nil, + error: "Something went wrong." + }) + end + end + def remote_interaction(%{body_params: %{ap_id: ap_id, profile: profile}} = conn, _params) do with {:ok, %{"subscribe_address" => template}} <- WebFinger.finger(profile) do conn diff --git a/test/pleroma/web/twitter_api/util_controller_test.exs b/test/pleroma/web/twitter_api/util_controller_test.exs index 5dc72b177..020a5e9a1 100644 --- a/test/pleroma/web/twitter_api/util_controller_test.exs +++ b/test/pleroma/web/twitter_api/util_controller_test.exs @@ -233,6 +233,70 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do end end + describe "POST /main/ostatus - remote_subscribe/2 - with statuses" do + setup do: clear_config([:instance, :federating], true) + + test "renders subscribe form", %{conn: conn} do + user = insert(:user) + status = insert(:note_activity, %{user: user}) + status_id = status.id + + assert is_binary(status_id) + + response = + conn + |> post("/main/ostatus", %{"status_id" => status_id, "profile" => ""}) + |> response(:ok) + + refute response =~ "Could not find status" + assert response =~ "Interacting with" + end + + test "renders subscribe form with error when status not found", %{conn: conn} do + response = + conn + |> post("/main/ostatus", %{"status_id" => "somerandomid", "profile" => ""}) + |> response(:ok) + + assert response =~ "Could not find status" + refute response =~ "Interacting with" + end + + test "it redirect to webfinger url", %{conn: conn} do + user = insert(:user) + status = insert(:note_activity, %{user: user}) + status_id = status.id + status_ap_id = status.data["object"] + + assert is_binary(status_id) + assert is_binary(status_ap_id) + + user2 = insert(:user, ap_id: "shp@social.heldscal.la") + + conn = + conn + |> post("/main/ostatus", %{ + "status" => %{"status_id" => status_id, "profile" => user2.ap_id} + }) + + assert redirected_to(conn) == + "https://social.heldscal.la/main/ostatussub?profile=#{status_ap_id}" + end + + test "it renders form with error when status not found", %{conn: conn} do + user2 = insert(:user, ap_id: "shp@social.heldscal.la") + + response = + conn + |> post("/main/ostatus", %{ + "status" => %{"status_id" => "somerandomid", "profile" => user2.ap_id} + }) + |> response(:ok) + + assert response =~ "Something went wrong." + end + end + test "it returns new captcha", %{conn: conn} do with_mock Pleroma.Captcha, new: fn -> "test_captcha" end do From a243a217a7006352542a22aca605e60fc80f9ff0 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 28 Dec 2021 16:12:00 -0500 Subject: [PATCH 098/123] Fix form item name in status_interact.html --- .../web/templates/twitter_api/util/status_interact.html.eex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex b/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex index bb3d0a0af..6354b409f 100644 --- a/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex +++ b/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex @@ -6,7 +6,7 @@ <%= @status_id %> <%= form_for @conn, Routes.util_path(@conn, :remote_subscribe), [as: "status"], fn f -> %> - <%= hidden_input f, :status, value: @status_id %> + <%= hidden_input f, :status_id, value: @status_id %> <%= text_input f, :profile, placeholder: "Your account ID, e.g. lain@quitter.se" %> <%= submit "Interact" %> <% end %> From 779457d9a4e6b3e5e8b7823119907c1eb24a3b87 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 28 Dec 2021 16:41:46 -0500 Subject: [PATCH 099/123] Add GET endpoints for remote subscription forms There are two reasons for adding a GET endpoint: 0: Barely displaying the form does not change anything on the server. 1: It makes frontend development easier as they can now use a link, instead of a form, to allow remote users to interact with local ones. --- .../operations/twitter_util_operation.ex | 10 ++++++ lib/pleroma/web/router.ex | 1 + .../controllers/util_controller.ex | 16 +++++++--- .../web/twitter_api/util_controller_test.exs | 32 +++++++++++++++++++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex index 1cc90990f..29df03e34 100644 --- a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -405,6 +405,16 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do } end + def show_subscribe_form_operation do + %Operation{ + tags: ["Accounts"], + summary: "Show remote subscribe form", + operationId: "UtilController.show_subscribe_form", + parameters: [], + responses: %{200 => Operation.response("Web Page", "test/html", %Schema{type: :string})} + } + end + defp delete_account_request do %Schema{ title: "AccountDeleteRequest", diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 842596e97..846ba8363 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -337,6 +337,7 @@ defmodule Pleroma.Web.Router do pipe_through(:pleroma_html) post("/main/ostatus", UtilController, :remote_subscribe) + get("/main/ostatus", UtilController, :show_subscribe_form) get("/ostatus_subscribe", RemoteFollowController, :follow) post("/ostatus_subscribe", RemoteFollowController, :do_follow) end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index ee99aab3e..049329c38 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -17,8 +17,8 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.Web.WebFinger - plug(Pleroma.Web.ApiSpec.CastAndValidate when action != :remote_subscribe) - plug(Pleroma.Web.Plugs.FederatingPlug when action == :remote_subscribe) + plug(Pleroma.Web.ApiSpec.CastAndValidate when action != :remote_subscribe and action != :show_subscribe_form) + plug(Pleroma.Web.Plugs.FederatingPlug when action == :remote_subscribe when action == :show_subscribe_form) plug( OAuthScopesPlug, @@ -45,7 +45,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.TwitterUtilOperation - def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do + def show_subscribe_form(conn, %{"nickname" => nick}) do with %User{} = user <- User.get_cached_by_nickname(nick), avatar = User.avatar_url(user) do conn @@ -60,7 +60,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end - def remote_subscribe(conn, %{"status_id" => id, "profile" => _}) do + def show_subscribe_form(conn, %{"status_id" => id}) do with %Activity{} = activity <- Activity.get_by_id(id), %User{} = user <- User.get_cached_by_ap_id(activity.actor), avatar = User.avatar_url(user) do @@ -81,6 +81,14 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end + def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do + show_subscribe_form(conn, %{"nickname" => nick}) + end + + def remote_subscribe(conn, %{"status_id" => id, "profile" => _}) do + show_subscribe_form(conn, %{"status_id" => id}) + end + def remote_subscribe(conn, %{"user" => %{"nickname" => nick, "profile" => profile}}) do with {:ok, %{"subscribe_address" => template}} <- WebFinger.finger(profile), %User{ap_id: ap_id} <- User.get_cached_by_nickname(nick) do diff --git a/test/pleroma/web/twitter_api/util_controller_test.exs b/test/pleroma/web/twitter_api/util_controller_test.exs index 020a5e9a1..a4da23635 100644 --- a/test/pleroma/web/twitter_api/util_controller_test.exs +++ b/test/pleroma/web/twitter_api/util_controller_test.exs @@ -297,6 +297,38 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do end end + describe "GET /main/ostatus - show_subscribe_form/2" do + setup do: clear_config([:instance, :federating], true) + + test "it works with users", %{conn: conn} do + user = insert(:user) + + response = + conn + |> get("/main/ostatus", %{"nickname" => user.nickname}) + |> response(:ok) + + refute response =~ "Could not find user" + assert response =~ "Remotely follow #{user.nickname}" + end + + test "it works with statuses", %{conn: conn} do + user = insert(:user) + status = insert(:note_activity, %{user: user}) + status_id = status.id + + assert is_binary(status_id) + + response = + conn + |> get("/main/ostatus", %{"status_id" => status_id}) + |> response(:ok) + + refute response =~ "Could not find status" + assert response =~ "Interacting with" + end + end + test "it returns new captcha", %{conn: conn} do with_mock Pleroma.Captcha, new: fn -> "test_captcha" end do From b7c75db0f7f2c048d45fc387dfcf00073cbf8d62 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 28 Dec 2021 16:58:08 -0500 Subject: [PATCH 100/123] Lint --- .../web/twitter_api/controllers/util_controller.ex | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 049329c38..24b419c31 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -17,8 +17,16 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.Web.WebFinger - plug(Pleroma.Web.ApiSpec.CastAndValidate when action != :remote_subscribe and action != :show_subscribe_form) - plug(Pleroma.Web.Plugs.FederatingPlug when action == :remote_subscribe when action == :show_subscribe_form) + plug( + Pleroma.Web.ApiSpec.CastAndValidate + when action != :remote_subscribe and action != :show_subscribe_form + ) + + plug( + Pleroma.Web.Plugs.FederatingPlug + when action == :remote_subscribe + when action == :show_subscribe_form + ) plug( OAuthScopesPlug, From 1218adacc52f1235aedb1bb102d2e9385507efa4 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 28 Dec 2021 19:37:56 -0500 Subject: [PATCH 101/123] Display status link in remote interaction form --- .../twitter_api/util/status_interact.html.eex | 5 +---- .../controllers/util_controller.ex | 22 ++++++++++--------- .../web/twitter_api/views/util_view.ex | 1 + 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex b/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex index 6354b409f..695c5d64b 100644 --- a/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex +++ b/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex @@ -1,10 +1,7 @@ <%= if @error do %>

Error: <%= @error %>

<% else %> -

Interacting with <%= @nickname %>

-
- <%= @status_id %> -
+

Interacting with <%= @nickname %>'s <%= link("status", to: @status_link) %>

<%= form_for @conn, Routes.util_path(@conn, :remote_subscribe), [as: "status"], fn f -> %> <%= hidden_input f, :status_id, value: @status_id %> <%= text_input f, :profile, placeholder: "Your account ID, e.g. lain@quitter.se" %> diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 24b419c31..2c3103185 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -70,10 +70,12 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do def show_subscribe_form(conn, %{"status_id" => id}) do with %Activity{} = activity <- Activity.get_by_id(id), + {:ok, ap_id} <- get_ap_id(activity), %User{} = user <- User.get_cached_by_ap_id(activity.actor), avatar = User.avatar_url(user) do conn |> render("status_interact.html", %{ + status_link: ap_id, status_id: id, nickname: user.nickname, avatar: avatar, @@ -113,18 +115,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end def remote_subscribe(conn, %{"status" => %{"status_id" => id, "profile" => profile}}) do - get_ap_id = fn activity -> - object = Pleroma.Object.normalize(activity, fetch: false) - - case object do - %{data: %{"id" => ap_id}} -> {:ok, ap_id} - _ -> {:no_ap_id, nil} - end - end - with {:ok, %{"subscribe_address" => template}} <- WebFinger.finger(profile), %Activity{} = activity <- Activity.get_by_id(id), - {:ok, ap_id} <- get_ap_id.(activity) do + {:ok, ap_id} <- get_ap_id(activity) do conn |> Phoenix.Controller.redirect(external: String.replace(template, "{uri}", ap_id)) else @@ -146,6 +139,15 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end + defp get_ap_id(activity) do + object = Pleroma.Object.normalize(activity, fetch: false) + + case object do + %{data: %{"id" => ap_id}} -> {:ok, ap_id} + _ -> {:no_ap_id, nil} + end + end + def frontend_configurations(conn, _params) do render(conn, "frontend_configurations.json") end diff --git a/lib/pleroma/web/twitter_api/views/util_view.ex b/lib/pleroma/web/twitter_api/views/util_view.ex index 69f243097..2365a396b 100644 --- a/lib/pleroma/web/twitter_api/views/util_view.ex +++ b/lib/pleroma/web/twitter_api/views/util_view.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilView do use Pleroma.Web, :view import Phoenix.HTML.Form + import Phoenix.HTML.Link alias Pleroma.Config alias Pleroma.Web.Endpoint alias Pleroma.Web.Gettext From ec0e912c52f9c44ef78dbb8971d39ab4ef53bf30 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 29 Dec 2021 00:29:00 -0500 Subject: [PATCH 102/123] Add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d0ef4e11..a979ff325 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Make backend-rendered pages translatable. This includes emails. Pages returned as a HTTP response are translated using the language specified in the `userLanguage` cookie, or the `Accept-Language` header. Emails are translated using the `language` field when registering. This language can be changed by `PATCH /api/v1/accounts/update_credentials` with the `language` field. - Uploadfilter `Pleroma.Upload.Filter.Exiftool.ReadDescription` returns description values to the FE so they can pre fill the image description field - Added move account API +- Enable remote users to interact with posts ### Fixed - Subscription(Bell) Notifications: Don't create from Pipeline Ingested replies From 4ec9eeb3f8f3502841cd136ea7afe9298b477120 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 25 Mar 2022 22:05:28 -0400 Subject: [PATCH 103/123] Make remote interaction page translatable --- .../twitter_api/util/status_interact.html.eex | 8 +++--- .../controllers/util_controller.ex | 28 ++++++++++++++++--- .../web/twitter_api/views/util_view.ex | 1 + 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex b/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex index 695c5d64b..d77174967 100644 --- a/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex +++ b/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex @@ -1,10 +1,10 @@ <%= if @error do %> -

Error: <%= @error %>

+

<%= Gettext.dpgettext("static_pages", "status interact error", "Error: %{error}", error: @error) %>

<% else %> -

Interacting with <%= @nickname %>'s <%= link("status", to: @status_link) %>

+

<%= raw Gettext.dpgettext("static_pages", "status interact header", "Interacting with %{nickname}'s %{status_link}", nickname: safe_to_string(html_escape(@nickname)), status_link: safe_to_string(link(Gettext.dpgettext("static_pages", "status interact header - status link text", "status"), to: @status_link))) %>

<%= form_for @conn, Routes.util_path(@conn, :remote_subscribe), [as: "status"], fn f -> %> <%= hidden_input f, :status_id, value: @status_id %> - <%= text_input f, :profile, placeholder: "Your account ID, e.g. lain@quitter.se" %> - <%= submit "Interact" %> + <%= text_input f, :profile, placeholder: Gettext.dpgettext("static_pages", "placeholder text for account id", "Your account ID, e.g. lain@quitter.se") %> + <%= submit Gettext.dpgettext("static_pages", "status interact authorization button", "Interact") %> <% end %> <% end %> diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 2c3103185..d5a24ae6c 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -63,7 +63,12 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do render(conn, "subscribe.html", %{ nickname: nick, avatar: nil, - error: "Could not find user" + error: + Pleroma.Web.Gettext.dpgettext( + "static_pages", + "remote follow error message - user not found", + "Could not find user" + ) }) end end @@ -86,7 +91,12 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do render(conn, "status_interact.html", %{ status_id: id, avatar: nil, - error: "Could not find status" + error: + Pleroma.Web.Gettext.dpgettext( + "static_pages", + "status interact error message - status not found", + "Could not find status" + ) }) end end @@ -109,7 +119,12 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do render(conn, "subscribe.html", %{ nickname: nick, avatar: nil, - error: "Something went wrong." + error: + Pleroma.Web.Gettext.dpgettext( + "static_pages", + "remote follow error message - unknown error", + "Something went wrong." + ) }) end end @@ -125,7 +140,12 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do render(conn, "status_interact.html", %{ status_id: id, avatar: nil, - error: "Something went wrong." + error: + Pleroma.Web.Gettext.dpgettext( + "static_pages", + "status interact error message - unknown error", + "Something went wrong." + ) }) end end diff --git a/lib/pleroma/web/twitter_api/views/util_view.ex b/lib/pleroma/web/twitter_api/views/util_view.ex index 2365a396b..31b7c0c0c 100644 --- a/lib/pleroma/web/twitter_api/views/util_view.ex +++ b/lib/pleroma/web/twitter_api/views/util_view.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilView do use Pleroma.Web, :view + import Phoenix.HTML import Phoenix.HTML.Form import Phoenix.HTML.Link alias Pleroma.Config From c59ee1f172628d37e1396e080876f0f3aebaf730 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 20 Aug 2022 21:19:31 -0400 Subject: [PATCH 104/123] Expose availability of GET /main/ostatus via instance --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 62931bd41..dc44295e5 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -98,7 +98,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do end, if Config.get([:instance, :profile_directory]) do "profile_directory" - end + end, + "pleroma:get:main/ostatus" ] |> Enum.filter(& &1) end From c59a0bd12fe0f9b935ad7cc2ddda6d562d21eb01 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 20 Aug 2022 21:52:20 -0400 Subject: [PATCH 105/123] Add margin to forms and make inputs fill whole width --- priv/static/instance/static.css | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/priv/static/instance/static.css b/priv/static/instance/static.css index 487e1ec27..48c74c125 100644 --- a/priv/static/instance/static.css +++ b/priv/static/instance/static.css @@ -51,6 +51,7 @@ body { overflow: hidden; margin: 35px auto; box-shadow: 0 1px 4px 0px rgba(0, 0, 0, 0.5); + padding: 0em 1em 0em 1em; } .container__content { @@ -86,7 +87,6 @@ form { } input { - box-sizing: content-box; padding: 10px; margin-top: 5px; margin-bottom: 10px; @@ -97,6 +97,8 @@ input { transition-duration: 0.35s; border-bottom: 2px solid #2a384a; font-size: 14px; + width: inherit; + box-sizing: border-box; } .scopes-input { From 439c1baf25b19723bcbaac78b10d00181074e3b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Tue, 23 Aug 2022 17:15:06 +0200 Subject: [PATCH 106/123] OAuthPlug: use user cache instead of joining As this plug is called on every request, this should reduce load on the database by not requiring to select on the users table every single time, and to instead use the by-ID user cache whenever possible. --- lib/pleroma/web/plugs/o_auth_plug.ex | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/plugs/o_auth_plug.ex b/lib/pleroma/web/plugs/o_auth_plug.ex index 0f74d626b..ba04ddb72 100644 --- a/lib/pleroma/web/plugs/o_auth_plug.ex +++ b/lib/pleroma/web/plugs/o_auth_plug.ex @@ -47,15 +47,17 @@ defmodule Pleroma.Web.Plugs.OAuthPlug do # @spec fetch_user_and_token(String.t()) :: {:ok, User.t(), Token.t()} | nil defp fetch_user_and_token(token) do - query = + token_query = from(t in Token, - where: t.token == ^token, - join: user in assoc(t, :user), - preload: [user: user] + where: t.token == ^token ) - with %Token{user: user} = token_record <- Repo.one(query) do + with %Token{user_id: user_id} = token_record <- Repo.one(token_query), + false <- is_nil(user_id), + %User{} = user <- User.get_cached_by_id(user_id) do {:ok, user, token_record} + else + _ -> nil end end From 47e3a72b6ecff3fcc9eedf0dc23bffef5f8c9060 Mon Sep 17 00:00:00 2001 From: Ilja <672-ilja@users.noreply.git.pleroma.social> Date: Wed, 24 Aug 2022 15:24:07 +0000 Subject: [PATCH 107/123] fix flaky test_user_relationship_test.exs:81 --- lib/pleroma/user_relationship.ex | 5 +++-- test/pleroma/user_relationship_test.exs | 10 ++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index 5b3e593d3..fbecf3129 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -91,8 +91,9 @@ defmodule Pleroma.UserRelationship do expires_at: expires_at }) |> Repo.insert( - on_conflict: {:replace_all_except, [:id]}, - conflict_target: [:source_id, :relationship_type, :target_id] + on_conflict: {:replace_all_except, [:id, :inserted_at]}, + conflict_target: [:source_id, :relationship_type, :target_id], + returning: true ) end diff --git a/test/pleroma/user_relationship_test.exs b/test/pleroma/user_relationship_test.exs index 2811aff4c..7d205a746 100644 --- a/test/pleroma/user_relationship_test.exs +++ b/test/pleroma/user_relationship_test.exs @@ -5,8 +5,9 @@ defmodule Pleroma.UserRelationshipTest do alias Pleroma.UserRelationship - use Pleroma.DataCase, async: true + use Pleroma.DataCase, async: false + import Mock import Pleroma.Factory describe "*_exists?/2" do @@ -79,7 +80,12 @@ defmodule Pleroma.UserRelationshipTest do end test "if record already exists, returns it", %{users: [user1, user2]} do - user_block = UserRelationship.create_block(user1, user2) + user_block = + with_mock NaiveDateTime, [:passthrough], utc_now: fn -> ~N[2017-03-17 17:09:58] end do + {:ok, %{inserted_at: ~N[2017-03-17 17:09:58]}} = + UserRelationship.create_block(user1, user2) + end + assert user_block == UserRelationship.create_block(user1, user2) end end From 5b2e3a303c5c7cde37e55a463c3ef79324c59bbe Mon Sep 17 00:00:00 2001 From: Ilja <672-ilja@users.noreply.git.pleroma.social> Date: Wed, 24 Aug 2022 15:24:57 +0000 Subject: [PATCH 108/123] fix flaky test filter_controller_test.exs:200 --- .../controllers/filter_controller_test.exs | 57 ++++++++----------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/test/pleroma/web/mastodon_api/controllers/filter_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/filter_controller_test.exs index ba4628fc5..faa35f199 100644 --- a/test/pleroma/web/mastodon_api/controllers/filter_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/filter_controller_test.exs @@ -3,9 +3,10 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do - use Pleroma.Web.ConnCase, async: true + use Pleroma.Web.ConnCase, async: false use Oban.Testing, repo: Pleroma.Repo + import Mock import Pleroma.Factory alias Pleroma.Filter @@ -53,24 +54,19 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do in_seconds = 600 response = - conn - |> put_req_header("content-type", "application/json") - |> post("/api/v1/filters", %{ - "phrase" => "knights", - context: ["home"], - expires_in: in_seconds - }) - |> json_response_and_validate_schema(200) + with_mock NaiveDateTime, [:passthrough], utc_now: fn -> ~N[2017-03-17 17:09:58] end do + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/filters", %{ + "phrase" => "knights", + context: ["home"], + expires_in: in_seconds + }) + |> json_response_and_validate_schema(200) + end assert response["irreversible"] == false - - expected_expiration = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(in_seconds) - - {:ok, actual_expiration} = NaiveDateTime.from_iso8601(response["expires_at"]) - - assert abs(NaiveDateTime.diff(expected_expiration, actual_expiration)) <= 5 + assert response["expires_at"] == "2017-03-17T17:19:58.000Z" filter = Filter.get(response["id"], user) @@ -177,28 +173,25 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do assert response["whole_word"] == true end - @tag :erratic test "with adding expires_at", %{conn: conn, user: user} do filter = insert(:filter, user: user) in_seconds = 600 response = - conn - |> put_req_header("content-type", "application/json") - |> put("/api/v1/filters/#{filter.filter_id}", %{ - phrase: "nii", - context: ["public"], - expires_in: in_seconds, - irreversible: true - }) - |> json_response_and_validate_schema(200) + with_mock NaiveDateTime, [:passthrough], utc_now: fn -> ~N[2017-03-17 17:09:58] end do + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/filters/#{filter.filter_id}", %{ + phrase: "nii", + context: ["public"], + expires_in: in_seconds, + irreversible: true + }) + |> json_response_and_validate_schema(200) + end assert response["irreversible"] == true - - assert response["expires_at"] == - NaiveDateTime.utc_now() - |> NaiveDateTime.add(in_seconds) - |> Pleroma.Web.CommonAPI.Utils.to_masto_date() + assert response["expires_at"] == "2017-03-17T17:19:58.000Z" filter = Filter.get(response["id"], user) From d67d19134400e79d3a773229bd55438684ffb7ed Mon Sep 17 00:00:00 2001 From: Sean King Date: Wed, 24 Aug 2022 23:39:02 -0600 Subject: [PATCH 109/123] Fix fedi-fe build URL --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 666268a0a..1653358a0 100644 --- a/config/config.exs +++ b/config/config.exs @@ -741,7 +741,7 @@ config :pleroma, :frontends, "name" => "fedi-fe", "git" => "https://git.pleroma.social/pleroma/fedi-fe", "build_url" => - "https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build", + "https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build_release", "ref" => "master", "custom-http-headers" => [ {"service-worker-allowed", "/"} From dc72a523c4ee4218086904105b39cedb4bd1f4dc Mon Sep 17 00:00:00 2001 From: Ilja <672-ilja@users.noreply.git.pleroma.social> Date: Thu, 25 Aug 2022 18:36:46 +0000 Subject: [PATCH 110/123] fix flaky participation_test.exs --- test/pleroma/conversation/participation_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pleroma/conversation/participation_test.exs b/test/pleroma/conversation/participation_test.exs index 6f71cc040..a84437677 100644 --- a/test/pleroma/conversation/participation_test.exs +++ b/test/pleroma/conversation/participation_test.exs @@ -122,11 +122,11 @@ defmodule Pleroma.Conversation.ParticipationTest do end test "it marks a participation as read" do - participation = insert(:participation, %{read: false}) + participation = insert(:participation, %{updated_at: ~N[2017-07-17 17:09:58], read: false}) {:ok, updated_participation} = Participation.mark_as_read(participation) assert updated_participation.read - assert updated_participation.updated_at == participation.updated_at + assert :gt = NaiveDateTime.compare(updated_participation.updated_at, participation.updated_at) end test "it marks a participation as unread" do From 3afa1903ee202cc0acb4170bc06c491c15875145 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 27 Aug 2022 17:51:41 -0400 Subject: [PATCH 111/123] Do not stream out Create of ChatMessage --- lib/pleroma/activity/ir/topics.ex | 8 ++++++++ test/pleroma/activity/ir/topics_test.exs | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex index 56c52e9d1..f058cc0c9 100644 --- a/lib/pleroma/activity/ir/topics.ex +++ b/lib/pleroma/activity/ir/topics.ex @@ -13,6 +13,14 @@ defmodule Pleroma.Activity.Ir.Topics do |> List.flatten() end + defp generate_topics(%{data: %{"type" => "ChatMessage"}}, %{data: %{"type" => "Delete"}}) do + ["user", "user:pleroma_chat"] + end + + defp generate_topics(%{data: %{"type" => "ChatMessage"}}, %{data: %{"type" => "Create"}}) do + [] + end + defp generate_topics(%{data: %{"type" => "Answer"}}, _) do [] end diff --git a/test/pleroma/activity/ir/topics_test.exs b/test/pleroma/activity/ir/topics_test.exs index 311f85dea..0abda60b0 100644 --- a/test/pleroma/activity/ir/topics_test.exs +++ b/test/pleroma/activity/ir/topics_test.exs @@ -13,6 +13,29 @@ defmodule Pleroma.Activity.Ir.TopicsTest do import Mock + describe "chat message" do + test "Create produces no topics" do + activity = %Activity{ + object: %Object{data: %{"type" => "ChatMessage"}}, + data: %{"type" => "Create"} + } + + assert [] == Topics.get_activity_topics(activity) + end + + test "Delete produces user and user:pleroma_chat" do + activity = %Activity{ + object: %Object{data: %{"type" => "ChatMessage"}}, + data: %{"type" => "Delete"} + } + + topics = Topics.get_activity_topics(activity) + assert [_, _] = topics + assert "user" in topics + assert "user:pleroma_chat" in topics + end + end + describe "poll answer" do test "produce no topics" do activity = %Activity{object: %Object{data: %{"type" => "Answer"}}} From f9b86c3c22c10d54a721cfe632f9c7455a8b2f9c Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 27 Aug 2022 19:34:56 -0400 Subject: [PATCH 112/123] Make local-only posts stream in local timeline --- lib/pleroma/activity/ir/topics.ex | 17 +++++++- test/pleroma/activity/ir/topics_test.exs | 53 ++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex index f058cc0c9..fa4350797 100644 --- a/lib/pleroma/activity/ir/topics.ex +++ b/lib/pleroma/activity/ir/topics.ex @@ -39,6 +39,10 @@ defmodule Pleroma.Activity.Ir.Topics do end |> item_creation_tags(object, activity) + "local" -> + ["public:local"] + |> item_creation_tags(object, activity) + "direct" -> ["direct"] @@ -71,7 +75,18 @@ defmodule Pleroma.Activity.Ir.Topics do defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: [] - defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"] + defp attachment_topics(_object, %{local: true} = activity) do + case Visibility.get_visibility(activity) do + "public" -> + ["public:media", "public:local:media"] + + "local" -> + ["public:local:media"] + + _ -> + [] + end + end defp attachment_topics(_object, %{actor: actor}) when is_binary(actor), do: ["public:media", "public:remote:media:" <> URI.parse(actor).host] diff --git a/test/pleroma/activity/ir/topics_test.exs b/test/pleroma/activity/ir/topics_test.exs index 0abda60b0..8317868a5 100644 --- a/test/pleroma/activity/ir/topics_test.exs +++ b/test/pleroma/activity/ir/topics_test.exs @@ -137,6 +137,36 @@ defmodule Pleroma.Activity.Ir.TopicsTest do end end + describe "local-public visibility create events" do + setup do + activity = %Activity{ + object: %Object{data: %{"attachment" => []}}, + data: %{"type" => "Create", "to" => [Pleroma.Web.ActivityPub.Utils.as_local_public()]} + } + + {:ok, activity: activity} + end + + test "doesn't produce public topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public") + end + + test "produces public:local topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:local") + end + + test "with no attachments doesn't produce public:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:media") + refute Enum.member?(topics, "public:local:media") + end + end + describe "public visibility create events with attachments" do setup do activity = %Activity{ @@ -175,6 +205,29 @@ defmodule Pleroma.Activity.Ir.TopicsTest do end end + describe "local-public visibility create events with attachments" do + setup do + activity = %Activity{ + object: %Object{data: %{"attachment" => ["foo"]}}, + data: %{"type" => "Create", "to" => [Pleroma.Web.ActivityPub.Utils.as_local_public()]} + } + + {:ok, activity: activity} + end + + test "do not produce public:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:media") + end + + test "produces public:local:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:local:media") + end + end + describe "non-public visibility" do test "produces direct topic" do activity = %Activity{object: %Object{data: %{"type" => "Note"}}, data: %{"to" => []}} From ffd379456bb9e4f125ce5e2480be4d2819b88147 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 31 Aug 2022 15:57:06 -0400 Subject: [PATCH 113/123] Do not stream out Announces to public timelines --- lib/pleroma/activity/ir/topics.ex | 6 +++++- test/pleroma/activity/ir/topics_test.exs | 27 ++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex index fa4350797..b9fcd6693 100644 --- a/lib/pleroma/activity/ir/topics.ex +++ b/lib/pleroma/activity/ir/topics.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Activity.Ir.Topics do ["user", "list"] ++ visibility_tags(object, activity) end - defp visibility_tags(object, activity) do + defp visibility_tags(object, %{data: %{"type" => "Create"}} = activity) do case Visibility.get_visibility(activity) do "public" -> if activity.local do @@ -51,6 +51,10 @@ defmodule Pleroma.Activity.Ir.Topics do end end + defp visibility_tags(_object, _activity) do + [] + end + defp item_creation_tags(tags, object, %{data: %{"type" => "Create"}} = activity) do tags ++ remote_topics(activity) ++ hashtags_to_topics(object) ++ attachment_topics(object, activity) diff --git a/test/pleroma/activity/ir/topics_test.exs b/test/pleroma/activity/ir/topics_test.exs index 8317868a5..d299fea63 100644 --- a/test/pleroma/activity/ir/topics_test.exs +++ b/test/pleroma/activity/ir/topics_test.exs @@ -58,7 +58,7 @@ defmodule Pleroma.Activity.Ir.TopicsTest do setup do activity = %Activity{ object: %Object{data: %{"type" => "Note"}}, - data: %{"to" => [Pleroma.Constants.as_public()]} + data: %{"to" => [Pleroma.Constants.as_public()], "type" => "Create"} } {:ok, activity: activity} @@ -137,6 +137,25 @@ defmodule Pleroma.Activity.Ir.TopicsTest do end end + describe "public visibility Announces" do + setup do + activity = %Activity{ + object: %Object{data: %{"attachment" => []}}, + data: %{"type" => "Announce", "to" => [Pleroma.Constants.as_public()]} + } + + {:ok, activity: activity} + end + + test "does not generate public topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + refute "public" in topics + refute "public:remote" in topics + refute "public:local" in topics + end + end + describe "local-public visibility create events" do setup do activity = %Activity{ @@ -230,7 +249,11 @@ defmodule Pleroma.Activity.Ir.TopicsTest do describe "non-public visibility" do test "produces direct topic" do - activity = %Activity{object: %Object{data: %{"type" => "Note"}}, data: %{"to" => []}} + activity = %Activity{ + object: %Object{data: %{"type" => "Note"}}, + data: %{"to" => [], "type" => "Create"} + } + topics = Topics.get_activity_topics(activity) assert Enum.member?(topics, "direct") From 20a0dd6516e453cdedb5a7a2b7356c529eeacf84 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 31 Aug 2022 22:14:54 -0400 Subject: [PATCH 114/123] Exclude Announce instead of restricting to Create in visibility_tags --- lib/pleroma/activity/ir/topics.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex index b9fcd6693..8249cbe27 100644 --- a/lib/pleroma/activity/ir/topics.ex +++ b/lib/pleroma/activity/ir/topics.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Activity.Ir.Topics do ["user", "list"] ++ visibility_tags(object, activity) end - defp visibility_tags(object, %{data: %{"type" => "Create"}} = activity) do + defp visibility_tags(object, %{data: %{"type" => type}} = activity) when type != "Announce" do case Visibility.get_visibility(activity) do "public" -> if activity.local do From c32e28e1b09021647d6229236a689852410dffe5 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Thu, 1 Sep 2022 07:33:58 -0400 Subject: [PATCH 115/123] Fix SideEffectsTest --- test/pleroma/web/activity_pub/side_effects_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 64c4a8c14..2cbb64e8f 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -545,7 +545,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do {:ok, announce, _} = SideEffects.handle(announce) assert called( - Pleroma.Web.Streamer.stream(["user", "list", "public", "public:local"], announce) + Pleroma.Web.Streamer.stream(["user", "list"], announce) ) assert called(Pleroma.Web.Push.send(:_)) From e94937847669ffe318518ddc39257d726da0affc Mon Sep 17 00:00:00 2001 From: Fristi Date: Sat, 13 Aug 2022 10:24:16 +0000 Subject: [PATCH 116/123] Added translation using Weblate (Dutch) --- priv/gettext/nl/LC_MESSAGES/static_pages.po | 525 ++++++++++++++++++++ 1 file changed, 525 insertions(+) create mode 100644 priv/gettext/nl/LC_MESSAGES/static_pages.po diff --git a/priv/gettext/nl/LC_MESSAGES/static_pages.po b/priv/gettext/nl/LC_MESSAGES/static_pages.po new file mode 100644 index 000000000..91e2fa3c8 --- /dev/null +++ b/priv/gettext/nl/LC_MESSAGES/static_pages.po @@ -0,0 +1,525 @@ +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2022-08-13 13:24+0300\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: Automatically generated\n" +"Language-Team: none\n" +"Language: nl\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"X-Generator: Translate Toolkit 3.7.2\n" + +## This file is a PO Template file. +## +## "msgid"s here are often extracted from source code. +## Add new translations manually only if they're dynamic +## translations that can't be statically extracted. +## +## Run "mix gettext.extract" to bring this file up to +## date. Leave "msgstr"s empty as changing them here as no +## effect: edit them in PO (.po) files instead. + +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex:9 +#, elixir-autogen, elixir-format +msgctxt "remote follow authorization button" +msgid "Authorize" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex:2 +#, elixir-autogen, elixir-format +msgctxt "remote follow error" +msgid "Error fetching user" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex:4 +#, elixir-autogen, elixir-format +msgctxt "remote follow header" +msgid "Remote follow" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex:8 +#, elixir-autogen, elixir-format +msgctxt "placeholder text for auth code entry" +msgid "Authentication code" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:10 +#, elixir-autogen, elixir-format +msgctxt "placeholder text for password entry" +msgid "Password" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:8 +#, elixir-autogen, elixir-format +msgctxt "placeholder text for username entry" +msgid "Username" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:13 +#, elixir-autogen, elixir-format +msgctxt "remote follow authorization button for login" +msgid "Authorize" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex:12 +#, elixir-autogen, elixir-format +msgctxt "remote follow authorization button for mfa" +msgid "Authorize" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/remote_follow/followed.html.eex:2 +#, elixir-autogen, elixir-format +msgctxt "remote follow error" +msgid "Error following account" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:4 +#, elixir-autogen, elixir-format +msgctxt "remote follow header, need login" +msgid "Log in to follow" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex:4 +#, elixir-autogen, elixir-format +msgctxt "remote follow mfa header" +msgid "Two-factor authentication" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/remote_follow/followed.html.eex:4 +#, elixir-autogen, elixir-format +msgctxt "remote follow success" +msgid "Account followed!" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:7 +#, elixir-autogen, elixir-format +msgctxt "placeholder text for account id" +msgid "Your account ID, e.g. lain@quitter.se" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:8 +#, elixir-autogen, elixir-format +msgctxt "remote follow authorization button for following with a remote account" +msgid "Follow" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:2 +#, elixir-autogen, elixir-format +msgctxt "remote follow error" +msgid "Error: %{error}" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:4 +#, elixir-autogen, elixir-format +msgctxt "remote follow header" +msgid "Remotely follow %{nickname}" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/password/reset.html.eex:12 +#, elixir-autogen, elixir-format +msgctxt "password reset button" +msgid "Reset" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/password/reset_failed.html.eex:4 +#, elixir-autogen, elixir-format +msgctxt "password reset failed homepage link" +msgid "Homepage" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/password/reset_failed.html.eex:1 +#, elixir-autogen, elixir-format +msgctxt "password reset failed message" +msgid "Password reset failed" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/password/reset.html.eex:8 +#, elixir-autogen, elixir-format +msgctxt "password reset form confirm password prompt" +msgid "Confirmation" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/password/reset.html.eex:4 +#, elixir-autogen, elixir-format +msgctxt "password reset form password prompt" +msgid "Password" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/password/invalid_token.html.eex:1 +#, elixir-autogen, elixir-format +msgctxt "password reset invalid token message" +msgid "Invalid Token" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/password/reset_success.html.eex:2 +#, elixir-autogen, elixir-format +msgctxt "password reset successful homepage link" +msgid "Homepage" +msgstr "" + +#: lib/pleroma/web/templates/twitter_api/password/reset_success.html.eex:1 +#, elixir-autogen, elixir-format +msgctxt "password reset successful message" +msgid "Password changed!" +msgstr "" + +#: lib/pleroma/web/templates/feed/feed/tag.atom.eex:15 +#: lib/pleroma/web/templates/feed/feed/tag.rss.eex:7 +#, elixir-autogen, elixir-format +msgctxt "tag feed description" +msgid "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse." +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex:1 +#, elixir-autogen, elixir-format +msgctxt "oauth authorization exists page title" +msgid "Authorization exists" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:32 +#, elixir-autogen, elixir-format +msgctxt "oauth authorize approve button" +msgid "Approve" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:30 +#, elixir-autogen, elixir-format +msgctxt "oauth authorize cancel button" +msgid "Cancel" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:23 +#, elixir-autogen, elixir-format +msgctxt "oauth authorize message" +msgid "Application %{client_name} is requesting access to your account." +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex:1 +#, elixir-autogen, elixir-format +msgctxt "oauth authorized page title" +msgid "Successfully authorized" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex:1 +#, elixir-autogen, elixir-format +msgctxt "oauth external provider page title" +msgid "Sign in with external provider" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex:13 +#, elixir-autogen, elixir-format +msgctxt "oauth external provider sign in button" +msgid "Sign in with %{strategy}" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:54 +#, elixir-autogen, elixir-format +msgctxt "oauth login button" +msgid "Log In" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:51 +#, elixir-autogen, elixir-format +msgctxt "oauth login password prompt" +msgid "Password" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:47 +#, elixir-autogen, elixir-format +msgctxt "oauth login username prompt" +msgid "Username" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:39 +#, elixir-autogen, elixir-format +msgctxt "oauth register nickname prompt" +msgid "Pleroma Handle" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:37 +#, elixir-autogen, elixir-format +msgctxt "oauth register nickname unchangeable warning" +msgid "Choose carefully! You won't be able to change this later. You will be able to change your display name, though." +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:18 +#, elixir-autogen, elixir-format +msgctxt "oauth register page email prompt" +msgid "Email" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:10 +#, elixir-autogen, elixir-format +msgctxt "oauth register page fill form prompt" +msgid "If you'd like to register a new account, please provide the details below." +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:35 +#, elixir-autogen, elixir-format +msgctxt "oauth register page login button" +msgid "Proceed as existing user" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:31 +#, elixir-autogen, elixir-format +msgctxt "oauth register page login password prompt" +msgid "Password" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:24 +#, elixir-autogen, elixir-format +msgctxt "oauth register page login prompt" +msgid "Alternatively, sign in to connect to existing account." +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:27 +#, elixir-autogen, elixir-format +msgctxt "oauth register page login username prompt" +msgid "Name or email" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:14 +#, elixir-autogen, elixir-format +msgctxt "oauth register page nickname prompt" +msgid "Nickname" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:22 +#, elixir-autogen, elixir-format +msgctxt "oauth register page register button" +msgid "Proceed as new user" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:8 +#, elixir-autogen, elixir-format +msgctxt "oauth register page title" +msgid "Registration Details" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:36 +#, elixir-autogen, elixir-format +msgctxt "oauth register page title" +msgid "This is the first time you visit! Please enter your Pleroma handle." +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex:2 +#, elixir-autogen, elixir-format +msgctxt "oauth scopes message" +msgid "The following permissions will be granted" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex:2 +#: lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex:2 +#, elixir-autogen, elixir-format +msgctxt "oauth token code message" +msgid "Token code is
%{token}" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:12 +#, elixir-autogen, elixir-format +msgctxt "mfa auth code prompt" +msgid "Authentication code" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:8 +#, elixir-autogen, elixir-format +msgctxt "mfa auth page title" +msgid "Two-factor authentication" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:23 +#, elixir-autogen, elixir-format +msgctxt "mfa auth page use recovery code link" +msgid "Enter a two-factor recovery code" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:20 +#, elixir-autogen, elixir-format +msgctxt "mfa auth verify code button" +msgid "Verify" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:8 +#, elixir-autogen, elixir-format +msgctxt "mfa recover page title" +msgid "Two-factor recovery" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:12 +#, elixir-autogen, elixir-format +msgctxt "mfa recover recovery code prompt" +msgid "Recovery code" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:23 +#, elixir-autogen, elixir-format +msgctxt "mfa recover use 2fa code link" +msgid "Enter a two-factor code" +msgstr "" + +#: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:20 +#, elixir-autogen, elixir-format +msgctxt "mfa recover verify recovery code button" +msgid "Verify" +msgstr "" + +#: lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex:8 +#, elixir-autogen, elixir-format +msgctxt "static fe profile page remote follow button" +msgid "Remote follow" +msgstr "" + +#: lib/pleroma/web/templates/email/digest.html.eex:163 +#, elixir-autogen, elixir-format +msgctxt "digest email header line" +msgid "Hey %{nickname}, here is what you've missed!" +msgstr "" + +#: lib/pleroma/web/templates/email/digest.html.eex:544 +#, elixir-autogen, elixir-format +msgctxt "digest email receiver address" +msgid "The email address you are subscribed as is %{email}. " +msgstr "" + +#: lib/pleroma/web/templates/email/digest.html.eex:538 +#, elixir-autogen, elixir-format +msgctxt "digest email sending reason" +msgid "You have received this email because you have signed up to receive digest emails from %{instance} Pleroma instance." +msgstr "" + +#: lib/pleroma/web/templates/email/digest.html.eex:547 +#, elixir-autogen, elixir-format +msgctxt "digest email unsubscribe action" +msgid "To unsubscribe, please go %{here}." +msgstr "" + +#: lib/pleroma/web/templates/email/digest.html.eex:547 +#, elixir-autogen, elixir-format +msgctxt "digest email unsubscribe action link text" +msgid "here" +msgstr "" + +#: lib/pleroma/web/templates/mailer/subscription/unsubscribe_failure.html.eex:1 +#, elixir-autogen, elixir-format +msgctxt "mailer unsubscribe failed message" +msgid "UNSUBSCRIBE FAILURE" +msgstr "" + +#: lib/pleroma/web/templates/mailer/subscription/unsubscribe_success.html.eex:1 +#, elixir-autogen, elixir-format +msgctxt "mailer unsubscribe successful message" +msgid "UNSUBSCRIBE SUCCESSFUL" +msgstr "" + +#: lib/pleroma/web/templates/email/digest.html.eex:385 +#, elixir-format +msgctxt "new followers count header" +msgid "%{count} New Follower" +msgid_plural "%{count} New Followers" +msgstr[0] "" +msgstr[1] "" + +#: lib/pleroma/emails/user_email.ex:356 +#, elixir-autogen, elixir-format +msgctxt "account archive email body - self-requested" +msgid "

You requested a full backup of your Pleroma account. It's ready for download:

\n

%{download_url}

\n" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:384 +#, elixir-autogen, elixir-format +msgctxt "account archive email subject" +msgid "Your account archive is ready" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:188 +#, elixir-autogen, elixir-format +msgctxt "approval pending email body" +msgid "

Awaiting Approval

\n

Your account at %{instance_name} is being reviewed by staff. You will receive another email once your account is approved.

\n" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:202 +#, elixir-autogen, elixir-format +msgctxt "approval pending email subject" +msgid "Your account is awaiting approval" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:158 +#, elixir-autogen, elixir-format +msgctxt "confirmation email body" +msgid "

Thank you for registering on %{instance_name}

\n

Email confirmation is required to activate the account.

\n

Please click the following link to activate your account.

\n" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:174 +#, elixir-autogen, elixir-format +msgctxt "confirmation email subject" +msgid "%{instance_name} account confirmation" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:310 +#, elixir-autogen, elixir-format +msgctxt "digest email subject" +msgid "Your digest from %{instance_name}" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:81 +#, elixir-autogen, elixir-format +msgctxt "password reset email body" +msgid "

Reset your password at %{instance_name}

\n

Someone has requested password change for your account at %{instance_name}.

\n

If it was you, visit the following link to proceed: reset password.

\n

If it was someone else, nothing to worry about: your data is secure and your password has not been changed.

\n" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:98 +#, elixir-autogen, elixir-format +msgctxt "password reset email subject" +msgid "Password reset" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:215 +#, elixir-autogen, elixir-format +msgctxt "successful registration email body" +msgid "

Hello @%{nickname},

\n

Your account at %{instance_name} has been registered successfully.

\n

No further action is required to activate your account.

\n" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:231 +#, elixir-autogen, elixir-format +msgctxt "successful registration email subject" +msgid "Account registered on %{instance_name}" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:119 +#, elixir-autogen, elixir-format +msgctxt "user invitation email body" +msgid "

You are invited to %{instance_name}

\n

%{inviter_name} invites you to join %{instance_name}, an instance of Pleroma federated social networking platform.

\n

Click the following link to register: accept invitation.

\n" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:136 +#, elixir-autogen, elixir-format +msgctxt "user invitation email subject" +msgid "Invitation to %{instance_name}" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:53 +#, elixir-autogen, elixir-format +msgctxt "welcome email html body" +msgid "Welcome to %{instance_name}!" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:41 +#, elixir-autogen, elixir-format +msgctxt "welcome email subject" +msgid "Welcome to %{instance_name}!" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:65 +#, elixir-autogen, elixir-format +msgctxt "welcome email text body" +msgid "Welcome to %{instance_name}!" +msgstr "" + +#: lib/pleroma/emails/user_email.ex:368 +#, elixir-autogen, elixir-format +msgctxt "account archive email body - admin requested" +msgid "

Admin @%{admin_nickname} requested a full backup of your Pleroma account. It's ready for download:

\n

%{download_url}

\n" +msgstr "" From a6195c7127ceee372a85ea7ff4af7fb9457588cf Mon Sep 17 00:00:00 2001 From: Fristi Date: Sat, 13 Aug 2022 10:32:52 +0000 Subject: [PATCH 117/123] Added translation using Weblate (Dutch) --- priv/gettext/nl/LC_MESSAGES/posix_errors.po | 163 ++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 priv/gettext/nl/LC_MESSAGES/posix_errors.po diff --git a/priv/gettext/nl/LC_MESSAGES/posix_errors.po b/priv/gettext/nl/LC_MESSAGES/posix_errors.po new file mode 100644 index 000000000..d64fca5fc --- /dev/null +++ b/priv/gettext/nl/LC_MESSAGES/posix_errors.po @@ -0,0 +1,163 @@ +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2022-08-13 13:32+0300\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: Automatically generated\n" +"Language-Team: none\n" +"Language: nl\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"X-Generator: Translate Toolkit 3.7.2\n" + +## This file is a PO Template file. +## +## `msgid`s here are often extracted from source code. +## Add new translations manually only if they're dynamic +## translations that can't be statically extracted. +## +## Run `mix gettext.extract` to bring this file up to +## date. Leave `msgstr`s empty as changing them here as no +## effect: edit them in PO (`.po`) files instead. +msgid "eperm" +msgstr "" + +msgid "eacces" +msgstr "" + +msgid "eagain" +msgstr "" + +msgid "ebadf" +msgstr "" + +msgid "ebadmsg" +msgstr "" + +msgid "ebusy" +msgstr "" + +msgid "edeadlk" +msgstr "" + +msgid "edeadlock" +msgstr "" + +msgid "edquot" +msgstr "" + +msgid "eexist" +msgstr "" + +msgid "efault" +msgstr "" + +msgid "efbig" +msgstr "" + +msgid "eftype" +msgstr "" + +msgid "eintr" +msgstr "" + +msgid "einval" +msgstr "" + +msgid "eio" +msgstr "" + +msgid "eisdir" +msgstr "" + +msgid "eloop" +msgstr "" + +msgid "emfile" +msgstr "" + +msgid "emlink" +msgstr "" + +msgid "emultihop" +msgstr "" + +msgid "enametoolong" +msgstr "" + +msgid "enfile" +msgstr "" + +msgid "enobufs" +msgstr "" + +msgid "enodev" +msgstr "" + +msgid "enolck" +msgstr "" + +msgid "enolink" +msgstr "" + +msgid "enoent" +msgstr "" + +msgid "enomem" +msgstr "" + +msgid "enospc" +msgstr "" + +msgid "enosr" +msgstr "" + +msgid "enostr" +msgstr "" + +msgid "enosys" +msgstr "" + +msgid "enotblk" +msgstr "" + +msgid "enotdir" +msgstr "" + +msgid "enotsup" +msgstr "" + +msgid "enxio" +msgstr "" + +msgid "eopnotsupp" +msgstr "" + +msgid "eoverflow" +msgstr "" + +msgid "epipe" +msgstr "" + +msgid "erange" +msgstr "" + +msgid "erofs" +msgstr "" + +msgid "espipe" +msgstr "" + +msgid "esrch" +msgstr "" + +msgid "estale" +msgstr "" + +msgid "etxtbsy" +msgstr "" + +msgid "exdev" +msgstr "" From 425fbce7be05f1e99bf6b31b011266d8fdb39da6 Mon Sep 17 00:00:00 2001 From: Fristi Date: Sat, 13 Aug 2022 10:30:41 +0000 Subject: [PATCH 118/123] Translated using Weblate (Dutch) Currently translated at 100.0% (106 of 106 strings) Translation: Pleroma/Pleroma Backend (domain errors) Translate-URL: http://weblate.pleroma-dev.ebin.club/projects/pleroma/pleroma-backend-domain-errors/nl/ --- priv/gettext/nl/LC_MESSAGES/errors.po | 158 ++++++++++++++------------ 1 file changed, 84 insertions(+), 74 deletions(-) diff --git a/priv/gettext/nl/LC_MESSAGES/errors.po b/priv/gettext/nl/LC_MESSAGES/errors.po index cfcb05fe6..ce1d794cf 100644 --- a/priv/gettext/nl/LC_MESSAGES/errors.po +++ b/priv/gettext/nl/LC_MESSAGES/errors.po @@ -3,16 +3,16 @@ msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2020-05-15 09:37+0000\n" -"PO-Revision-Date: 2020-06-02 07:36+0000\n" +"PO-Revision-Date: 2022-08-14 11:04+0000\n" "Last-Translator: Fristi \n" -"Language-Team: Dutch \n" +"Language-Team: Dutch \n" "Language: nl\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=n != 1;\n" -"X-Generator: Weblate 4.0.4\n" +"X-Generator: Weblate 4.13.1\n" ## This file is a PO Template file. ## @@ -118,7 +118,7 @@ msgstr "Al gestemd" #: lib/pleroma/web/oauth/oauth_controller.ex:360 #, elixir-format msgid "Bad request" -msgstr "Bad request" +msgstr "Ongeldig request" #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:425 #, elixir-format @@ -155,7 +155,7 @@ msgstr "Object kan niet geliked worden" #: lib/pleroma/web/common_api/utils.ex:556 #, elixir-format msgid "Cannot post an empty status without attachments" -msgstr "Status kan niet geplaatst worden zonder tekst of bijlagen" +msgstr "Bericht kan niet geplaatst worden zonder tekst of bijlagen" #: lib/pleroma/web/common_api/utils.ex:504 #, elixir-format @@ -165,122 +165,122 @@ msgstr "Opmerking dient maximaal %{max_size} karakters te bevatten" #: lib/pleroma/config/config_db.ex:222 #, elixir-format msgid "Config with params %{params} not found" -msgstr "" +msgstr "Instelling met parameters %{params} kon niet gevonden worden" #: lib/pleroma/web/common_api/common_api.ex:95 #, elixir-format msgid "Could not delete" -msgstr "" +msgstr "Verwijderen mislukt" #: lib/pleroma/web/common_api/common_api.ex:141 #, elixir-format msgid "Could not favorite" -msgstr "" +msgstr "Favoriet maken mislukt" #: lib/pleroma/web/common_api/common_api.ex:370 #, elixir-format msgid "Could not pin" -msgstr "" +msgstr "Vastmaken mislukt" #: lib/pleroma/web/common_api/common_api.ex:112 #, elixir-format msgid "Could not repeat" -msgstr "" +msgstr "Herhalen mislukt" #: lib/pleroma/web/common_api/common_api.ex:188 #, elixir-format msgid "Could not unfavorite" -msgstr "" +msgstr "Favoriet ongedaan maken mislukt" #: lib/pleroma/web/common_api/common_api.ex:380 #, elixir-format msgid "Could not unpin" -msgstr "" +msgstr "Vastmaken ongedaan maken mislukt" #: lib/pleroma/web/common_api/common_api.ex:126 #, elixir-format msgid "Could not unrepeat" -msgstr "" +msgstr "Herhalen ongedaan maken mislukt" #: lib/pleroma/web/common_api/common_api.ex:428 #: lib/pleroma/web/common_api/common_api.ex:437 #, elixir-format msgid "Could not update state" -msgstr "" +msgstr "Status bijwerken mislukt" #: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:202 #, elixir-format msgid "Error." -msgstr "" +msgstr "Fout." #: lib/pleroma/web/twitter_api/twitter_api.ex:106 #, elixir-format msgid "Invalid CAPTCHA" -msgstr "" +msgstr "Ongeldige CAPTCHA" #: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:117 #: lib/pleroma/web/oauth/oauth_controller.ex:569 #, elixir-format msgid "Invalid credentials" -msgstr "" +msgstr "Ongeldige inloggegevens" #: lib/pleroma/plugs/ensure_authenticated_plug.ex:38 #, elixir-format msgid "Invalid credentials." -msgstr "" +msgstr "Ongeldige inloggegevens." #: lib/pleroma/web/common_api/common_api.ex:265 #, elixir-format msgid "Invalid indices" -msgstr "" +msgstr "Ongeldige indexen" #: lib/pleroma/web/admin_api/admin_api_controller.ex:1147 #, elixir-format msgid "Invalid parameters" -msgstr "" +msgstr "Ongeldige parameters" #: lib/pleroma/web/common_api/utils.ex:411 #, elixir-format msgid "Invalid password." -msgstr "" +msgstr "Ongeldig wachtwoord." #: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:187 #, elixir-format msgid "Invalid request" -msgstr "" +msgstr "Ongeldig request" #: lib/pleroma/web/twitter_api/twitter_api.ex:109 #, elixir-format msgid "Kocaptcha service unavailable" -msgstr "" +msgstr "Kocaptcha service niet beschikbaar" #: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:113 #, elixir-format msgid "Missing parameters" -msgstr "" +msgstr "Ontbrekende parameters" #: lib/pleroma/web/common_api/utils.ex:540 #, elixir-format msgid "No such conversation" -msgstr "" +msgstr "Gesprek niet gevonden" #: lib/pleroma/web/admin_api/admin_api_controller.ex:439 #: lib/pleroma/web/admin_api/admin_api_controller.ex:465 lib/pleroma/web/admin_api/admin_api_controller.ex:507 #, elixir-format msgid "No such permission_group" -msgstr "" +msgstr "Permission_group niet gevonden" #: lib/pleroma/plugs/uploaded_media.ex:74 #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:485 lib/pleroma/web/admin_api/admin_api_controller.ex:1135 #: lib/pleroma/web/feed/user_controller.ex:73 lib/pleroma/web/ostatus/ostatus_controller.ex:143 #, elixir-format msgid "Not found" -msgstr "" +msgstr "Niet gevonden" #: lib/pleroma/web/common_api/common_api.ex:241 #, elixir-format msgid "Poll's author can't vote" -msgstr "" +msgstr "De peiling-auteur kan niet stemmen" #: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:20 #: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:37 lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:49 @@ -288,215 +288,215 @@ msgstr "" #: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:71 #, elixir-format msgid "Record not found" -msgstr "" +msgstr "Record niet gevonden" #: lib/pleroma/web/admin_api/admin_api_controller.ex:1153 #: lib/pleroma/web/feed/user_controller.ex:79 lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:32 #: lib/pleroma/web/ostatus/ostatus_controller.ex:149 #, elixir-format msgid "Something went wrong" -msgstr "" +msgstr "Er is iets misgegaan" #: lib/pleroma/web/common_api/activity_draft.ex:107 #, elixir-format msgid "The message visibility must be direct" -msgstr "" +msgstr "De zichtbaarheid van het bericht dient privé te zijn" #: lib/pleroma/web/common_api/utils.ex:566 #, elixir-format msgid "The status is over the character limit" -msgstr "" +msgstr "Het bericht is langer dan het karakter-limiet" #: lib/pleroma/plugs/ensure_public_or_authenticated_plug.ex:31 #, elixir-format msgid "This resource requires authentication." -msgstr "" +msgstr "Deze gegevens vereisen authenticatie." #: lib/pleroma/plugs/rate_limiter/rate_limiter.ex:206 #, elixir-format msgid "Throttled" -msgstr "" +msgstr "Geremd" #: lib/pleroma/web/common_api/common_api.ex:266 #, elixir-format msgid "Too many choices" -msgstr "" +msgstr "Teveel keuzes" #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:442 #, elixir-format msgid "Unhandled activity type" -msgstr "" +msgstr "Niet-ondersteund activiteits-type" #: lib/pleroma/web/admin_api/admin_api_controller.ex:536 #, elixir-format msgid "You can't revoke your own admin status." -msgstr "" +msgstr "Je kan je eigen beheerdersrechten niet intrekken." #: lib/pleroma/web/oauth/oauth_controller.ex:218 #: lib/pleroma/web/oauth/oauth_controller.ex:309 #, elixir-format msgid "Your account is currently disabled" -msgstr "" +msgstr "Je account is momenteel uitgeschakeld" #: lib/pleroma/web/oauth/oauth_controller.ex:180 #: lib/pleroma/web/oauth/oauth_controller.ex:332 #, elixir-format msgid "Your login is missing a confirmed e-mail address" -msgstr "" +msgstr "Je login bevat geen bevestigd e-mailadres" #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:389 #, elixir-format msgid "can't read inbox of %{nickname} as %{as_nickname}" -msgstr "" +msgstr "kan de inbox van %{nickname} niet lezen als %{as_nickname}" #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:472 #, elixir-format msgid "can't update outbox of %{nickname} as %{as_nickname}" -msgstr "" +msgstr "kan de outbox van %{nickname} niet bijwerken als %{as_nickname}" #: lib/pleroma/web/common_api/common_api.ex:388 #, elixir-format msgid "conversation is already muted" -msgstr "" +msgstr "gesprek is al genegeerd" #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:316 #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:491 #, elixir-format msgid "error" -msgstr "" +msgstr "fout" #: lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:29 #, elixir-format msgid "mascots can only be images" -msgstr "" +msgstr "mascottes kunnen alleen afbeeldingen zijn" #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:60 #, elixir-format msgid "not found" -msgstr "" +msgstr "niet gevonden" #: lib/pleroma/web/oauth/oauth_controller.ex:395 #, elixir-format msgid "Bad OAuth request." -msgstr "" +msgstr "Ongeldig OAuth request." #: lib/pleroma/web/twitter_api/twitter_api.ex:115 #, elixir-format msgid "CAPTCHA already used" -msgstr "" +msgstr "CAPTCHA is al gebruikt" #: lib/pleroma/web/twitter_api/twitter_api.ex:112 #, elixir-format msgid "CAPTCHA expired" -msgstr "" +msgstr "CAPTCHA is verlopen" #: lib/pleroma/plugs/uploaded_media.ex:55 #, elixir-format msgid "Failed" -msgstr "" +msgstr "Mislukt" #: lib/pleroma/web/oauth/oauth_controller.ex:411 #, elixir-format msgid "Failed to authenticate: %{message}." -msgstr "" +msgstr "Authenticatie mislukt: %{message}." #: lib/pleroma/web/oauth/oauth_controller.ex:442 #, elixir-format msgid "Failed to set up user account." -msgstr "" +msgstr "Aanmaken van gebruikersaccount is mislukt." #: lib/pleroma/plugs/oauth_scopes_plug.ex:38 #, elixir-format msgid "Insufficient permissions: %{permissions}." -msgstr "" +msgstr "Niet voldoende rechten: %{permissions}." #: lib/pleroma/plugs/uploaded_media.ex:94 #, elixir-format msgid "Internal Error" -msgstr "" +msgstr "Interne Fout" #: lib/pleroma/web/oauth/fallback_controller.ex:22 #: lib/pleroma/web/oauth/fallback_controller.ex:29 #, elixir-format msgid "Invalid Username/Password" -msgstr "" +msgstr "Ongeldige Gebruikersnaam/Wachtwoord" #: lib/pleroma/web/twitter_api/twitter_api.ex:118 #, elixir-format msgid "Invalid answer data" -msgstr "" +msgstr "Ongeldig antwoord" #: lib/pleroma/web/nodeinfo/nodeinfo_controller.ex:128 #, elixir-format msgid "Nodeinfo schema version not handled" -msgstr "" +msgstr "Nodeinfo schema wordt niet ondersteund" #: lib/pleroma/web/oauth/oauth_controller.ex:169 #, elixir-format msgid "This action is outside the authorized scopes" -msgstr "" +msgstr "Deze actie bevindt zich buiten de gemachtigde scopes" #: lib/pleroma/web/oauth/fallback_controller.ex:14 #, elixir-format msgid "Unknown error, please check the details and try again." -msgstr "" +msgstr "Onbekende fout, controleer a.u.b. de details en probeer het opnieuw." #: lib/pleroma/web/oauth/oauth_controller.ex:116 #: lib/pleroma/web/oauth/oauth_controller.ex:155 #, elixir-format msgid "Unlisted redirect_uri." -msgstr "" +msgstr "Niet-vermelde redirect_uri." #: lib/pleroma/web/oauth/oauth_controller.ex:391 #, elixir-format msgid "Unsupported OAuth provider: %{provider}." -msgstr "" +msgstr "Niet ondersteunde OAuth provider: %{provider}." #: lib/pleroma/uploaders/uploader.ex:72 #, elixir-format msgid "Uploader callback timeout" -msgstr "" +msgstr "Uploader terugkoppeling timeout" #: lib/pleroma/web/uploader_controller.ex:23 #, elixir-format msgid "bad request" -msgstr "" +msgstr "ongeldig request" #: lib/pleroma/web/twitter_api/twitter_api.ex:103 #, elixir-format msgid "CAPTCHA Error" -msgstr "" +msgstr "CAPTCHA Fout" #: lib/pleroma/web/common_api/common_api.ex:200 #, elixir-format msgid "Could not add reaction emoji" -msgstr "" +msgstr "Reactie-emoji toevoegen mislukt" #: lib/pleroma/web/common_api/common_api.ex:211 #, elixir-format msgid "Could not remove reaction emoji" -msgstr "" +msgstr "Reactie-emoji verwijderen mislukt" #: lib/pleroma/web/twitter_api/twitter_api.ex:129 #, elixir-format msgid "Invalid CAPTCHA (Missing parameter: %{name})" -msgstr "" +msgstr "Ongeldige CAPTCHA (Ontbrekende parameter: %{name})" #: lib/pleroma/web/mastodon_api/controllers/list_controller.ex:92 #, elixir-format msgid "List not found" -msgstr "" +msgstr "Lijst niet gevonden" #: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:124 #, elixir-format msgid "Missing parameter: %{name}" -msgstr "" +msgstr "Ontbrekende parameter: %{name}" #: lib/pleroma/web/oauth/oauth_controller.ex:207 #: lib/pleroma/web/oauth/oauth_controller.ex:322 #, elixir-format msgid "Password reset is required" -msgstr "" +msgstr "Wachtwoordherstel is vereist" #: lib/pleroma/tests/auth_test_controller.ex:9 #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:6 lib/pleroma/web/admin_api/admin_api_controller.ex:6 @@ -528,53 +528,63 @@ msgstr "" #, elixir-format msgid "Security violation: OAuth scopes check was neither handled nor explicitly skipped." msgstr "" +"Schending van beveiliging: OAuth scope-controle is niet uitgevoerd en niet " +"expliciet overgeslagen." #: lib/pleroma/plugs/ensure_authenticated_plug.ex:28 #, elixir-format msgid "Two-factor authentication enabled, you must use a access token." msgstr "" +"Tweefactor authenticatie is ingeschakeld, een toegangssleutel is verplicht." #: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:210 #, elixir-format msgid "Unexpected error occurred while adding file to pack." msgstr "" +"Er is een onverwachte fout opgetreden tijdens het toevoegen van het bestand." #: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:138 #, elixir-format msgid "Unexpected error occurred while creating pack." msgstr "" +"Er is een onverwachte fout opgetreden tijdens het aanmaken van het pakket." #: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:278 #, elixir-format msgid "Unexpected error occurred while removing file from pack." msgstr "" +"Er is een onverwachte fout opgetreden tijdens het verwijderen van het " +"bestand." #: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:250 #, elixir-format msgid "Unexpected error occurred while updating file in pack." msgstr "" +"Er is een onverwachte fout opgetreden tijdens het bijwerken van het bestand." #: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:179 #, elixir-format msgid "Unexpected error occurred while updating pack metadata." msgstr "" +"Er is een onverwachte fout opgetreden tijdens het bijwerken van de pakket-" +"metadata." #: lib/pleroma/plugs/user_is_admin_plug.ex:21 #, elixir-format msgid "User is not an admin." -msgstr "" +msgstr "Gebruiker is niet een beheerder." #: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:61 #, elixir-format msgid "Web push subscription is disabled on this Pleroma instance" -msgstr "" +msgstr "Web push abbonement is uitgeschakeld op deze Pleroma instantie" #: lib/pleroma/web/admin_api/admin_api_controller.ex:502 #, elixir-format msgid "You can't revoke your own admin/moderator status." -msgstr "" +msgstr "Je kan je eigen beheerders- of moderatorrechten niet intrekken." #: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:105 #, elixir-format msgid "authorization required for timeline view" -msgstr "" +msgstr "machtiging is vereist voor de tijdlijn weergave" From 9af5da66660cbc0cd2c71ea08484b947de5ba38b Mon Sep 17 00:00:00 2001 From: Fristi Date: Sat, 13 Aug 2022 10:25:34 +0000 Subject: [PATCH 119/123] Translated using Weblate (Dutch) Currently translated at 100.0% (83 of 83 strings) Translation: Pleroma/Pleroma Backend (domain static_pages) Translate-URL: http://weblate.pleroma-dev.ebin.club/projects/pleroma/pleroma-backend-domain-static_pages/nl/ --- priv/gettext/nl/LC_MESSAGES/static_pages.po | 194 ++++++++++++-------- 1 file changed, 118 insertions(+), 76 deletions(-) diff --git a/priv/gettext/nl/LC_MESSAGES/static_pages.po b/priv/gettext/nl/LC_MESSAGES/static_pages.po index 91e2fa3c8..2972384fc 100644 --- a/priv/gettext/nl/LC_MESSAGES/static_pages.po +++ b/priv/gettext/nl/LC_MESSAGES/static_pages.po @@ -3,14 +3,16 @@ msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2022-08-13 13:24+0300\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: Automatically generated\n" -"Language-Team: none\n" +"PO-Revision-Date: 2022-08-14 11:04+0000\n" +"Last-Translator: Fristi \n" +"Language-Team: Dutch \n" "Language: nl\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"X-Generator: Translate Toolkit 3.7.2\n" +"Plural-Forms: nplurals=2; plural=n != 1;\n" +"X-Generator: Weblate 4.13.1\n" ## This file is a PO Template file. ## @@ -21,150 +23,149 @@ msgstr "" ## Run "mix gettext.extract" to bring this file up to ## date. Leave "msgstr"s empty as changing them here as no ## effect: edit them in PO (.po) files instead. - #: lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex:9 #, elixir-autogen, elixir-format msgctxt "remote follow authorization button" msgid "Authorize" -msgstr "" +msgstr "Machtigen" #: lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex:2 #, elixir-autogen, elixir-format msgctxt "remote follow error" msgid "Error fetching user" -msgstr "" +msgstr "Fout bij ophalen gebruiker" #: lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex:4 #, elixir-autogen, elixir-format msgctxt "remote follow header" msgid "Remote follow" -msgstr "" +msgstr "Extern volgen" #: lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex:8 #, elixir-autogen, elixir-format msgctxt "placeholder text for auth code entry" msgid "Authentication code" -msgstr "" +msgstr "Authenticatiecode" #: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:10 #, elixir-autogen, elixir-format msgctxt "placeholder text for password entry" msgid "Password" -msgstr "" +msgstr "Wachtwoord" #: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:8 #, elixir-autogen, elixir-format msgctxt "placeholder text for username entry" msgid "Username" -msgstr "" +msgstr "Gebruikersnaam" #: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:13 #, elixir-autogen, elixir-format msgctxt "remote follow authorization button for login" msgid "Authorize" -msgstr "" +msgstr "Machtigen" #: lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex:12 #, elixir-autogen, elixir-format msgctxt "remote follow authorization button for mfa" msgid "Authorize" -msgstr "" +msgstr "Machtigen" #: lib/pleroma/web/templates/twitter_api/remote_follow/followed.html.eex:2 #, elixir-autogen, elixir-format msgctxt "remote follow error" msgid "Error following account" -msgstr "" +msgstr "Fout bij volgen van account" #: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:4 #, elixir-autogen, elixir-format msgctxt "remote follow header, need login" msgid "Log in to follow" -msgstr "" +msgstr "Log in om te volgen" #: lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex:4 #, elixir-autogen, elixir-format msgctxt "remote follow mfa header" msgid "Two-factor authentication" -msgstr "" +msgstr "Tweefactor authenticatie" #: lib/pleroma/web/templates/twitter_api/remote_follow/followed.html.eex:4 #, elixir-autogen, elixir-format msgctxt "remote follow success" msgid "Account followed!" -msgstr "" +msgstr "Account gevolgd!" #: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:7 #, elixir-autogen, elixir-format msgctxt "placeholder text for account id" msgid "Your account ID, e.g. lain@quitter.se" -msgstr "" +msgstr "Je account ID, b.v. gebruiker@instantie.net" #: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:8 #, elixir-autogen, elixir-format msgctxt "remote follow authorization button for following with a remote account" msgid "Follow" -msgstr "" +msgstr "Volgen" #: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:2 #, elixir-autogen, elixir-format msgctxt "remote follow error" msgid "Error: %{error}" -msgstr "" +msgstr "Fout: %{error}" #: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:4 #, elixir-autogen, elixir-format msgctxt "remote follow header" msgid "Remotely follow %{nickname}" -msgstr "" +msgstr "%{nickname} extern volgen" #: lib/pleroma/web/templates/twitter_api/password/reset.html.eex:12 #, elixir-autogen, elixir-format msgctxt "password reset button" msgid "Reset" -msgstr "" +msgstr "Herstellen" #: lib/pleroma/web/templates/twitter_api/password/reset_failed.html.eex:4 #, elixir-autogen, elixir-format msgctxt "password reset failed homepage link" msgid "Homepage" -msgstr "" +msgstr "Homepagina" #: lib/pleroma/web/templates/twitter_api/password/reset_failed.html.eex:1 #, elixir-autogen, elixir-format msgctxt "password reset failed message" msgid "Password reset failed" -msgstr "" +msgstr "Wachtwoordherstel mislukt" #: lib/pleroma/web/templates/twitter_api/password/reset.html.eex:8 #, elixir-autogen, elixir-format msgctxt "password reset form confirm password prompt" msgid "Confirmation" -msgstr "" +msgstr "Bevestiging" #: lib/pleroma/web/templates/twitter_api/password/reset.html.eex:4 #, elixir-autogen, elixir-format msgctxt "password reset form password prompt" msgid "Password" -msgstr "" +msgstr "Wachtwoord" #: lib/pleroma/web/templates/twitter_api/password/invalid_token.html.eex:1 #, elixir-autogen, elixir-format msgctxt "password reset invalid token message" msgid "Invalid Token" -msgstr "" +msgstr "Ongeldige Token" #: lib/pleroma/web/templates/twitter_api/password/reset_success.html.eex:2 #, elixir-autogen, elixir-format msgctxt "password reset successful homepage link" msgid "Homepage" -msgstr "" +msgstr "Homepagina" #: lib/pleroma/web/templates/twitter_api/password/reset_success.html.eex:1 #, elixir-autogen, elixir-format msgctxt "password reset successful message" msgid "Password changed!" -msgstr "" +msgstr "Wachtwoord gewijzigd!" #: lib/pleroma/web/templates/feed/feed/tag.atom.eex:15 #: lib/pleroma/web/templates/feed/feed/tag.rss.eex:7 @@ -172,354 +173,395 @@ msgstr "" msgctxt "tag feed description" msgid "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse." msgstr "" +"Dit zijn openbare berichten die getagd zijn met #%{tag}. Je kunt op deze " +"reageren indien je een account hebt in de fediverse." #: lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex:1 #, elixir-autogen, elixir-format msgctxt "oauth authorization exists page title" msgid "Authorization exists" -msgstr "" +msgstr "Machtiging bestaat" #: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:32 #, elixir-autogen, elixir-format msgctxt "oauth authorize approve button" msgid "Approve" -msgstr "" +msgstr "Goedkeuren" #: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:30 #, elixir-autogen, elixir-format msgctxt "oauth authorize cancel button" msgid "Cancel" -msgstr "" +msgstr "Annuleren" #: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:23 #, elixir-autogen, elixir-format msgctxt "oauth authorize message" msgid "Application %{client_name} is requesting access to your account." msgstr "" +"Applicatie %{client_name} vraagt om toegang tot je account." #: lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex:1 #, elixir-autogen, elixir-format msgctxt "oauth authorized page title" msgid "Successfully authorized" -msgstr "" +msgstr "Machtiging is geslaagd" #: lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex:1 #, elixir-autogen, elixir-format msgctxt "oauth external provider page title" msgid "Sign in with external provider" -msgstr "" +msgstr "Inloggen bij externe provider" #: lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex:13 #, elixir-autogen, elixir-format msgctxt "oauth external provider sign in button" msgid "Sign in with %{strategy}" -msgstr "" +msgstr "Inloggen met %{strategy}" #: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:54 #, elixir-autogen, elixir-format msgctxt "oauth login button" msgid "Log In" -msgstr "" +msgstr "Inloggen" #: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:51 #, elixir-autogen, elixir-format msgctxt "oauth login password prompt" msgid "Password" -msgstr "" +msgstr "Wachtwoord" #: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:47 #, elixir-autogen, elixir-format msgctxt "oauth login username prompt" msgid "Username" -msgstr "" +msgstr "Gebruikersnaam" #: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:39 #, elixir-autogen, elixir-format msgctxt "oauth register nickname prompt" msgid "Pleroma Handle" -msgstr "" +msgstr "Pleroma Gebruiker" #: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:37 #, elixir-autogen, elixir-format msgctxt "oauth register nickname unchangeable warning" msgid "Choose carefully! You won't be able to change this later. You will be able to change your display name, though." msgstr "" +"Let op! Je kunt je accountnaam hierna niet meer wijzigen. Je kunt echter wel " +"nog je weergavenaam wijzigen." #: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:18 #, elixir-autogen, elixir-format msgctxt "oauth register page email prompt" msgid "Email" -msgstr "" +msgstr "E-mail" #: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:10 #, elixir-autogen, elixir-format msgctxt "oauth register page fill form prompt" msgid "If you'd like to register a new account, please provide the details below." msgstr "" +"Indien je graag een nieuw account wilt registreren, vul dan a.u.b de " +"onderstaande details in." #: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:35 #, elixir-autogen, elixir-format msgctxt "oauth register page login button" msgid "Proceed as existing user" -msgstr "" +msgstr "Doorgaan als bestaande gebruiker" #: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:31 #, elixir-autogen, elixir-format msgctxt "oauth register page login password prompt" msgid "Password" -msgstr "" +msgstr "Wachtwoord" #: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:24 #, elixir-autogen, elixir-format msgctxt "oauth register page login prompt" msgid "Alternatively, sign in to connect to existing account." -msgstr "" +msgstr "Alternatief, log in om te verbinden met een bestaand account." #: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:27 #, elixir-autogen, elixir-format msgctxt "oauth register page login username prompt" msgid "Name or email" -msgstr "" +msgstr "Naam of e-mail" #: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:14 #, elixir-autogen, elixir-format msgctxt "oauth register page nickname prompt" msgid "Nickname" -msgstr "" +msgstr "Weergavenaam" #: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:22 #, elixir-autogen, elixir-format msgctxt "oauth register page register button" msgid "Proceed as new user" -msgstr "" +msgstr "Doorgaan als nieuwe gebruiker" #: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:8 #, elixir-autogen, elixir-format msgctxt "oauth register page title" msgid "Registration Details" -msgstr "" +msgstr "Registratiegegevens" #: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:36 #, elixir-autogen, elixir-format msgctxt "oauth register page title" msgid "This is the first time you visit! Please enter your Pleroma handle." -msgstr "" +msgstr "Dit is je eerste bezoek! Vul a.u.b. je Pleroma gebruikersnaam in." #: lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex:2 #, elixir-autogen, elixir-format msgctxt "oauth scopes message" msgid "The following permissions will be granted" -msgstr "" +msgstr "De volgende rechten zullen worden toegekend" #: lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex:2 #: lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex:2 #, elixir-autogen, elixir-format msgctxt "oauth token code message" msgid "Token code is
%{token}" -msgstr "" +msgstr "Token code is
%{token}" #: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:12 #, elixir-autogen, elixir-format msgctxt "mfa auth code prompt" msgid "Authentication code" -msgstr "" +msgstr "Authenticatiecode" #: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:8 #, elixir-autogen, elixir-format msgctxt "mfa auth page title" msgid "Two-factor authentication" -msgstr "" +msgstr "Tweefactor authenticatie" #: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:23 #, elixir-autogen, elixir-format msgctxt "mfa auth page use recovery code link" msgid "Enter a two-factor recovery code" -msgstr "" +msgstr "Voer een tweefactor herstelcode in" #: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:20 #, elixir-autogen, elixir-format msgctxt "mfa auth verify code button" msgid "Verify" -msgstr "" +msgstr "Controleren" #: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:8 #, elixir-autogen, elixir-format msgctxt "mfa recover page title" msgid "Two-factor recovery" -msgstr "" +msgstr "Tweefactor herstel" #: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:12 #, elixir-autogen, elixir-format msgctxt "mfa recover recovery code prompt" msgid "Recovery code" -msgstr "" +msgstr "Herstelcode" #: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:23 #, elixir-autogen, elixir-format msgctxt "mfa recover use 2fa code link" msgid "Enter a two-factor code" -msgstr "" +msgstr "Voer een tweefactor code in" #: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:20 #, elixir-autogen, elixir-format msgctxt "mfa recover verify recovery code button" msgid "Verify" -msgstr "" +msgstr "Controleren" #: lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex:8 #, elixir-autogen, elixir-format msgctxt "static fe profile page remote follow button" msgid "Remote follow" -msgstr "" +msgstr "Extern volgen" #: lib/pleroma/web/templates/email/digest.html.eex:163 #, elixir-autogen, elixir-format msgctxt "digest email header line" msgid "Hey %{nickname}, here is what you've missed!" -msgstr "" +msgstr "Hoi %{nickname}, dit is wat je hebt gemist!" #: lib/pleroma/web/templates/email/digest.html.eex:544 #, elixir-autogen, elixir-format msgctxt "digest email receiver address" msgid "The email address you are subscribed as is %{email}. " msgstr "" +"Het e-mailadres waarmee je bent ingeschreven is %{email}. " #: lib/pleroma/web/templates/email/digest.html.eex:538 #, elixir-autogen, elixir-format msgctxt "digest email sending reason" msgid "You have received this email because you have signed up to receive digest emails from %{instance} Pleroma instance." msgstr "" +"Je ontvangt deze e-mail omdat je bent ingeschreven voor overzichts-mails te " +"ontvangen van %{instance} Pleroma instantie." #: lib/pleroma/web/templates/email/digest.html.eex:547 #, elixir-autogen, elixir-format msgctxt "digest email unsubscribe action" msgid "To unsubscribe, please go %{here}." -msgstr "" +msgstr "Je kunt je %{here} uitschrijven voor deze e-mails." #: lib/pleroma/web/templates/email/digest.html.eex:547 #, elixir-autogen, elixir-format msgctxt "digest email unsubscribe action link text" msgid "here" -msgstr "" +msgstr "hier" #: lib/pleroma/web/templates/mailer/subscription/unsubscribe_failure.html.eex:1 #, elixir-autogen, elixir-format msgctxt "mailer unsubscribe failed message" msgid "UNSUBSCRIBE FAILURE" -msgstr "" +msgstr "UITSCHRIJVEN MISLUKT" #: lib/pleroma/web/templates/mailer/subscription/unsubscribe_success.html.eex:1 #, elixir-autogen, elixir-format msgctxt "mailer unsubscribe successful message" msgid "UNSUBSCRIBE SUCCESSFUL" -msgstr "" +msgstr "UITSCHRIJVEN GESLAAGD" #: lib/pleroma/web/templates/email/digest.html.eex:385 #, elixir-format msgctxt "new followers count header" msgid "%{count} New Follower" msgid_plural "%{count} New Followers" -msgstr[0] "" -msgstr[1] "" +msgstr[0] "%{count} Nieuwe Volger" +msgstr[1] "%{count} Nieuwe Volgers" #: lib/pleroma/emails/user_email.ex:356 #, elixir-autogen, elixir-format msgctxt "account archive email body - self-requested" msgid "

You requested a full backup of your Pleroma account. It's ready for download:

\n

%{download_url}

\n" msgstr "" +"

Je hebt een verzoek ingediend voor een volledige back-up van je Pleroma " +"account. Deze is gereed om te downloaden:

\n" +"

%{download_url}

\n" #: lib/pleroma/emails/user_email.ex:384 #, elixir-autogen, elixir-format msgctxt "account archive email subject" msgid "Your account archive is ready" -msgstr "" +msgstr "Je account archief is gereed" #: lib/pleroma/emails/user_email.ex:188 #, elixir-autogen, elixir-format msgctxt "approval pending email body" msgid "

Awaiting Approval

\n

Your account at %{instance_name} is being reviewed by staff. You will receive another email once your account is approved.

\n" msgstr "" +"

Goedkeuring in afwachting

\n" +"

Je account bij %{instance_name} zal worden beoordeeld door de beheerders. " +"Je zult een opvolgende e-mail ontvangen wanneer je account goed gekeurd " +"is.

\n" #: lib/pleroma/emails/user_email.ex:202 #, elixir-autogen, elixir-format msgctxt "approval pending email subject" msgid "Your account is awaiting approval" -msgstr "" +msgstr "Je account is in afwachting van goedkeuring" #: lib/pleroma/emails/user_email.ex:158 #, elixir-autogen, elixir-format msgctxt "confirmation email body" msgid "

Thank you for registering on %{instance_name}

\n

Email confirmation is required to activate the account.

\n

Please click the following link to activate your account.

\n" msgstr "" +"

Bedankt voor het registreren bij %{instance_name}

\n" +"

Bevestiging via e-mail is vereist om je account te activeren.

\n" +"

Je kunt je account activeren door op deze " +"link te klikken.

\n" #: lib/pleroma/emails/user_email.ex:174 #, elixir-autogen, elixir-format msgctxt "confirmation email subject" msgid "%{instance_name} account confirmation" -msgstr "" +msgstr "%{instance_name} account bevestiging" #: lib/pleroma/emails/user_email.ex:310 #, elixir-autogen, elixir-format msgctxt "digest email subject" msgid "Your digest from %{instance_name}" -msgstr "" +msgstr "Je overzicht van %{instance_name}" #: lib/pleroma/emails/user_email.ex:81 #, elixir-autogen, elixir-format msgctxt "password reset email body" msgid "

Reset your password at %{instance_name}

\n

Someone has requested password change for your account at %{instance_name}.

\n

If it was you, visit the following link to proceed: reset password.

\n

If it was someone else, nothing to worry about: your data is secure and your password has not been changed.

\n" msgstr "" +"

Herstel je wachtwoord bij %{instance_name}

\n" +"

Iemand heeft een verzoek ingediend om het wachtwoord van je account bij " +"%{instance_name} te herstellen.

\n" +"

Als je dit zelf geweest bent, volg dan de volgende link om door te gaan: " +"wachtwoord herstellen.

\n" +"

Indien je dit niet geweest bent, hoef je geen verdere acties te " +"ondernemen: je gegevens zijn veilig en je wachtwoord is niet gewijzigd.

\n" #: lib/pleroma/emails/user_email.ex:98 #, elixir-autogen, elixir-format msgctxt "password reset email subject" msgid "Password reset" -msgstr "" +msgstr "Wachtwoord herstellen" #: lib/pleroma/emails/user_email.ex:215 #, elixir-autogen, elixir-format msgctxt "successful registration email body" msgid "

Hello @%{nickname},

\n

Your account at %{instance_name} has been registered successfully.

\n

No further action is required to activate your account.

\n" msgstr "" +"

Hoi @%{nickname},

\n" +"

Het registreren van je account bij %{instance_name} is gelukt.

\n" +"

Er zijn geen verdere stappen vereist om je account te activeren.

\n" #: lib/pleroma/emails/user_email.ex:231 #, elixir-autogen, elixir-format msgctxt "successful registration email subject" msgid "Account registered on %{instance_name}" -msgstr "" +msgstr "Account registratie bij %{instance_name}" #: lib/pleroma/emails/user_email.ex:119 #, elixir-autogen, elixir-format msgctxt "user invitation email body" msgid "

You are invited to %{instance_name}

\n

%{inviter_name} invites you to join %{instance_name}, an instance of Pleroma federated social networking platform.

\n

Click the following link to register: accept invitation.

\n" msgstr "" +"

Je bent uitgenodigd bij %{instance_name}

\n" +"

%{inviter_name} nodigt je uit om je te registreren bij %{instance_name}, " +"een instantie van het Pleroma gefedereerde sociale netwerk.

\n" +"

Om je te registreren, klink op de volgende link: uitnodiging accepteren.

\n" #: lib/pleroma/emails/user_email.ex:136 #, elixir-autogen, elixir-format msgctxt "user invitation email subject" msgid "Invitation to %{instance_name}" -msgstr "" +msgstr "Uitnodiging van %{instance_name}" #: lib/pleroma/emails/user_email.ex:53 #, elixir-autogen, elixir-format msgctxt "welcome email html body" msgid "Welcome to %{instance_name}!" -msgstr "" +msgstr "Welkom bij %{instance_name}!" #: lib/pleroma/emails/user_email.ex:41 #, elixir-autogen, elixir-format msgctxt "welcome email subject" msgid "Welcome to %{instance_name}!" -msgstr "" +msgstr "Welkom bij %{instance_name}!" #: lib/pleroma/emails/user_email.ex:65 #, elixir-autogen, elixir-format msgctxt "welcome email text body" msgid "Welcome to %{instance_name}!" -msgstr "" +msgstr "Welkom bij %{instance_name}!" #: lib/pleroma/emails/user_email.ex:368 #, elixir-autogen, elixir-format msgctxt "account archive email body - admin requested" msgid "

Admin @%{admin_nickname} requested a full backup of your Pleroma account. It's ready for download:

\n

%{download_url}

\n" msgstr "" +"

Beheerder @%{admin_nickname} heeft een verzoek ingediend voor een " +"volledige back-up van je Pleroma account. Deze is gereed om te " +"downloaden:

\n" +"

%{download_url}

\n" From 0d8c6b0488c762bad79a6cdcfa8ed6d317b03f68 Mon Sep 17 00:00:00 2001 From: Fristi Date: Sat, 13 Aug 2022 10:33:23 +0000 Subject: [PATCH 120/123] Translated using Weblate (Dutch) Currently translated at 63.8% (30 of 47 strings) Translation: Pleroma/Pleroma Backend (domain posix_errors) Translate-URL: http://weblate.pleroma-dev.ebin.club/projects/pleroma/pleroma-backend-domain-posix_errors/nl/ --- priv/gettext/nl/LC_MESSAGES/posix_errors.po | 70 +++++++++++---------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/priv/gettext/nl/LC_MESSAGES/posix_errors.po b/priv/gettext/nl/LC_MESSAGES/posix_errors.po index d64fca5fc..cdb1f532f 100644 --- a/priv/gettext/nl/LC_MESSAGES/posix_errors.po +++ b/priv/gettext/nl/LC_MESSAGES/posix_errors.po @@ -3,14 +3,16 @@ msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2022-08-13 13:32+0300\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: Automatically generated\n" -"Language-Team: none\n" +"PO-Revision-Date: 2022-08-14 11:04+0000\n" +"Last-Translator: Fristi \n" +"Language-Team: Dutch \n" "Language: nl\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"X-Generator: Translate Toolkit 3.7.2\n" +"Plural-Forms: nplurals=2; plural=n != 1;\n" +"X-Generator: Weblate 4.13.1\n" ## This file is a PO Template file. ## @@ -22,94 +24,94 @@ msgstr "" ## date. Leave `msgstr`s empty as changing them here as no ## effect: edit them in PO (`.po`) files instead. msgid "eperm" -msgstr "" +msgstr "Uitvoering niet toegestaan" msgid "eacces" -msgstr "" +msgstr "Toegang geweigerd" msgid "eagain" -msgstr "" +msgstr "Resource tijdelijk niet beschikbaar" msgid "ebadf" -msgstr "" +msgstr "Ongeldige file descriptor" msgid "ebadmsg" -msgstr "" +msgstr "Ongeldig bericht" msgid "ebusy" -msgstr "" +msgstr "Apparaat of resource bezet" msgid "edeadlk" -msgstr "" +msgstr "Resource deadlock vermeden" msgid "edeadlock" -msgstr "" +msgstr "Resource deadlock vermeden" msgid "edquot" -msgstr "" +msgstr "Schijf-quota overschreden" msgid "eexist" -msgstr "" +msgstr "Bestand bestaat" msgid "efault" -msgstr "" +msgstr "Ongeldig adres" msgid "efbig" -msgstr "" +msgstr "Bestand is te groot" msgid "eftype" -msgstr "" +msgstr "Ongepast bestands-type of formaat" msgid "eintr" -msgstr "" +msgstr "Onderbroken systeem aanroep" msgid "einval" -msgstr "" +msgstr "Ongeldig argument" msgid "eio" -msgstr "" +msgstr "Input/output fout" msgid "eisdir" -msgstr "" +msgstr "Illegale bewerking op een directory" msgid "eloop" -msgstr "" +msgstr "Te veel niveau's van symbolische koppelingen" msgid "emfile" -msgstr "" +msgstr "Te veel geopende bestanden" msgid "emlink" -msgstr "" +msgstr "Te veel koppelingen" msgid "emultihop" -msgstr "" +msgstr "Multihop geprobeerd" msgid "enametoolong" -msgstr "" +msgstr "Bestandsnaam is te lang" msgid "enfile" -msgstr "" +msgstr "Te veel geopende bestanden in systeem" msgid "enobufs" -msgstr "" +msgstr "Geen buffer-ruimte beschikbaar" msgid "enodev" -msgstr "" +msgstr "Apparaat bestaat niet" msgid "enolck" -msgstr "" +msgstr "Geen sloten beschikbaar" msgid "enolink" -msgstr "" +msgstr "Koppeling is ongedaan gemaakt" msgid "enoent" -msgstr "" +msgstr "Bestand of directory bestaat niet" msgid "enomem" -msgstr "" +msgstr "Geheugen kon niet toegewezen worden" msgid "enospc" -msgstr "" +msgstr "Geen ruimte over op apparaat" msgid "enosr" msgstr "" From b439e91f57d1d7f26e94acc62703a64c747773da Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Fri, 2 Sep 2022 22:35:40 +0000 Subject: [PATCH 121/123] Revert "Merge branch 'rewrite/integration-test-websocket-client' into 'develop'" This reverts merge request !3649 --- mix.exs | 3 +- mix.lock | 1 - .../integration/mastodon_websocket_test.exs | 27 +-- test/support/websocket_client.ex | 195 +++--------------- 4 files changed, 37 insertions(+), 189 deletions(-) diff --git a/mix.exs b/mix.exs index 81c4cf9ae..6e84fe482 100644 --- a/mix.exs +++ b/mix.exs @@ -211,8 +211,7 @@ defmodule Pleroma.Mixfile do {:excoveralls, "0.12.3", only: :test}, {:hackney, "~> 1.18.0", override: true}, {:mox, "~> 1.0", only: :test}, - {:mint, "~> 1.4", only: :test, override: true}, - {:mint_web_socket, "~> 0.3.0", only: :test} + {:websocket_client, git: "https://github.com/jeremyong/websocket_client.git", only: :test} ] ++ oauth_deps() end diff --git a/mix.lock b/mix.lock index bd550041d..14e43c703 100644 --- a/mix.lock +++ b/mix.lock @@ -79,7 +79,6 @@ "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "mint": {:hex, :mint, "1.4.0", "cd7d2451b201fc8e4a8fd86257fb3878d9e3752899eb67b0c5b25b180bde1212", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "10a99e144b815cbf8522dccbc8199d15802440fc7a64d67b6853adb6fa170217"}, - "mint_web_socket": {:hex, :mint_web_socket, "0.3.0", "c9e130dcc778d673fd713eb66434e16cf7d89cee0754e75f26f8bd9a9e592b63", [:mix], [{:mint, "~> 1.4 and >= 1.4.1", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "0605bc3fa684e1a7719b22a3f74be4de5e6a16dd43ac18ebcea72e2adc33b532"}, "mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"}, "mock": {:hex, :mock, "0.3.7", "75b3bbf1466d7e486ea2052a73c6e062c6256fb429d6797999ab02fa32f29e03", [:mix], [{:meck, "~> 0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "4da49a4609e41fd99b7836945c26f373623ea968cfb6282742bcb94440cf7e5c"}, "mogrify": {:hex, :mogrify, "0.9.1", "a26f107c4987477769f272bd0f7e3ac4b7b75b11ba597fd001b877beffa9c068", [:mix], [], "hexpm", "134edf189337d2125c0948bf0c228fdeef975c594317452d536224069a5b7f05"}, diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs index 16525c740..2d4c7f63b 100644 --- a/test/pleroma/integration/mastodon_websocket_test.exs +++ b/test/pleroma/integration/mastodon_websocket_test.exs @@ -28,28 +28,21 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do qs -> @path <> qs end - WebsocketClient.connect(self(), path, headers) + WebsocketClient.start_link(self(), path, headers) end test "refuses invalid requests" do capture_log(fn -> - assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 404}} = start_socket() - - assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 404}} = - start_socket("?stream=ncjdk") - + assert {:error, {404, _}} = start_socket() + assert {:error, {404, _}} = start_socket("?stream=ncjdk") Process.sleep(30) end) end test "requires authentication and a valid token for protected streams" do capture_log(fn -> - assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = - start_socket("?stream=user&access_token=aaaaaaaaaaaa") - - assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = - start_socket("?stream=user") - + assert {:error, {401, _}} = start_socket("?stream=user&access_token=aaaaaaaaaaaa") + assert {:error, {401, _}} = start_socket("?stream=user") Process.sleep(30) end) end @@ -109,9 +102,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") capture_log(fn -> - assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = - start_socket("?stream=user") - + assert {:error, {401, _}} = start_socket("?stream=user") Process.sleep(30) end) end @@ -120,9 +111,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") capture_log(fn -> - assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = - start_socket("?stream=user:notification") - + assert {:error, {401, _}} = start_socket("?stream=user:notification") Process.sleep(30) end) end @@ -131,7 +120,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) capture_log(fn -> - assert {:error, %Mint.WebSocket.UpgradeFailureError{status_code: 401}} = + assert {:error, {401, _}} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", "I am a friend"}]) Process.sleep(30) diff --git a/test/support/websocket_client.ex b/test/support/websocket_client.ex index 43f2854de..d149b324e 100644 --- a/test/support/websocket_client.ex +++ b/test/support/websocket_client.ex @@ -3,199 +3,60 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Integration.WebsocketClient do - @moduledoc """ - A WebSocket client used to test Mastodon API streaming - - Based on Phoenix Framework's WebsocketClient - https://github.com/phoenixframework/phoenix/blob/master/test/support/websocket_client.exs - """ - - use GenServer - import Kernel, except: [send: 2] - - defstruct [ - :conn, - :request_ref, - :websocket, - :caller, - :status, - :resp_headers, - :sender, - closing?: false - ] + # https://github.com/phoenixframework/phoenix/blob/master/test/support/websocket_client.exs @doc """ - Starts the WebSocket client for given ws URL. `Phoenix.Socket.Message`s - received from the server are forwarded to the sender pid. + Starts the WebSocket server for given ws URL. Received Socket.Message's + are forwarded to the sender pid """ - def connect(sender, url, headers \\ []) do - with {:ok, socket} <- GenServer.start_link(__MODULE__, {sender}), - {:ok, :connected} <- GenServer.call(socket, {:connect, url, headers}) do - {:ok, socket} - end + def start_link(sender, url, headers \\ []) do + :crypto.start() + :ssl.start() + + :websocket_client.start_link( + String.to_charlist(url), + __MODULE__, + [sender], + extra_headers: headers + ) end @doc """ Closes the socket """ def close(socket) do - GenServer.cast(socket, :close) + send(socket, :close) end @doc """ Sends a low-level text message to the client. """ def send_text(server_pid, msg) do - GenServer.call(server_pid, {:text, msg}) + send(server_pid, {:text, msg}) end @doc false - def init({sender}) do - state = %__MODULE__{sender: sender} + def init([sender], _conn_state) do + {:ok, %{sender: sender}} + end + @doc false + def websocket_handle(frame, _conn_state, state) do + send(state.sender, frame) {:ok, state} end @doc false - def handle_call({:connect, url, headers}, from, state) do - uri = URI.parse(url) + def websocket_info({:text, msg}, _conn_state, state) do + {:reply, {:text, msg}, state} + end - http_scheme = - case uri.scheme do - "ws" -> :http - "wss" -> :https - end - - ws_scheme = - case uri.scheme do - "ws" -> :ws - "wss" -> :wss - end - - path = - case uri.query do - nil -> uri.path - query -> uri.path <> "?" <> query - end - - with {:ok, conn} <- Mint.HTTP.connect(http_scheme, uri.host, uri.port), - {:ok, conn, ref} <- Mint.WebSocket.upgrade(ws_scheme, conn, path, headers) do - state = %{state | conn: conn, request_ref: ref, caller: from} - {:noreply, state} - else - {:error, reason} -> - {:reply, {:error, reason}, state} - - {:error, conn, reason} -> - {:reply, {:error, reason}, put_in(state.conn, conn)} - end + def websocket_info(:close, _conn_state, _state) do + {:close, <<>>, "done"} end @doc false - def handle_info(message, state) do - case Mint.WebSocket.stream(state.conn, message) do - {:ok, conn, responses} -> - state = put_in(state.conn, conn) |> handle_responses(responses) - if state.closing?, do: do_close(state), else: {:noreply, state} - - {:error, conn, reason, _responses} -> - state = put_in(state.conn, conn) |> reply({:error, reason}) - {:noreply, state} - - :unknown -> - {:noreply, state} - end - end - - defp do_close(state) do - # Streaming a close frame may fail if the server has already closed - # for writing. - _ = stream_frame(state, :close) - Mint.HTTP.close(state.conn) - {:stop, :normal, state} - end - - defp handle_responses(state, responses) - - defp handle_responses(%{request_ref: ref} = state, [{:status, ref, status} | rest]) do - put_in(state.status, status) - |> handle_responses(rest) - end - - defp handle_responses(%{request_ref: ref} = state, [{:headers, ref, resp_headers} | rest]) do - put_in(state.resp_headers, resp_headers) - |> handle_responses(rest) - end - - defp handle_responses(%{request_ref: ref} = state, [{:done, ref} | rest]) do - case Mint.WebSocket.new(state.conn, ref, state.status, state.resp_headers) do - {:ok, conn, websocket} -> - %{state | conn: conn, websocket: websocket, status: nil, resp_headers: nil} - |> reply({:ok, :connected}) - |> handle_responses(rest) - - {:error, conn, reason} -> - put_in(state.conn, conn) - |> reply({:error, reason}) - end - end - - defp handle_responses(%{request_ref: ref, websocket: websocket} = state, [ - {:data, ref, data} | rest - ]) - when websocket != nil do - case Mint.WebSocket.decode(websocket, data) do - {:ok, websocket, frames} -> - put_in(state.websocket, websocket) - |> handle_frames(frames) - |> handle_responses(rest) - - {:error, websocket, reason} -> - put_in(state.websocket, websocket) - |> reply({:error, reason}) - end - end - - defp handle_responses(state, [_response | rest]) do - handle_responses(state, rest) - end - - defp handle_responses(state, []), do: state - - defp handle_frames(state, frames) do - {frames, state} = - Enum.flat_map_reduce(frames, state, fn - # prepare to close the connection when a close frame is received - {:close, _code, _data}, state -> - {[], put_in(state.closing?, true)} - - frame, state -> - {[frame], state} - end) - - Enum.each(frames, &Kernel.send(state.sender, &1)) - - state - end - - defp reply(state, response) do - if state.caller, do: GenServer.reply(state.caller, response) - put_in(state.caller, nil) - end - - # Encodes a frame as a binary and sends it along the wire, keeping `conn` - # and `websocket` up to date in `state`. - defp stream_frame(state, frame) do - with {:ok, websocket, data} <- Mint.WebSocket.encode(state.websocket, frame), - state = put_in(state.websocket, websocket), - {:ok, conn} <- Mint.WebSocket.stream_request_body(state.conn, state.request_ref, data) do - {:ok, put_in(state.conn, conn)} - else - {:error, %Mint.WebSocket{} = websocket, reason} -> - {:error, put_in(state.websocket, websocket), reason} - - {:error, conn, reason} -> - {:error, put_in(state.conn, conn), reason} - end + def websocket_terminate(_reason, _conn_state, _state) do + :ok end end From d19696cf60c6eb98ec751fb5d67f994cc39cd4b6 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 2 Sep 2022 22:58:35 -0400 Subject: [PATCH 122/123] Lint --- test/pleroma/web/activity_pub/side_effects_test.exs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 2cbb64e8f..6d6f9b2b6 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -544,9 +544,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do ]) do {:ok, announce, _} = SideEffects.handle(announce) - assert called( - Pleroma.Web.Streamer.stream(["user", "list"], announce) - ) + assert called(Pleroma.Web.Streamer.stream(["user", "list"], announce)) assert called(Pleroma.Web.Push.send(:_)) end From 4477c6baff6ea3c17ceca5d9113960b5b78d5ac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Tue, 23 Aug 2022 14:49:05 +0200 Subject: [PATCH 123/123] Metadata/Utils: use summary as description if set When generating OpenGraph and TwitterCard metadata for a post, the summary field will be used first if it is set to generate the post description. --- lib/pleroma/web/metadata/utils.ex | 15 ++++++-- .../metadata/providers/twitter_card_test.exs | 33 +++++++++++++++++ test/pleroma/web/metadata/utils_test.exs | 36 ++++++++++++++++++- 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex index 8052eaa44..15414a988 100644 --- a/lib/pleroma/web/metadata/utils.ex +++ b/lib/pleroma/web/metadata/utils.ex @@ -8,8 +8,8 @@ defmodule Pleroma.Web.Metadata.Utils do alias Pleroma.Formatter alias Pleroma.HTML - def scrub_html_and_truncate(%{data: %{"content" => content}} = object) do - content + defp scrub_html_and_truncate_object_field(field, object) do + field # html content comes from DB already encoded, decode first and scrub after |> HtmlEntities.decode() |> String.replace(~r//, " ") @@ -19,6 +19,17 @@ defmodule Pleroma.Web.Metadata.Utils do |> Formatter.truncate() end + def scrub_html_and_truncate(%{data: %{"summary" => summary}} = object) + when is_binary(summary) and summary != "" do + summary + |> scrub_html_and_truncate_object_field(object) + end + + def scrub_html_and_truncate(%{data: %{"content" => content}} = object) do + content + |> scrub_html_and_truncate_object_field(object) + end + def scrub_html_and_truncate(content, max_length \\ 200) when is_binary(content) do content |> scrub_html diff --git a/test/pleroma/web/metadata/providers/twitter_card_test.exs b/test/pleroma/web/metadata/providers/twitter_card_test.exs index 392496993..1a0cea9ce 100644 --- a/test/pleroma/web/metadata/providers/twitter_card_test.exs +++ b/test/pleroma/web/metadata/providers/twitter_card_test.exs @@ -39,6 +39,7 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCardTest do "actor" => user.ap_id, "tag" => [], "id" => "https://pleroma.gov/objects/whatever", + "summary" => "", "content" => "pleroma in a nutshell" } }) @@ -54,6 +55,36 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCardTest do ] == result end + test "it uses summary as description if post has one" do + user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994") + {:ok, activity} = CommonAPI.post(user, %{status: "HI"}) + + note = + insert(:note, %{ + data: %{ + "actor" => user.ap_id, + "tag" => [], + "id" => "https://pleroma.gov/objects/whatever", + "summary" => "Public service announcement on caffeine consumption", + "content" => "cofe" + } + }) + + result = TwitterCard.build_tags(%{object: note, user: user, activity_id: activity.id}) + + assert [ + {:meta, [property: "twitter:title", content: Utils.user_name_string(user)], []}, + {:meta, + [ + property: "twitter:description", + content: "Public service announcement on caffeine consumption" + ], []}, + {:meta, [property: "twitter:image", content: "http://localhost:4001/images/avi.png"], + []}, + {:meta, [property: "twitter:card", content: "summary"], []} + ] == result + end + test "it renders avatar not attachment if post is nsfw and unfurl_nsfw is disabled" do clear_config([Pleroma.Web.Metadata, :unfurl_nsfw], false) user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994") @@ -65,6 +96,7 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCardTest do "actor" => user.ap_id, "tag" => [], "id" => "https://pleroma.gov/objects/whatever", + "summary" => "", "content" => "pleroma in a nutshell", "sensitive" => true, "attachment" => [ @@ -109,6 +141,7 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCardTest do "actor" => user.ap_id, "tag" => [], "id" => "https://pleroma.gov/objects/whatever", + "summary" => "", "content" => "pleroma in a nutshell", "attachment" => [ %{ diff --git a/test/pleroma/web/metadata/utils_test.exs b/test/pleroma/web/metadata/utils_test.exs index 5f2f4a056..85ef6033a 100644 --- a/test/pleroma/web/metadata/utils_test.exs +++ b/test/pleroma/web/metadata/utils_test.exs @@ -8,7 +8,7 @@ defmodule Pleroma.Web.Metadata.UtilsTest do alias Pleroma.Web.Metadata.Utils describe "scrub_html_and_truncate/1" do - test "it returns text without encode HTML" do + test "it returns content text without encode HTML if summary is nil" do user = insert(:user) note = @@ -16,6 +16,7 @@ defmodule Pleroma.Web.Metadata.UtilsTest do data: %{ "actor" => user.ap_id, "id" => "https://pleroma.gov/objects/whatever", + "summary" => nil, "content" => "Pleroma's really cool!" } }) @@ -23,6 +24,39 @@ defmodule Pleroma.Web.Metadata.UtilsTest do assert Utils.scrub_html_and_truncate(note) == "Pleroma's really cool!" end + test "it returns context text without encode HTML if summary is empty" do + user = insert(:user) + + note = + insert(:note, %{ + data: %{ + "actor" => user.ap_id, + "id" => "https://pleroma.gov/objects/whatever", + "summary" => "", + "content" => "Pleroma's really cool!" + } + }) + + assert Utils.scrub_html_and_truncate(note) == "Pleroma's really cool!" + end + + test "it returns summary text without encode HTML if summary is filled" do + user = insert(:user) + + note = + insert(:note, %{ + data: %{ + "actor" => user.ap_id, + "id" => "https://pleroma.gov/objects/whatever", + "summary" => "Public service announcement on caffeine consumption", + "content" => "cofe" + } + }) + + assert Utils.scrub_html_and_truncate(note) == + "Public service announcement on caffeine consumption" + end + test "it does not return old content after editing" do user = insert(:user)