From 718e8e1edb537aca984216be39b3be5c8af4e6da Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sun, 16 May 2021 21:39:58 -0500
Subject: [PATCH 001/823] Create NsfwApiPolicy
---
config/config.exs | 7 +
.../web/activity_pub/mrf/nsfw_api_policy.ex | 185 ++++++++++++++++++
2 files changed, 192 insertions(+)
create mode 100644 lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
diff --git a/config/config.exs b/config/config.exs
index 66aee3264..d96dc1646 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -404,6 +404,13 @@ config :pleroma, :mrf_object_age,
threshold: 604_800,
actions: [:delist, :strip_followers]
+config :pleroma, :mrf_nsfw_api,
+ url: "http://127.0.0.1:5000/",
+ threshold: 0.7,
+ mark_sensitive: true,
+ unlist: false,
+ reject: false
+
config :pleroma, :rich_media,
enabled: true,
ignore_hosts: [],
diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
new file mode 100644
index 000000000..9ad175b1b
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
@@ -0,0 +1,185 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
+ @moduledoc """
+ Hide, delete, or mark sensitive NSFW content with artificial intelligence.
+
+ Requires a NSFW API server, configured like so:
+
+ config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF,
+ url: "http://127.0.0.1:5000/",
+ threshold: 0.8,
+ mark_sensitive: true,
+ unlist: false,
+ reject: false
+
+ The NSFW API server must implement an HTTP endpoint like this:
+
+ curl http://localhost:5000/?url=https://fedi.com/images/001.jpg
+
+ Returning a response like this:
+
+ {"score", 0.314}
+
+ Where a score is 0-1, with `1` being definitely NSFW.
+
+ A good API server is here: https://github.com/EugenCepoi/nsfw_api
+ You can run it with Docker with a one-liner:
+
+ docker run -it -p 127.0.0.1:5000:5000/tcp --env PORT=5000 eugencepoi/nsfw_api:latest
+
+ Options:
+
+ - `url`: Base URL of the API server. Default: "http://127.0.0.1:5000/"
+ - `threshold`: Lowest score to take action on. Default: `0.7`
+ - `mark_sensitive`: Mark sensitive all detected NSFW content? Default: `true`
+ - `unlist`: Unlist all detected NSFW content? Default: `false`
+ - `reject`: Reject all detected NSFW content (takes precedence)? Default: `false`
+ """
+ alias Pleroma.Config
+ alias Pleroma.Constants
+ alias Pleroma.HTTP
+ alias Pleroma.User
+
+ require Logger
+ require Pleroma.Constants
+
+ @behaviour Pleroma.Web.ActivityPub.MRF
+ @policy :mrf_nsfw_api
+
+ defp build_request_url(url) do
+ Config.get([@policy, :url])
+ |> URI.parse()
+ |> Map.put(:query, "url=#{url}")
+ |> URI.to_string()
+ end
+
+ defp parse_url(url) do
+ request = build_request_url(url)
+
+ with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do
+ Jason.decode(body)
+ else
+ error ->
+ Logger.warn("""
+ [NsfwApiPolicy]: The API server failed. Skipping.
+ #{inspect(error)}
+ """)
+
+ error
+ end
+ end
+
+ defp check_url_nsfw(url) when is_binary(url) do
+ threshold = Config.get([@policy, :threshold])
+
+ case parse_url(url) do
+ {:ok, %{"score" => score}} when score >= threshold ->
+ {:nsfw, %{url: url, score: score, threshold: threshold}}
+
+ _ ->
+ {:sfw, url}
+ end
+ end
+
+ defp check_url_nsfw(%{"href" => url}) when is_binary(url) do
+ check_url_nsfw(url)
+ end
+
+ defp check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do
+ if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do
+ {:sfw, attachment}
+ else
+ {:nsfw, attachment}
+ end
+ end
+
+ defp check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do
+ if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do
+ {:sfw, object}
+ else
+ {:nsfw, object}
+ end
+ end
+
+ defp check_object_nsfw(%{"object" => %{} = child_object} = object) do
+ case check_object_nsfw(child_object) do
+ {:sfw, _} -> {:sfw, object}
+ {:nsfw, _} -> {:nsfw, object}
+ end
+ end
+
+ defp check_object_nsfw(object), do: {:sfw, object}
+
+ @impl true
+ def filter(object) do
+ with {:sfw, object} <- check_object_nsfw(object) do
+ {:ok, object}
+ else
+ {:nsfw, _data} -> handle_nsfw(object)
+ _ -> {:reject, "NSFW: Attachment rejected"}
+ end
+ end
+
+ defp handle_nsfw(object) do
+ if Config.get([@policy, :reject]) do
+ {:reject, object}
+ else
+ {:ok,
+ object
+ |> maybe_unlist()
+ |> maybe_mark_sensitive()}
+ end
+ end
+
+ defp maybe_unlist(object) do
+ if Config.get([@policy, :unlist]) do
+ unlist(object)
+ else
+ object
+ end
+ end
+
+ defp maybe_mark_sensitive(object) do
+ if Config.get([@policy, :mark_sensitive]) do
+ mark_sensitive(object)
+ else
+ object
+ end
+ end
+
+ defp unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do
+ with %User{} = user <- User.get_cached_by_ap_id(actor) do
+ to =
+ [user.follower_address | to]
+ |> List.delete(Constants.as_public())
+ |> Enum.uniq()
+
+ cc =
+ [Constants.as_public() | cc]
+ |> List.delete(user.follower_address)
+ |> Enum.uniq()
+
+ object
+ |> Map.put("to", to)
+ |> Map.put("cc", cc)
+ end
+ end
+
+ defp mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do
+ Map.put(object, "object", mark_sensitive(child_object))
+ end
+
+ defp mark_sensitive(object) when is_map(object) do
+ tags = (object["tag"] || []) ++ ["nsfw"]
+
+ object
+ |> Map.put("tag", tags)
+ |> Map.put("sensitive", true)
+ end
+
+ @impl true
+ def describe, do: {:ok, %{}}
+end
From f15d419062b5f9aba2a2e84257dc2379b44f92e8 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Wed, 16 Jun 2021 22:30:18 -0500
Subject: [PATCH 002/823] NsfwApiPolicy: raise if can't fetch user
---
lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
index 9ad175b1b..63e6af0a0 100644
--- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
@@ -165,6 +165,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
object
|> Map.put("to", to)
|> Map.put("cc", cc)
+ else
+ _ -> raise "[NsfwApiPolicy]: Could not fetch user #{actor}"
end
end
From 2b3dfbb42f7ec0c5604876276a81d55a05955416 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Thu, 17 Jun 2021 14:36:51 -0500
Subject: [PATCH 003/823] NsfwApiPolicy: add tests
---
.../web/activity_pub/mrf/nsfw_api_policy.ex | 45 ++-
.../activity_pub/mrf/nsfw_api_policy_test.exs | 267 ++++++++++++++++++
2 files changed, 299 insertions(+), 13 deletions(-)
create mode 100644 test/pleroma/web/activity_pub/mrf/nsfw_api_policy_test.exs
diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
index 63e6af0a0..9dcdf560e 100644
--- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
@@ -49,14 +49,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
@behaviour Pleroma.Web.ActivityPub.MRF
@policy :mrf_nsfw_api
- defp build_request_url(url) do
+ def build_request_url(url) do
Config.get([@policy, :url])
|> URI.parse()
+ |> fix_path()
|> Map.put(:query, "url=#{url}")
|> URI.to_string()
end
- defp parse_url(url) do
+ def parse_url(url) do
request = build_request_url(url)
with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do
@@ -72,23 +73,26 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
end
end
- defp check_url_nsfw(url) when is_binary(url) do
+ def check_url_nsfw(url) when is_binary(url) do
threshold = Config.get([@policy, :threshold])
case parse_url(url) do
{:ok, %{"score" => score}} when score >= threshold ->
{:nsfw, %{url: url, score: score, threshold: threshold}}
+ {:ok, %{"score" => score}} ->
+ {:sfw, %{url: url, score: score, threshold: threshold}}
+
_ ->
- {:sfw, url}
+ {:sfw, %{url: url, score: nil, threshold: threshold}}
end
end
- defp check_url_nsfw(%{"href" => url}) when is_binary(url) do
+ def check_url_nsfw(%{"href" => url}) when is_binary(url) do
check_url_nsfw(url)
end
- defp check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do
+ def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do
if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do
{:sfw, attachment}
else
@@ -96,7 +100,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
end
end
- defp check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do
+ def check_attachment_nsfw(%{"url" => url} = attachment) when is_binary(url) do
+ case check_url_nsfw(url) do
+ {:sfw, _} -> {:sfw, attachment}
+ {:nsfw, _} -> {:nsfw, attachment}
+ end
+ end
+
+ def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do
if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do
{:sfw, object}
else
@@ -104,14 +115,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
end
end
- defp check_object_nsfw(%{"object" => %{} = child_object} = object) do
+ def check_object_nsfw(%{"object" => %{} = child_object} = object) do
case check_object_nsfw(child_object) do
{:sfw, _} -> {:sfw, object}
{:nsfw, _} -> {:nsfw, object}
end
end
- defp check_object_nsfw(object), do: {:sfw, object}
+ def check_object_nsfw(object), do: {:sfw, object}
@impl true
def filter(object) do
@@ -150,7 +161,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
end
end
- defp unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do
+ def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do
with %User{} = user <- User.get_cached_by_ap_id(actor) do
to =
[user.follower_address | to]
@@ -166,15 +177,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
|> Map.put("to", to)
|> Map.put("cc", cc)
else
- _ -> raise "[NsfwApiPolicy]: Could not fetch user #{actor}"
+ _ -> raise "[NsfwApiPolicy]: Could not find user #{actor}"
end
end
- defp mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do
+ def mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do
Map.put(object, "object", mark_sensitive(child_object))
end
- defp mark_sensitive(object) when is_map(object) do
+ def mark_sensitive(object) when is_map(object) do
tags = (object["tag"] || []) ++ ["nsfw"]
object
@@ -182,6 +193,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
|> Map.put("sensitive", true)
end
+ # Hackney needs a trailing slash
+ defp fix_path(%URI{path: path} = uri) when is_binary(path) do
+ path = String.trim_trailing(path, "/") <> "/"
+ Map.put(uri, :path, path)
+ end
+
+ defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/")
+
@impl true
def describe, do: {:ok, %{}}
end
diff --git a/test/pleroma/web/activity_pub/mrf/nsfw_api_policy_test.exs b/test/pleroma/web/activity_pub/mrf/nsfw_api_policy_test.exs
new file mode 100644
index 000000000..0beb9c2cb
--- /dev/null
+++ b/test/pleroma/web/activity_pub/mrf/nsfw_api_policy_test.exs
@@ -0,0 +1,267 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicyTest do
+ use Pleroma.DataCase
+
+ import ExUnit.CaptureLog
+ import Pleroma.Factory
+
+ alias Pleroma.Constants
+ alias Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy
+
+ require Pleroma.Constants
+
+ @policy :mrf_nsfw_api
+
+ @sfw_url "https://kittens.co/kitty.gif"
+ @nsfw_url "https://b00bies.com/nsfw.jpg"
+ @timeout_url "http://time.out/i.jpg"
+
+ setup_all do
+ clear_config(@policy,
+ url: "http://127.0.0.1:5000/",
+ threshold: 0.7,
+ mark_sensitive: true,
+ unlist: false,
+ reject: false
+ )
+ end
+
+ setup do
+ Tesla.Mock.mock(fn
+ # NSFW URL
+ %{method: :get, url: "http://127.0.0.1:5000/?url=#{@nsfw_url}"} ->
+ %Tesla.Env{status: 200, body: ~s({"score":0.99772077798843384,"url":"#{@nsfw_url}"})}
+
+ # SFW URL
+ %{method: :get, url: "http://127.0.0.1:5000/?url=#{@sfw_url}"} ->
+ %Tesla.Env{status: 200, body: ~s({"score":0.00011714912398019806,"url":"#{@sfw_url}"})}
+
+ # Timeout URL
+ %{method: :get, url: "http://127.0.0.1:5000/?url=#{@timeout_url}"} ->
+ {:error, :timeout}
+
+ # Fallback URL
+ %{method: :get, url: "http://127.0.0.1:5000/?url=" <> url} ->
+ body =
+ ~s({"error_code":500,"error_reason":"[Errno -2] Name or service not known","url":"#{url}"})
+
+ %Tesla.Env{status: 500, body: body}
+ end)
+
+ :ok
+ end
+
+ describe "build_request_url/1" do
+ test "it works" do
+ expected = "http://127.0.0.1:5000/?url=https://b00bies.com/nsfw.jpg"
+ assert NsfwApiPolicy.build_request_url(@nsfw_url) == expected
+ end
+
+ test "it adds a trailing slash" do
+ clear_config([@policy, :url], "http://localhost:5000")
+
+ expected = "http://localhost:5000/?url=https://b00bies.com/nsfw.jpg"
+ assert NsfwApiPolicy.build_request_url(@nsfw_url) == expected
+ end
+
+ test "it adds a trailing slash preserving the path" do
+ clear_config([@policy, :url], "http://localhost:5000/nsfw_api")
+
+ expected = "http://localhost:5000/nsfw_api/?url=https://b00bies.com/nsfw.jpg"
+ assert NsfwApiPolicy.build_request_url(@nsfw_url) == expected
+ end
+ end
+
+ describe "parse_url/1" do
+ test "returns decoded JSON from the API server" do
+ expected = %{"score" => 0.99772077798843384, "url" => @nsfw_url}
+ assert NsfwApiPolicy.parse_url(@nsfw_url) == {:ok, expected}
+ end
+
+ test "warns when the API server fails" do
+ expected = "[NsfwApiPolicy]: The API server failed. Skipping."
+ assert capture_log(fn -> NsfwApiPolicy.parse_url(@timeout_url) end) =~ expected
+ end
+
+ test "returns {:error, _} tuple when the API server fails" do
+ capture_log(fn ->
+ assert {:error, _} = NsfwApiPolicy.parse_url(@timeout_url)
+ end)
+ end
+ end
+
+ describe "check_url_nsfw/1" do
+ test "returns {:nsfw, _} tuple" do
+ expected = {:nsfw, %{url: @nsfw_url, score: 0.99772077798843384, threshold: 0.7}}
+ assert NsfwApiPolicy.check_url_nsfw(@nsfw_url) == expected
+ end
+
+ test "returns {:sfw, _} tuple" do
+ expected = {:sfw, %{url: @sfw_url, score: 0.00011714912398019806, threshold: 0.7}}
+ assert NsfwApiPolicy.check_url_nsfw(@sfw_url) == expected
+ end
+
+ test "returns {:sfw, _} on failure" do
+ expected = {:sfw, %{url: @timeout_url, score: nil, threshold: 0.7}}
+
+ capture_log(fn ->
+ assert NsfwApiPolicy.check_url_nsfw(@timeout_url) == expected
+ end)
+ end
+
+ test "works with map URL" do
+ expected = {:nsfw, %{url: @nsfw_url, score: 0.99772077798843384, threshold: 0.7}}
+ assert NsfwApiPolicy.check_url_nsfw(%{"href" => @nsfw_url}) == expected
+ end
+ end
+
+ describe "check_attachment_nsfw/1" do
+ test "returns {:nsfw, _} if any items are NSFW" do
+ attachment = %{"url" => [%{"href" => @nsfw_url}, @nsfw_url, @sfw_url]}
+ assert NsfwApiPolicy.check_attachment_nsfw(attachment) == {:nsfw, attachment}
+ end
+
+ test "returns {:sfw, _} if all items are SFW" do
+ attachment = %{"url" => [%{"href" => @sfw_url}, @sfw_url, @sfw_url]}
+ assert NsfwApiPolicy.check_attachment_nsfw(attachment) == {:sfw, attachment}
+ end
+
+ test "works with binary URL" do
+ attachment = %{"url" => @nsfw_url}
+ assert NsfwApiPolicy.check_attachment_nsfw(attachment) == {:nsfw, attachment}
+ end
+ end
+
+ describe "check_object_nsfw/1" do
+ test "returns {:nsfw, _} if any items are NSFW" do
+ object = %{"attachment" => [%{"url" => [%{"href" => @nsfw_url}, @sfw_url]}]}
+ assert NsfwApiPolicy.check_object_nsfw(object) == {:nsfw, object}
+ end
+
+ test "returns {:sfw, _} if all items are SFW" do
+ object = %{"attachment" => [%{"url" => [%{"href" => @sfw_url}, @sfw_url]}]}
+ assert NsfwApiPolicy.check_object_nsfw(object) == {:sfw, object}
+ end
+
+ test "works with embedded object" do
+ object = %{"object" => %{"attachment" => [%{"url" => [%{"href" => @nsfw_url}, @sfw_url]}]}}
+ assert NsfwApiPolicy.check_object_nsfw(object) == {:nsfw, object}
+ end
+ end
+
+ describe "unlist/1" do
+ test "unlist addressing" do
+ user = insert(:user)
+
+ object = %{
+ "to" => [Constants.as_public()],
+ "cc" => [user.follower_address, "https://hello.world/users/alex"],
+ "actor" => user.ap_id
+ }
+
+ expected = %{
+ "to" => [user.follower_address],
+ "cc" => [Constants.as_public(), "https://hello.world/users/alex"],
+ "actor" => user.ap_id
+ }
+
+ assert NsfwApiPolicy.unlist(object) == expected
+ end
+
+ test "raise if user isn't found" do
+ object = %{
+ "to" => [Constants.as_public()],
+ "cc" => [],
+ "actor" => "https://hello.world/users/alex"
+ }
+
+ assert_raise(RuntimeError, fn ->
+ NsfwApiPolicy.unlist(object)
+ end)
+ end
+ end
+
+ describe "mark_sensitive/1" do
+ test "adds nsfw tag and marks sensitive" do
+ object = %{"tag" => ["yolo"]}
+ expected = %{"tag" => ["yolo", "nsfw"], "sensitive" => true}
+ assert NsfwApiPolicy.mark_sensitive(object) == expected
+ end
+
+ test "works with embedded object" do
+ object = %{"object" => %{"tag" => ["yolo"]}}
+ expected = %{"object" => %{"tag" => ["yolo", "nsfw"], "sensitive" => true}}
+ assert NsfwApiPolicy.mark_sensitive(object) == expected
+ end
+ end
+
+ describe "filter/1" do
+ setup do
+ user = insert(:user)
+
+ nsfw_object = %{
+ "to" => [Constants.as_public()],
+ "cc" => [user.follower_address],
+ "actor" => user.ap_id,
+ "attachment" => [%{"url" => @nsfw_url}]
+ }
+
+ sfw_object = %{
+ "to" => [Constants.as_public()],
+ "cc" => [user.follower_address],
+ "actor" => user.ap_id,
+ "attachment" => [%{"url" => @sfw_url}]
+ }
+
+ %{user: user, nsfw_object: nsfw_object, sfw_object: sfw_object}
+ end
+
+ test "passes SFW object through", %{sfw_object: object} do
+ {:ok, _} = NsfwApiPolicy.filter(object)
+ end
+
+ test "passes NSFW object through when actions are disabled", %{nsfw_object: object} do
+ clear_config([@policy, :mark_sensitive], false)
+ clear_config([@policy, :unlist], false)
+ clear_config([@policy, :reject], false)
+ {:ok, _} = NsfwApiPolicy.filter(object)
+ end
+
+ test "passes NSFW object through when :threshold is 1", %{nsfw_object: object} do
+ clear_config([@policy, :reject], true)
+ clear_config([@policy, :threshold], 1)
+ {:ok, _} = NsfwApiPolicy.filter(object)
+ end
+
+ test "rejects SFW object through when :threshold is 0", %{sfw_object: object} do
+ clear_config([@policy, :reject], true)
+ clear_config([@policy, :threshold], 0)
+ {:reject, _} = NsfwApiPolicy.filter(object)
+ end
+
+ test "rejects NSFW when :reject is enabled", %{nsfw_object: object} do
+ clear_config([@policy, :reject], true)
+ {:reject, _} = NsfwApiPolicy.filter(object)
+ end
+
+ test "passes NSFW through when :reject is disabled", %{nsfw_object: object} do
+ clear_config([@policy, :reject], false)
+ {:ok, _} = NsfwApiPolicy.filter(object)
+ end
+
+ test "unlists NSFW when :unlist is enabled", %{user: user, nsfw_object: object} do
+ clear_config([@policy, :unlist], true)
+ {:ok, object} = NsfwApiPolicy.filter(object)
+ assert object["to"] == [user.follower_address]
+ end
+
+ test "passes NSFW through when :unlist is disabled", %{nsfw_object: object} do
+ clear_config([@policy, :unlist], false)
+ {:ok, object} = NsfwApiPolicy.filter(object)
+ assert object["to"] == [Constants.as_public()]
+ end
+ end
+end
From b293c14a1b01398029dfa80aea306946efc2f284 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Thu, 17 Jun 2021 14:52:07 -0500
Subject: [PATCH 004/823] NsfwApiPolicy: add describe/0 and
config_description/0
---
.../web/activity_pub/mrf/nsfw_api_policy.ex | 56 ++++++++++++++++++-
1 file changed, 54 insertions(+), 2 deletions(-)
diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
index 9dcdf560e..a1560c584 100644
--- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF,
url: "http://127.0.0.1:5000/",
- threshold: 0.8,
+ threshold: 0.7,
mark_sensitive: true,
unlist: false,
reject: false
@@ -202,5 +202,57 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/")
@impl true
- def describe, do: {:ok, %{}}
+ def describe do
+ options = %{
+ threshold: Config.get([@policy, :threshold]),
+ mark_sensitive: Config.get([@policy, :mark_sensitive]),
+ unlist: Config.get([@policy, :unlist]),
+ reject: Config.get([@policy, :reject])
+ }
+
+ {:ok, %{@policy => options}}
+ end
+
+ @impl true
+ def config_description do
+ %{
+ key: @policy,
+ related_policy: to_string(__MODULE__),
+ label: "NSFW API Policy",
+ description:
+ "Hide, delete, or mark sensitive NSFW content with artificial intelligence. Requires running an external API server.",
+ children: [
+ %{
+ key: :url,
+ type: :string,
+ description: "Base URL of the API server.",
+ suggestions: ["http://127.0.0.1:5000/"]
+ },
+ %{
+ key: :threshold,
+ type: :float,
+ description: "Lowest score to take action on. Between 0 and 1.",
+ suggestions: [0.7]
+ },
+ %{
+ key: :mark_sensitive,
+ type: :boolean,
+ description: "Mark sensitive all detected NSFW content?",
+ suggestions: [true]
+ },
+ %{
+ key: :unlist,
+ type: :boolean,
+ description: "Unlist sensitive all detected NSFW content?",
+ suggestions: [false]
+ },
+ %{
+ key: :reject,
+ type: :boolean,
+ description: "Reject sensitive all detected NSFW content (takes precedence)?",
+ suggestions: [false]
+ }
+ ]
+ }
+ end
end
From c802c3055ef6c1f763d5df68f9e5308093f7d565 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Thu, 17 Jun 2021 15:04:40 -0500
Subject: [PATCH 005/823] NsfwApiPolicy: add systemd example file
---
installation/nsfw-api.service | 15 +++++++++++++++
1 file changed, 15 insertions(+)
create mode 100644 installation/nsfw-api.service
diff --git a/installation/nsfw-api.service b/installation/nsfw-api.service
new file mode 100644
index 000000000..ec629df67
--- /dev/null
+++ b/installation/nsfw-api.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=NSFW API
+After=docker.service
+Requires=docker.service
+
+[Service]
+TimeoutStartSec=0
+Restart=always
+ExecStartPre=-/usr/bin/docker stop %n
+ExecStartPre=-/usr/bin/docker rm %n
+ExecStartPre=/usr/bin/docker pull eugencepoi/nsfw_api:latest
+ExecStart=/usr/bin/docker run --rm -p 127.0.0.1:5000:5000/tcp --env PORT=5000 --name %n eugencepoi/nsfw_api:latest
+
+[Install]
+WantedBy=multi-user.target
From a704d5499c03cb5609ea38a5f2ef06095ced3ef3 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Thu, 17 Jun 2021 15:32:42 -0500
Subject: [PATCH 006/823] NsfwApiPolicy: Fall back more generously when
functions don't match
---
lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
index a1560c584..920821f38 100644
--- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
@@ -92,6 +92,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
check_url_nsfw(url)
end
+ def check_url_nsfw(url) do
+ threshold = Config.get([@policy, :threshold])
+ {:sfw, %{url: url, score: nil, threshold: threshold}}
+ end
+
def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do
if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do
{:sfw, attachment}
@@ -107,6 +112,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
end
end
+ def check_attachment_nsfw(attachment), do: {:sfw, attachment}
+
def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do
if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do
{:sfw, object}
From 9423052e9217aa1358950d37c5c96b11d554b37a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Mon, 25 Apr 2022 12:39:36 +0200
Subject: [PATCH 007/823] Add "status" notification type
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
lib/pleroma/notification.ex | 38 ++++++++++++--
.../operations/notification_operation.ex | 4 +-
.../controllers/notification_controller.ex | 1 +
.../mastodon_api/views/notification_view.ex | 3 ++
lib/pleroma/web/push/impl.ex | 1 +
...00000_add_status_to_notifications_enum.exs | 50 +++++++++++++++++++
test/pleroma/notification_test.exs | 1 +
7 files changed, 92 insertions(+), 6 deletions(-)
create mode 100644 priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs
diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 52fd2656b..d142baa8b 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -73,6 +73,7 @@ defmodule Pleroma.Notification do
pleroma:report
reblog
poll
+ status
}
def changeset(%Notification{} = notification, attrs) do
@@ -397,11 +398,18 @@ defmodule Pleroma.Notification do
{enabled_receivers, disabled_receivers} = get_notified_from_activity(activity)
potential_receivers = enabled_receivers ++ disabled_receivers
+ {enabled_subscribers, disabled_subscribers} = get_notified_subscribers_from_activity(activity)
+ potential_subscribers = (enabled_subscribers ++ disabled_subscribers) -- potential_receivers
+
notifications =
- Enum.map(potential_receivers, fn user ->
- do_send = do_send && user in enabled_receivers
- create_notification(activity, user, do_send: do_send)
- end)
+ (Enum.map(potential_receivers, fn user ->
+ do_send = do_send && user in enabled_receivers
+ create_notification(activity, user, do_send: do_send)
+ end) ++
+ Enum.map(potential_subscribers, fn user ->
+ do_send = do_send && user in enabled_subscribers
+ create_notification(activity, user, do_send: do_send, type: "status")
+ end))
|> Enum.reject(&is_nil/1)
{:ok, notifications}
@@ -533,6 +541,27 @@ defmodule Pleroma.Notification do
def get_notified_from_activity(_, _local_only), do: {[], []}
+ def get_notified_subscribers_from_activity(activity, local_only \\ true)
+
+ def get_notified_subscribers_from_activity(
+ %Activity{data: %{"type" => "Create"}} = activity,
+ local_only
+ ) do
+ notification_enabled_ap_ids =
+ []
+ |> Utils.maybe_notify_subscribers(activity)
+
+ potential_receivers =
+ User.get_users_from_set(notification_enabled_ap_ids, local_only: local_only)
+
+ notification_enabled_users =
+ Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
+
+ {notification_enabled_users, potential_receivers -- notification_enabled_users}
+ end
+
+ def get_notified_subscribers_from_activity(_, _), do: {[], []}
+
# For some activities, only notify the author of the object
def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}})
when type in ~w{Like Announce EmojiReact} do
@@ -557,7 +586,6 @@ defmodule Pleroma.Notification do
[]
|> Utils.maybe_notify_to_recipients(activity)
|> Utils.maybe_notify_mentioned_recipients(activity)
- |> Utils.maybe_notify_subscribers(activity)
|> Utils.maybe_notify_followers(activity)
|> Enum.uniq()
end
diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex
index 7f2336ff6..aa965fabb 100644
--- a/lib/pleroma/web/api_spec/operations/notification_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex
@@ -196,7 +196,8 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
"pleroma:report",
"move",
"follow_request",
- "poll"
+ "poll",
+ "status"
],
description: """
The type of event that resulted in the notification.
@@ -210,6 +211,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
- `pleroma:emoji_reaction` - Someone reacted with emoji to your status
- `pleroma:chat_mention` - Someone mentioned you in a chat message
- `pleroma:report` - Someone was reported
+ - `status` - Someone you are subscribed to created a status
"""
}
end
diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex
index 932bc6423..9209e8ebd 100644
--- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex
@@ -51,6 +51,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do
move
pleroma:emoji_reaction
poll
+ status
}
def index(%{assigns: %{user: user}} = conn, params) do
params =
diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex
index 0dc7f3beb..b10b0893c 100644
--- a/lib/pleroma/web/mastodon_api/views/notification_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex
@@ -103,6 +103,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
"mention" ->
put_status(response, activity, reading_user, status_render_opts)
+ "status" ->
+ put_status(response, activity, reading_user, status_render_opts)
+
"favourite" ->
put_status(response, parent_activity_fn.(), reading_user, status_render_opts)
diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index daf3eeb9e..77bc2941d 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -183,6 +183,7 @@ defmodule Pleroma.Web.Push.Impl do
def format_title(%{type: type}, mastodon_type) do
case mastodon_type || type do
"mention" -> "New Mention"
+ "status" -> "New Status"
"follow" -> "New Follower"
"follow_request" -> "New Follow Request"
"reblog" -> "New Repeat"
diff --git a/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs b/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs
new file mode 100644
index 000000000..62c0afb63
--- /dev/null
+++ b/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs
@@ -0,0 +1,50 @@
+defmodule Pleroma.Repo.Migrations.AddStatusToNotificationsEnum do
+ use Ecto.Migration
+
+ @disable_ddl_transaction true
+
+ def up do
+ """
+ alter type notification_type add value 'status'
+ """
+ |> execute()
+ end
+
+ def down do
+ alter table(:notifications) do
+ modify(:type, :string)
+ end
+
+ """
+ delete from notifications where type = 'status'
+ """
+ |> execute()
+
+ """
+ drop type if exists notification_type
+ """
+ |> execute()
+
+ """
+ create type notification_type as enum (
+ 'follow',
+ 'follow_request',
+ 'mention',
+ 'move',
+ 'pleroma:emoji_reaction',
+ 'pleroma:chat_mention',
+ 'reblog',
+ 'favourite',
+ 'pleroma:report',
+ 'poll
+ )
+ """
+ |> execute()
+
+ """
+ alter table notifications
+ alter column type type notification_type using (type::notification_type)
+ """
+ |> execute()
+ end
+end
diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs
index 805764ea4..eea2fcb67 100644
--- a/test/pleroma/notification_test.exs
+++ b/test/pleroma/notification_test.exs
@@ -104,6 +104,7 @@ defmodule Pleroma.NotificationTest do
{:ok, [notification]} = Notification.create_notifications(status)
assert notification.user_id == subscriber.id
+ assert notification.type == "status"
end
test "does not create a notification for subscribed users if status is a reply" do
From bd52e2aec7f07da3bc3609f72f7f1bf5969c9baf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Sat, 5 Feb 2022 18:03:53 +0100
Subject: [PATCH 008/823] Instance rules
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
lib/pleroma/rule.ex | 56 +++++++++
.../admin_api/controllers/rule_controller.ex | 64 ++++++++++
lib/pleroma/web/admin_api/views/rule_view.ex | 21 ++++
.../operations/admin/rule_operation.ex | 113 ++++++++++++++++++
.../web/mastodon_api/views/instance_view.ex | 7 ++
lib/pleroma/web/router.ex | 5 +
.../20220203224011_create_rules.exs | 12 ++
.../controllers/rule_controller_test.exs | 78 ++++++++++++
.../controllers/instance_controller_test.exs | 18 ++-
9 files changed, 373 insertions(+), 1 deletion(-)
create mode 100644 lib/pleroma/rule.ex
create mode 100644 lib/pleroma/web/admin_api/controllers/rule_controller.ex
create mode 100644 lib/pleroma/web/admin_api/views/rule_view.ex
create mode 100644 lib/pleroma/web/api_spec/operations/admin/rule_operation.ex
create mode 100644 priv/repo/migrations/20220203224011_create_rules.exs
create mode 100644 test/pleroma/web/admin_api/controllers/rule_controller_test.exs
diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex
new file mode 100644
index 000000000..d772a32bd
--- /dev/null
+++ b/lib/pleroma/rule.ex
@@ -0,0 +1,56 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Rule do
+ use Ecto.Schema
+
+ import Ecto.Changeset
+ import Ecto.Query
+
+ alias Pleroma.Repo
+ alias Pleroma.Rule
+
+ schema "rules" do
+ field(:priority, :integer, default: 0)
+ field(:text, :string)
+
+ timestamps()
+ end
+
+ def changeset(%Rule{} = rule, params \\ %{}) do
+ rule
+ |> cast(params, [:priority, :text])
+ |> validate_required([:text])
+ end
+
+ def query do
+ Rule
+ |> order_by(asc: :priority)
+ end
+
+ def get(id), do: Repo.get(__MODULE__, id)
+
+ def create(params) do
+ {:ok, rule} =
+ %Rule{}
+ |> changeset(params)
+ |> Repo.insert()
+
+ rule
+ end
+
+ def update(params, id) do
+ {:ok, rule} =
+ get(id)
+ |> changeset(params)
+ |> Repo.update()
+
+ rule
+ end
+
+ def delete(id) do
+ get(id)
+ |> Repo.delete()
+ end
+end
diff --git a/lib/pleroma/web/admin_api/controllers/rule_controller.ex b/lib/pleroma/web/admin_api/controllers/rule_controller.ex
new file mode 100644
index 000000000..2db88b6ba
--- /dev/null
+++ b/lib/pleroma/web/admin_api/controllers/rule_controller.ex
@@ -0,0 +1,64 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.RuleController do
+ use Pleroma.Web, :controller
+
+ alias Pleroma.Repo
+ alias Pleroma.Rule
+ alias Pleroma.Web.Plugs.OAuthScopesPlug
+
+ import Pleroma.Web.ControllerHelper,
+ only: [
+ json_response: 3
+ ]
+
+ require Logger
+
+ plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+ plug(
+ OAuthScopesPlug,
+ %{scopes: ["admin:write"]}
+ when action in [:create, :update, :delete]
+ )
+
+ plug(OAuthScopesPlug, %{scopes: ["admin:read"]} when action == :index)
+
+ action_fallback(AdminAPI.FallbackController)
+
+ defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.RuleOperation
+
+ def index(conn, _) do
+ rules =
+ Rule.query()
+ |> Repo.all()
+
+ render(conn, "index.json", rules: rules)
+ end
+
+ def create(%{body_params: params} = conn, _) do
+ rule =
+ params
+ |> Rule.create()
+
+ render(conn, "show.json", rule: rule)
+ end
+
+ def update(%{body_params: params} = conn, %{id: id}) do
+ rule =
+ params
+ |> Rule.update(id)
+
+ render(conn, "show.json", rule: rule)
+ end
+
+ def delete(conn, %{id: id}) do
+ with {:ok, _} <- Rule.delete(id) do
+ json(conn, %{})
+ else
+ _ -> json_response(conn, :bad_request, "")
+ end
+ end
+end
diff --git a/lib/pleroma/web/admin_api/views/rule_view.ex b/lib/pleroma/web/admin_api/views/rule_view.ex
new file mode 100644
index 000000000..f29145248
--- /dev/null
+++ b/lib/pleroma/web/admin_api/views/rule_view.ex
@@ -0,0 +1,21 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.RuleView do
+ use Pleroma.Web, :view
+
+ require Pleroma.Constants
+
+ def render("index.json", %{rules: rules} = _opts) do
+ render_many(rules, __MODULE__, "show.json")
+ end
+
+ def render("show.json", %{rule: rule} = _opts) do
+ %{
+ id: rule.id,
+ priority: rule.priority,
+ text: rule.text
+ }
+ end
+end
diff --git a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex
new file mode 100644
index 000000000..28f2be5e7
--- /dev/null
+++ b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex
@@ -0,0 +1,113 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do
+ alias OpenApiSpex.Operation
+ alias OpenApiSpex.Schema
+ alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+ import Pleroma.Web.ApiSpec.Helpers
+
+ def open_api_operation(action) do
+ operation = String.to_existing_atom("#{action}_operation")
+ apply(__MODULE__, operation, [])
+ end
+
+ def index_operation do
+ %Operation{
+ tags: ["Instance rule managment"],
+ summary: "Retrieve a list of instance rules",
+ operationId: "AdminAPI.RuleController.index",
+ security: [%{"oAuth" => ["admin:read"]}],
+ responses: %{
+ 200 =>
+ Operation.response("Response", "application/json", %Schema{
+ type: :array,
+ items: rule()
+ }),
+ 403 => Operation.response("Forbidden", "application/json", ApiError)
+ }
+ }
+ end
+
+ def create_operation do
+ %Operation{
+ tags: ["Instance rule managment"],
+ summary: "Create new rule",
+ operationId: "AdminAPI.RuleController.create",
+ security: [%{"oAuth" => ["admin:write"]}],
+ parameters: admin_api_params(),
+ requestBody: request_body("Parameters", create_request(), required: true),
+ responses: %{
+ 200 => Operation.response("Response", "application/json", rule()),
+ 400 => Operation.response("Bad Request", "application/json", ApiError),
+ 403 => Operation.response("Forbidden", "application/json", ApiError)
+ }
+ }
+ end
+
+ def update_operation do
+ %Operation{
+ tags: ["Instance rule managment"],
+ summary: "Modify existing rule",
+ operationId: "AdminAPI.RuleController.update",
+ security: [%{"oAuth" => ["admin:write"]}],
+ parameters: [Operation.parameter(:id, :path, :string, "Rule ID")],
+ requestBody: request_body("Parameters", update_request(), required: true),
+ responses: %{
+ 200 => Operation.response("Response", "application/json", rule()),
+ 400 => Operation.response("Bad Request", "application/json", ApiError),
+ 403 => Operation.response("Forbidden", "application/json", ApiError)
+ }
+ }
+ end
+
+ def delete_operation do
+ %Operation{
+ tags: ["Instance rule managment"],
+ summary: "Delete rule",
+ operationId: "AdminAPI.RuleController.delete",
+ parameters: [Operation.parameter(:id, :path, :string, "Rule ID")],
+ security: [%{"oAuth" => ["admin:write"]}],
+ responses: %{
+ 200 => empty_object_response(),
+ 404 => Operation.response("Not Found", "application/json", ApiError),
+ 403 => Operation.response("Forbidden", "application/json", ApiError)
+ }
+ }
+ end
+
+ defp create_request do
+ %Schema{
+ type: :object,
+ required: [:text],
+ properties: %{
+ priority: %Schema{type: :integer},
+ text: %Schema{type: :string}
+ }
+ }
+ end
+
+ defp update_request do
+ %Schema{
+ type: :object,
+ properties: %{
+ priority: %Schema{type: :integer},
+ text: %Schema{type: :string}
+ }
+ }
+ end
+
+ defp rule do
+ %Schema{
+ type: :object,
+ properties: %{
+ id: %Schema{type: :integer},
+ priority: %Schema{type: :integer},
+ text: %Schema{type: :string},
+ created_at: %Schema{type: :string, format: :"date-time"}
+ }
+ }
+ end
+end
diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index ee52475d5..9652da53a 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -40,6 +40,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image),
shout_limit: Config.get([:shout, :limit]),
description_limit: Keyword.get(instance, :description_limit),
+ rules: rules(),
pleroma: %{
metadata: %{
account_activation_required: Keyword.get(instance, :account_activation_required),
@@ -137,4 +138,10 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
value_length: Config.get([:instance, :account_field_value_length])
}
end
+
+ def rules do
+ Pleroma.Rule.query()
+ |> Pleroma.Repo.all()
+ |> Enum.map(&%{id: &1.id, text: &1.text})
+ end
end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index ceb6c3cfd..e8b7b17aa 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -229,6 +229,11 @@ defmodule Pleroma.Web.Router do
post("/frontends/install", FrontendController, :install)
post("/backups", AdminAPIController, :create_backup)
+
+ get("/rules", RuleController, :index)
+ post("/rules", RuleController, :create)
+ patch("/rules/:id", RuleController, :update)
+ delete("/rules/:id", RuleController, :delete)
end
# AdminAPI: admins and mods (staff) can perform these actions (if enabled by config)
diff --git a/priv/repo/migrations/20220203224011_create_rules.exs b/priv/repo/migrations/20220203224011_create_rules.exs
new file mode 100644
index 000000000..16f29ca53
--- /dev/null
+++ b/priv/repo/migrations/20220203224011_create_rules.exs
@@ -0,0 +1,12 @@
+defmodule Pleroma.Repo.Migrations.CreateRules do
+ use Ecto.Migration
+
+ def change do
+ create_if_not_exists table(:rules) do
+ add(:priority, :integer, default: 0, null: false)
+ add(:text, :text, null: false)
+
+ timestamps()
+ end
+ end
+end
diff --git a/test/pleroma/web/admin_api/controllers/rule_controller_test.exs b/test/pleroma/web/admin_api/controllers/rule_controller_test.exs
new file mode 100644
index 000000000..c5c72d293
--- /dev/null
+++ b/test/pleroma/web/admin_api/controllers/rule_controller_test.exs
@@ -0,0 +1,78 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.RuleControllerTest do
+ use Pleroma.Web.ConnCase, async: true
+
+ import Pleroma.Factory
+
+ alias Pleroma.Rule
+
+ setup do
+ admin = insert(:user, is_admin: true)
+ token = insert(:oauth_admin_token, user: admin)
+
+ conn =
+ build_conn()
+ |> assign(:user, admin)
+ |> assign(:token, token)
+
+ {:ok, %{admin: admin, token: token, conn: conn}}
+ end
+
+ describe "GET /api/pleroma/admin/rules" do
+ test "sorts rules by priority", %{conn: conn} do
+ %{id: id1} = Rule.create(%{text: "Example rule"})
+ %{id: id2} = Rule.create(%{text: "Second rule", priority: 2})
+ %{id: id3} = Rule.create(%{text: "Third rule", priority: 1})
+
+ response =
+ conn
+ |> get("/api/pleroma/admin/rules")
+ |> json_response_and_validate_schema(:ok)
+
+ assert [%{"id" => ^id1}, %{"id" => ^id3}, %{"id" => ^id2}] = response
+ end
+ end
+
+ describe "POST /api/pleroma/admin/rules" do
+ test "creates a rule", %{conn: conn} do
+ %{"id" => id} =
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/pleroma/admin/rules", %{text: "Example rule"})
+ |> json_response_and_validate_schema(:ok)
+
+ assert %{text: "Example rule"} = Rule.get(id)
+ end
+ end
+
+ describe "PATCH /api/pleroma/admin/rules" do
+ test "edits a rule", %{conn: conn} do
+ %{id: id} = Rule.create(%{text: "Example rule"})
+
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> patch("/api/pleroma/admin/rules/#{id}", %{text: "There are no rules", priority: 2})
+ |> json_response_and_validate_schema(:ok)
+
+ assert %{text: "There are no rules", priority: 2} = Rule.get(id)
+ end
+ end
+
+ describe "DELETE /api/pleroma/admin/rules" do
+ test "deletes a rule", %{conn: conn} do
+ %{id: id} = Rule.create(%{text: "Example rule"})
+
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> delete("/api/pleroma/admin/rules/#{id}")
+ |> json_response_and_validate_schema(:ok)
+
+ assert [] =
+ Rule.query()
+ |> Pleroma.Repo.all()
+ end
+ end
+end
diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs
index 9845408d6..87beacd64 100644
--- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do
# TODO: Should not need Cachex
use Pleroma.Web.ConnCase
+ alias Pleroma.Rule
alias Pleroma.User
import Pleroma.Factory
@@ -39,7 +40,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do
"banner_upload_limit" => _,
"background_image" => from_config_background,
"shout_limit" => _,
- "description_limit" => _
+ "description_limit" => _,
+ "rules" => _
} = result
assert result["pleroma"]["metadata"]["account_activation_required"] != nil
@@ -91,4 +93,18 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do
assert ["peer1.com", "peer2.com"] == Enum.sort(result)
end
+
+ test "get instance rules", %{conn: conn} do
+ Rule.create(%{text: "Example rule"})
+ Rule.create(%{text: "Second rule"})
+ Rule.create(%{text: "Third rule"})
+
+ conn = get(conn, "/api/v1/instance")
+
+ assert result = json_response_and_validate_schema(conn, 200)
+
+ rules = result["rules"]
+
+ assert length(rules) == 3
+ end
end
From 432599311d3aab8829ed2cc7f795a1662e7a8f82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Sat, 5 Feb 2022 19:13:30 +0100
Subject: [PATCH 009/823] Add GET /api/v1/instance/rules
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
.../operations/admin/rule_operation.ex | 2 +-
.../api_spec/operations/instance_operation.ex | 27 ++++++++++++++++++-
.../controllers/instance_controller.ex | 5 ++++
.../web/mastodon_api/views/instance_view.ex | 4 +++
lib/pleroma/web/router.ex | 1 +
5 files changed, 37 insertions(+), 2 deletions(-)
diff --git a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex
index 28f2be5e7..ed0d9eaf6 100644
--- a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex
@@ -17,7 +17,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do
def index_operation do
%Operation{
tags: ["Instance rule managment"],
- summary: "Retrieve a list of instance rules",
+ summary: "Retrieve list of instance rules",
operationId: "AdminAPI.RuleController.index",
security: [%{"oAuth" => ["admin:read"]}],
responses: %{
diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex
index 3c4b504fe..e66e5b7a3 100644
--- a/lib/pleroma/web/api_spec/operations/instance_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex
@@ -34,6 +34,17 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
}
end
+ def rules_operation do
+ %Operation{
+ tags: ["Instance"],
+ summary: "Retrieve list of instance rules",
+ operationId: "InstanceController.rules",
+ responses: %{
+ 200 => Operation.response("Array of domains", "application/json", array_of_rules())
+ }
+ }
+ end
+
defp instance do
%Schema{
type: :object,
@@ -160,7 +171,8 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
"urls" => %{
"streaming_api" => "wss://lain.com"
},
- "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)"
+ "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)",
+ "rules" => array_of_rules()
}
}
end
@@ -172,4 +184,17 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
example: ["pleroma.site", "lain.com", "bikeshed.party"]
}
end
+
+ defp array_of_rules do
+ %Schema{
+ type: :array,
+ items: %Schema{
+ type: :object,
+ properties: %{
+ id: %Schema{type: :integer},
+ text: %Schema{type: :string}
+ }
+ }
+ }
+ end
end
diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex
index 6410e872c..d6aa89432 100644
--- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex
@@ -20,4 +20,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do
def peers(conn, _params) do
json(conn, Pleroma.Stats.get_peers())
end
+
+ @doc "GET /api/v1/instance/rules"
+ def rules(conn, _params) do
+ render(conn, "rules.json")
+ end
end
diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index 9652da53a..feb94b40b 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -58,6 +58,10 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
}
end
+ def render("rules.json", _) do
+ rules()
+ end
+
def features do
[
"pleroma_api",
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index e8b7b17aa..1246989b0 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -606,6 +606,7 @@ defmodule Pleroma.Web.Router do
get("/instance", InstanceController, :show)
get("/instance/peers", InstanceController, :peers)
+ get("/instance/rules", InstanceController, :rules)
get("/statuses", StatusController, :index)
get("/statuses/:id", StatusController, :show)
From 384f8bfa786f51f3abec101e2ab78917f324a4fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Mon, 7 Feb 2022 23:41:41 +0100
Subject: [PATCH 010/823] Instance rules: Use render_many
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
.../admin_api/controllers/rule_controller.ex | 2 --
.../web/mastodon_api/views/instance_view.ex | 19 +++++++++++--------
2 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/lib/pleroma/web/admin_api/controllers/rule_controller.ex b/lib/pleroma/web/admin_api/controllers/rule_controller.ex
index 2db88b6ba..43b2f209a 100644
--- a/lib/pleroma/web/admin_api/controllers/rule_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/rule_controller.ex
@@ -14,8 +14,6 @@ defmodule Pleroma.Web.AdminAPI.RuleController do
json_response: 3
]
- require Logger
-
plug(Pleroma.Web.ApiSpec.CastAndValidate)
plug(
diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index feb94b40b..8379731e4 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -40,7 +40,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image),
shout_limit: Config.get([:shout, :limit]),
description_limit: Keyword.get(instance, :description_limit),
- rules: rules(),
+ rules: render(__MODULE__, "rules.json"),
pleroma: %{
metadata: %{
account_activation_required: Keyword.get(instance, :account_activation_required),
@@ -59,7 +59,16 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
end
def render("rules.json", _) do
- rules()
+ Pleroma.Rule.query()
+ |> Pleroma.Repo.all()
+ |> render_many(__MODULE__, "rule.json", as: :rule)
+ end
+
+ def render("rule.json", %{rule: rule}) do
+ %{
+ id: rule.id,
+ text: rule.text
+ }
end
def features do
@@ -142,10 +151,4 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
value_length: Config.get([:instance, :account_field_value_length])
}
end
-
- def rules do
- Pleroma.Rule.query()
- |> Pleroma.Repo.all()
- |> Enum.map(&%{id: &1.id, text: &1.text})
- end
end
From bbf3bc2228ca4bc5c209e418538665240e2aa9ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Mon, 7 Feb 2022 23:55:20 +0100
Subject: [PATCH 011/823] Add RuleTest
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
test/pleroma/rule_test.exs | 46 ++++++++++++++++++++++++++++++++++++++
1 file changed, 46 insertions(+)
create mode 100644 test/pleroma/rule_test.exs
diff --git a/test/pleroma/rule_test.exs b/test/pleroma/rule_test.exs
new file mode 100644
index 000000000..012ac902c
--- /dev/null
+++ b/test/pleroma/rule_test.exs
@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.RuleTest do
+ use Pleroma.DataCase, async: true
+
+ alias Pleroma.Repo
+ alias Pleroma.Rule
+
+ test "getting a list of rules sorted by priority" do
+ %{id: id1} = Rule.create(%{text: "Example rule"})
+ %{id: id2} = Rule.create(%{text: "Second rule", priority: 2})
+ %{id: id3} = Rule.create(%{text: "Third rule", priority: 1})
+
+ rules =
+ Rule.query()
+ |> Repo.all()
+
+ assert [%{id: ^id1}, %{id: ^id3}, %{id: ^id2}] = rules
+ end
+
+ test "creating rules" do
+ %{id: id} = Rule.create(%{text: "Example rule"})
+
+ assert %{text: "Example rule"} = Rule.get(id)
+ end
+
+ test "editing rules" do
+ %{id: id} = Rule.create(%{text: "Example rule"})
+
+ Rule.update(%{text: "There are no rules", priority: 2}, id)
+
+ assert %{text: "There are no rules", priority: 2} = Rule.get(id)
+ end
+
+ test "deleting rules" do
+ %{id: id} = Rule.create(%{text: "Example rule"})
+
+ Rule.delete(id)
+
+ assert [] =
+ Rule.query()
+ |> Pleroma.Repo.all()
+ end
+end
From 574db5b988e3caca14d1729a729af83d2f23e214 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Sun, 20 Feb 2022 21:44:42 +0100
Subject: [PATCH 012/823] Allow submitting an array of rule_ids to
/api/v1/reports
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
lib/pleroma/constants.ex | 3 ++-
lib/pleroma/rule.ex | 5 ++++
lib/pleroma/web/activity_pub/utils.ex | 8 +++++--
.../web/admin_api/views/report_view.ex | 15 +++++++++++-
.../operations/admin/report_operation.ex | 10 ++++++++
.../api_spec/operations/report_operation.ex | 9 +++++++-
lib/pleroma/web/common_api.ex | 17 ++++++++++++--
.../web/admin_api/views/report_view_test.exs | 23 +++++++++++++++++--
.../controllers/report_controller_test.exs | 21 +++++++++++++++++
9 files changed, 102 insertions(+), 9 deletions(-)
diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex
index a42c71d23..bf43becb3 100644
--- a/lib/pleroma/constants.ex
+++ b/lib/pleroma/constants.ex
@@ -19,7 +19,8 @@ defmodule Pleroma.Constants do
"context_id",
"deleted_activity_id",
"pleroma_internal",
- "generator"
+ "generator",
+ "rules"
]
)
diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex
index d772a32bd..486cff8cc 100644
--- a/lib/pleroma/rule.ex
+++ b/lib/pleroma/rule.ex
@@ -29,6 +29,11 @@ defmodule Pleroma.Rule do
|> order_by(asc: :priority)
end
+ def get(ids) when is_list(ids) do
+ from(r in __MODULE__, where: r.id in ^ids)
+ |> Repo.all()
+ end
+
def get(id), do: Repo.get(__MODULE__, id)
def create(params) do
diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex
index 9cde7805c..72d17e2aa 100644
--- a/lib/pleroma/web/activity_pub/utils.ex
+++ b/lib/pleroma/web/activity_pub/utils.ex
@@ -692,14 +692,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do
#### Flag-related helpers
@spec make_flag_data(map(), map()) :: map()
- def make_flag_data(%{actor: actor, context: context, content: content} = params, additional) do
+ def make_flag_data(
+ %{actor: actor, context: context, content: content} = params,
+ additional
+ ) do
%{
"type" => "Flag",
"actor" => actor.ap_id,
"content" => content,
"object" => build_flag_object(params),
"context" => context,
- "state" => "open"
+ "state" => "open",
+ "rules" => Map.get(params, :rules, nil)
}
|> Map.merge(additional)
end
diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex
index b761dbb22..ca70f4359 100644
--- a/lib/pleroma/web/admin_api/views/report_view.ex
+++ b/lib/pleroma/web/admin_api/views/report_view.ex
@@ -6,10 +6,12 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
use Pleroma.Web, :view
alias Pleroma.HTML
+ alias Pleroma.Rule
alias Pleroma.User
alias Pleroma.Web.AdminAPI
alias Pleroma.Web.AdminAPI.Report
alias Pleroma.Web.CommonAPI.Utils
+ alias Pleroma.Web.MastodonAPI.InstanceView
alias Pleroma.Web.MastodonAPI.StatusView
defdelegate merge_account_views(user), to: AdminAPI.AccountView
@@ -46,7 +48,8 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
as: :activity
}),
state: report.data["state"],
- notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes})
+ notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes}),
+ rules: rules(Map.get(report.data, "rules", nil))
}
end
@@ -71,4 +74,14 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
created_at: Utils.to_masto_date(inserted_at)
}
end
+
+ defp rules(nil) do
+ []
+ end
+
+ defp rules(rule_ids) do
+ rule_ids
+ |> Rule.get()
+ |> render_many(InstanceView, "rule.json", as: :rule)
+ end
end
diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex
index 312e091a5..bb71abbd1 100644
--- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex
@@ -169,6 +169,16 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do
inserted_at: %Schema{type: :string, format: :"date-time"}
}
}
+ },
+ rules: %Schema{
+ type: :array,
+ items: %Schema{
+ type: :object,
+ properties: %{
+ id: %Schema{type: :integer},
+ text: %Schema{type: :string}
+ }
+ }
}
}
}
diff --git a/lib/pleroma/web/api_spec/operations/report_operation.ex b/lib/pleroma/web/api_spec/operations/report_operation.ex
index c74ac7d5f..fd68f67a2 100644
--- a/lib/pleroma/web/api_spec/operations/report_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/report_operation.ex
@@ -53,6 +53,12 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do
default: false,
description:
"If the account is remote, should the report be forwarded to the remote admin?"
+ },
+ rule_ids: %Schema{
+ type: :array,
+ nullable: true,
+ items: %Schema{type: :number},
+ description: "Array of rules"
}
},
required: [:account_id],
@@ -60,7 +66,8 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do
"account_id" => "123",
"status_ids" => ["1337"],
"comment" => "bad status!",
- "forward" => "false"
+ "forward" => "false",
+ "rule_ids" => [3]
}
}
end
diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex
index 1b95ee89c..9f8d4def4 100644
--- a/lib/pleroma/web/common_api.ex
+++ b/lib/pleroma/web/common_api.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.CommonAPI do
alias Pleroma.Conversation.Participation
alias Pleroma.Formatter
alias Pleroma.Object
+ alias Pleroma.Rule
alias Pleroma.ThreadMute
alias Pleroma.User
alias Pleroma.UserRelationship
@@ -505,14 +506,16 @@ defmodule Pleroma.Web.CommonAPI do
def report(user, data) do
with {:ok, account} <- get_reported_account(data.account_id),
{:ok, {content_html, _, _}} <- make_report_content_html(data[:comment]),
- {:ok, statuses} <- get_report_statuses(account, data) do
+ {:ok, statuses} <- get_report_statuses(account, data),
+ rules <- get_report_rules(Map.get(data, :rule_ids, nil)) do
ActivityPub.flag(%{
context: Utils.generate_context_id(),
actor: user,
account: account,
statuses: statuses,
content: content_html,
- forward: Map.get(data, :forward, false)
+ forward: Map.get(data, :forward, false),
+ rules: rules
})
end
end
@@ -524,6 +527,16 @@ defmodule Pleroma.Web.CommonAPI do
end
end
+ defp get_report_rules(nil) do
+ nil
+ end
+
+ defp get_report_rules(rule_ids) do
+ rule_ids
+ |> Rule.get()
+ |> Enum.map(& &1.id)
+ end
+
def update_report_state(activity_ids, state) when is_list(activity_ids) do
case Utils.update_report_state(activity_ids, state) do
:ok -> {:ok, activity_ids}
diff --git a/test/pleroma/web/admin_api/views/report_view_test.exs b/test/pleroma/web/admin_api/views/report_view_test.exs
index 9637c2b90..519208b45 100644
--- a/test/pleroma/web/admin_api/views/report_view_test.exs
+++ b/test/pleroma/web/admin_api/views/report_view_test.exs
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do
import Pleroma.Factory
+ alias Pleroma.Rule
alias Pleroma.Web.AdminAPI
alias Pleroma.Web.AdminAPI.Report
alias Pleroma.Web.AdminAPI.ReportView
@@ -38,7 +39,8 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do
statuses: [],
notes: [],
state: "open",
- id: activity.id
+ id: activity.id,
+ rules: []
}
result =
@@ -76,7 +78,8 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do
statuses: [StatusView.render("show.json", %{activity: activity})],
state: "open",
notes: [],
- id: report_activity.id
+ id: report_activity.id,
+ rules: []
}
result =
@@ -168,4 +171,20 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do
assert report2.id == rendered |> Enum.at(0) |> Map.get(:id)
assert report1.id == rendered |> Enum.at(1) |> Map.get(:id)
end
+
+ test "renders included rules" do
+ user = insert(:user)
+ other_user = insert(:user)
+
+ %{id: id, text: text} = Rule.create(%{text: "Example rule"})
+
+ {:ok, activity} =
+ CommonAPI.report(user, %{
+ account_id: other_user.id,
+ rule_ids: [id]
+ })
+
+ assert %{rules: [%{id: ^id, text: ^text}]} =
+ ReportView.render("show.json", Report.extract_report_info(activity))
+ end
end
diff --git a/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs
index 6d1a63334..77626b821 100644
--- a/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs
@@ -5,6 +5,8 @@
defmodule Pleroma.Web.MastodonAPI.ReportControllerTest do
use Pleroma.Web.ConnCase, async: true
+ alias Pleroma.Activity
+ alias Pleroma.Rule
alias Pleroma.Web.CommonAPI
import Pleroma.Factory
@@ -44,6 +46,25 @@ defmodule Pleroma.Web.MastodonAPI.ReportControllerTest do
|> json_response_and_validate_schema(200)
end
+ test "submit a report with rule_ids", %{
+ conn: conn,
+ target_user: target_user
+ } do
+ %{id: rule_id} = Rule.create(%{text: "There are no rules"})
+
+ assert %{"action_taken" => false, "id" => id} =
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/v1/reports", %{
+ "account_id" => target_user.id,
+ "forward" => "false",
+ "rule_ids" => [rule_id]
+ })
+ |> json_response_and_validate_schema(200)
+
+ assert %Activity{data: %{"rules" => [^rule_id]}} = Activity.get_report(id)
+ end
+
test "account_id is required", %{
conn: conn,
activity: activity
From d26aadb743c0de6fee7653ac2b90f989862d3c02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Tue, 1 Mar 2022 23:50:09 +0100
Subject: [PATCH 013/823] Add tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
test/pleroma/rule_test.exs | 11 +++++++++++
test/pleroma/web/common_api_test.exs | 28 ++++++++++++++++++++++++++++
2 files changed, 39 insertions(+)
diff --git a/test/pleroma/rule_test.exs b/test/pleroma/rule_test.exs
index 012ac902c..d710a6312 100644
--- a/test/pleroma/rule_test.exs
+++ b/test/pleroma/rule_test.exs
@@ -43,4 +43,15 @@ defmodule Pleroma.RuleTest do
Rule.query()
|> Pleroma.Repo.all()
end
+
+ test "getting rules by ids" do
+ %{id: id1} = Rule.create(%{text: "Example rule"})
+ %{id: id2} = Rule.create(%{text: "Second rule"})
+ %{id: _id3} = Rule.create(%{text: "Third rule"})
+
+ rules = Rule.get([id1, id2])
+
+ assert Enum.all?(rules, &(&1.id in [id1, id2]))
+ assert length(rules) == 2
+ end
end
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index b502aaa03..103aab397 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -12,6 +12,7 @@ defmodule Pleroma.Web.CommonAPITest do
alias Pleroma.Notification
alias Pleroma.Object
alias Pleroma.Repo
+ alias Pleroma.Rule
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Transmogrifier
@@ -1186,6 +1187,33 @@ defmodule Pleroma.Web.CommonAPITest do
assert first_report.data["state"] == "resolved"
assert second_report.data["state"] == "resolved"
end
+
+ test "creates a report with provided rules" do
+ reporter = insert(:user)
+ target_user = insert(:user)
+
+ %{id: rule_id} = Rule.create(%{text: "There are no rules"})
+
+ reporter_ap_id = reporter.ap_id
+ target_ap_id = target_user.ap_id
+
+ report_data = %{
+ account_id: target_user.id,
+ rule_ids: [rule_id]
+ }
+
+ assert {:ok, flag_activity} = CommonAPI.report(reporter, report_data)
+
+ assert %Activity{
+ actor: ^reporter_ap_id,
+ data: %{
+ "type" => "Flag",
+ "object" => [^target_ap_id],
+ "state" => "open",
+ "rules" => [^rule_id]
+ }
+ } = flag_activity
+ end
end
describe "reblog muting" do
From 5c383ada8ad7491125a8264a8a03d85fe822f61e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Sat, 5 Mar 2022 21:45:34 +0100
Subject: [PATCH 014/823] Correctly order rules by id/creation date
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
lib/pleroma/rule.ex | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex
index 486cff8cc..b1db1dc0c 100644
--- a/lib/pleroma/rule.ex
+++ b/lib/pleroma/rule.ex
@@ -27,6 +27,7 @@ defmodule Pleroma.Rule do
def query do
Rule
|> order_by(asc: :priority)
+ |> order_by(asc: :id)
end
def get(ids) when is_list(ids) do
From b354d70e85bbf0f685f3d56f7377fde2efce4187 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Mon, 30 May 2022 12:30:03 +0200
Subject: [PATCH 015/823] Apply, suggestions, use strings for actual Mastodon
API compatibility
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
lib/pleroma/rule.ex | 2 ++
lib/pleroma/web/admin_api/views/report_view.ex | 10 ++++++----
lib/pleroma/web/admin_api/views/rule_view.ex | 2 +-
.../operations/admin/report_operation.ex | 2 +-
.../api_spec/operations/admin/rule_operation.ex | 5 ++---
.../api_spec/operations/instance_operation.ex | 2 +-
.../web/api_spec/operations/report_operation.ex | 4 ++--
lib/pleroma/web/common_api.ex | 3 +--
.../web/mastodon_api/views/instance_view.ex | 2 +-
.../controllers/rule_controller_test.exs | 4 ++++
.../controllers/report_controller_test.exs | 17 +++++++++++++++++
11 files changed, 38 insertions(+), 15 deletions(-)
diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex
index b1db1dc0c..611e945b3 100644
--- a/lib/pleroma/rule.ex
+++ b/lib/pleroma/rule.ex
@@ -37,6 +37,8 @@ defmodule Pleroma.Rule do
def get(id), do: Repo.get(__MODULE__, id)
+ def exists?(id), do: not is_nil(get(id))
+
def create(params) do
{:ok, rule} =
%Rule{}
diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex
index ca70f4359..b4b0be267 100644
--- a/lib/pleroma/web/admin_api/views/report_view.ex
+++ b/lib/pleroma/web/admin_api/views/report_view.ex
@@ -10,8 +10,8 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
alias Pleroma.User
alias Pleroma.Web.AdminAPI
alias Pleroma.Web.AdminAPI.Report
+ alias Pleroma.Web.AdminAPI.RuleView
alias Pleroma.Web.CommonAPI.Utils
- alias Pleroma.Web.MastodonAPI.InstanceView
alias Pleroma.Web.MastodonAPI.StatusView
defdelegate merge_account_views(user), to: AdminAPI.AccountView
@@ -80,8 +80,10 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
end
defp rules(rule_ids) do
- rule_ids
- |> Rule.get()
- |> render_many(InstanceView, "rule.json", as: :rule)
+ rules =
+ rule_ids
+ |> Rule.get()
+
+ render(RuleView, "index.json", rules: rules)
end
end
diff --git a/lib/pleroma/web/admin_api/views/rule_view.ex b/lib/pleroma/web/admin_api/views/rule_view.ex
index f29145248..abfdd593f 100644
--- a/lib/pleroma/web/admin_api/views/rule_view.ex
+++ b/lib/pleroma/web/admin_api/views/rule_view.ex
@@ -13,7 +13,7 @@ defmodule Pleroma.Web.AdminAPI.RuleView do
def render("show.json", %{rule: rule} = _opts) do
%{
- id: rule.id,
+ id: to_string(rule.id),
priority: rule.priority,
text: rule.text
}
diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex
index bb71abbd1..b90bbd592 100644
--- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex
@@ -175,7 +175,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do
items: %Schema{
type: :object,
properties: %{
- id: %Schema{type: :integer},
+ id: %Schema{type: :string},
text: %Schema{type: :string}
}
}
diff --git a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex
index ed0d9eaf6..2360880e4 100644
--- a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex
@@ -103,10 +103,9 @@ defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do
%Schema{
type: :object,
properties: %{
- id: %Schema{type: :integer},
+ id: %Schema{type: :string},
priority: %Schema{type: :integer},
- text: %Schema{type: :string},
- created_at: %Schema{type: :string, format: :"date-time"}
+ text: %Schema{type: :string}
}
}
end
diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex
index e66e5b7a3..f3dba108e 100644
--- a/lib/pleroma/web/api_spec/operations/instance_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex
@@ -191,7 +191,7 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
items: %Schema{
type: :object,
properties: %{
- id: %Schema{type: :integer},
+ id: %Schema{type: :string},
text: %Schema{type: :string}
}
}
diff --git a/lib/pleroma/web/api_spec/operations/report_operation.ex b/lib/pleroma/web/api_spec/operations/report_operation.ex
index fd68f67a2..f5f88974c 100644
--- a/lib/pleroma/web/api_spec/operations/report_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/report_operation.ex
@@ -57,7 +57,7 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do
rule_ids: %Schema{
type: :array,
nullable: true,
- items: %Schema{type: :number},
+ items: %Schema{type: :string},
description: "Array of rules"
}
},
@@ -67,7 +67,7 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do
"status_ids" => ["1337"],
"comment" => "bad status!",
"forward" => "false",
- "rule_ids" => [3]
+ "rule_ids" => ["3"]
}
}
end
diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex
index 9f8d4def4..6fd744ddc 100644
--- a/lib/pleroma/web/common_api.ex
+++ b/lib/pleroma/web/common_api.ex
@@ -533,8 +533,7 @@ defmodule Pleroma.Web.CommonAPI do
defp get_report_rules(rule_ids) do
rule_ids
- |> Rule.get()
- |> Enum.map(& &1.id)
+ |> Enum.filter(&Rule.exists?/1)
end
def update_report_state(activity_ids, state) when is_list(activity_ids) do
diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index 8379731e4..c7f5ff554 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -66,7 +66,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
def render("rule.json", %{rule: rule}) do
%{
- id: rule.id,
+ id: to_string(rule.id),
text: rule.text
}
end
diff --git a/test/pleroma/web/admin_api/controllers/rule_controller_test.exs b/test/pleroma/web/admin_api/controllers/rule_controller_test.exs
index c5c72d293..96b52b272 100644
--- a/test/pleroma/web/admin_api/controllers/rule_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/rule_controller_test.exs
@@ -27,6 +27,10 @@ defmodule Pleroma.Web.AdminAPI.RuleControllerTest do
%{id: id2} = Rule.create(%{text: "Second rule", priority: 2})
%{id: id3} = Rule.create(%{text: "Third rule", priority: 1})
+ id1 = to_string(id1)
+ id2 = to_string(id2)
+ id3 = to_string(id3)
+
response =
conn
|> get("/api/pleroma/admin/rules")
diff --git a/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs
index 77626b821..689a7f375 100644
--- a/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs
@@ -65,6 +65,23 @@ defmodule Pleroma.Web.MastodonAPI.ReportControllerTest do
assert %Activity{data: %{"rules" => [^rule_id]}} = Activity.get_report(id)
end
+ test "rules field is empty if provided wrong rule id", %{
+ conn: conn,
+ target_user: target_user
+ } do
+ assert %{"id" => id} =
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/v1/reports", %{
+ "account_id" => target_user.id,
+ "forward" => "false",
+ "rule_ids" => ["-1"]
+ })
+ |> json_response_and_validate_schema(200)
+
+ assert %Activity{data: %{"rules" => []}} = Activity.get_report(id)
+ end
+
test "account_id is required", %{
conn: conn,
activity: activity
From 0ecd6ba35e868eac296b013f743d82a120dd68db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Mon, 30 May 2022 12:50:44 +0200
Subject: [PATCH 016/823] AdminAPI: Allow filtering reports by rule_id
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
lib/pleroma/web/activity_pub/activity_pub.ex | 10 +++++++
.../operations/admin/report_operation.ex | 6 ++++
.../controllers/report_controller_test.exs | 29 +++++++++++++++++++
.../web/admin_api/views/report_view_test.exs | 8 +++--
.../controllers/report_controller_test.exs | 2 ++
5 files changed, 52 insertions(+), 3 deletions(-)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 064f93b22..e54adf611 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1191,6 +1191,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp restrict_filtered(query, _), do: query
+ defp restrict_rule(query, %{rule_id: rule_id}) do
+ from(
+ activity in query,
+ where: fragment("(?)->'rules' \\? (?)", activity.data, ^rule_id)
+ )
+ end
+
+ defp restrict_rule(query, _), do: query
+
defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query
defp exclude_poll_votes(query, _) do
@@ -1353,6 +1362,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|> restrict_instance(opts)
|> restrict_announce_object_actor(opts)
|> restrict_filtered(opts)
+ |> restrict_rule(opts)
|> Activity.restrict_deactivated_users()
|> exclude_poll_votes(opts)
|> exclude_chat_messages(opts)
diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex
index b90bbd592..18386296f 100644
--- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex
@@ -30,6 +30,12 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do
report_state(),
"Filter by report state"
),
+ Operation.parameter(
+ :rule_id,
+ :query,
+ %Schema{type: :string},
+ "Filter by selected rule id"
+ ),
Operation.parameter(
:limit,
:query,
diff --git a/test/pleroma/web/admin_api/controllers/report_controller_test.exs b/test/pleroma/web/admin_api/controllers/report_controller_test.exs
index 30dcb87e2..5dc3e7491 100644
--- a/test/pleroma/web/admin_api/controllers/report_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/report_controller_test.exs
@@ -11,6 +11,7 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do
alias Pleroma.ModerationLog
alias Pleroma.Repo
alias Pleroma.ReportNote
+ alias Pleroma.Rule
alias Pleroma.Web.CommonAPI
setup do
@@ -313,6 +314,34 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do
"error" => "Invalid credentials."
}
end
+
+ test "returns reports with specified role_id", %{conn: conn} do
+ [reporter, target_user] = insert_pair(:user)
+
+ %{id: rule_id} = Rule.create(%{text: "Example rule"})
+
+ rule_id = to_string(rule_id)
+
+ {:ok, %{id: report_id}} =
+ CommonAPI.report(reporter, %{
+ account_id: target_user.id,
+ comment: "",
+ rule_ids: [rule_id]
+ })
+
+ {:ok, _report} =
+ CommonAPI.report(reporter, %{
+ account_id: target_user.id,
+ comment: ""
+ })
+
+ response =
+ conn
+ |> get("/api/pleroma/admin/reports?rule_id=#{rule_id}")
+ |> json_response_and_validate_schema(:ok)
+
+ assert %{"reports" => [%{"id" => ^report_id}]} = response
+ end
end
describe "POST /api/pleroma/admin/reports/:id/notes" do
diff --git a/test/pleroma/web/admin_api/views/report_view_test.exs b/test/pleroma/web/admin_api/views/report_view_test.exs
index 519208b45..1b16aca6a 100644
--- a/test/pleroma/web/admin_api/views/report_view_test.exs
+++ b/test/pleroma/web/admin_api/views/report_view_test.exs
@@ -176,15 +176,17 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do
user = insert(:user)
other_user = insert(:user)
- %{id: id, text: text} = Rule.create(%{text: "Example rule"})
+ %{id: rule_id, text: text} = Rule.create(%{text: "Example rule"})
+
+ rule_id = to_string(rule_id)
{:ok, activity} =
CommonAPI.report(user, %{
account_id: other_user.id,
- rule_ids: [id]
+ rule_ids: [rule_id]
})
- assert %{rules: [%{id: ^id, text: ^text}]} =
+ assert %{rules: [%{id: ^rule_id, text: ^text}]} =
ReportView.render("show.json", Report.extract_report_info(activity))
end
end
diff --git a/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs
index 689a7f375..509de6899 100644
--- a/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs
@@ -52,6 +52,8 @@ defmodule Pleroma.Web.MastodonAPI.ReportControllerTest do
} do
%{id: rule_id} = Rule.create(%{text: "There are no rules"})
+ rule_id = to_string(rule_id)
+
assert %{"action_taken" => false, "id" => id} =
conn
|> put_req_header("content-type", "application/json")
From 5846e7d5f6b91ab63270f2104543d874589d39ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Wed, 1 Jun 2022 16:03:22 +0200
Subject: [PATCH 017/823] Use Repo.exists?
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
lib/pleroma/rule.ex | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex
index 611e945b3..c8e3470c7 100644
--- a/lib/pleroma/rule.ex
+++ b/lib/pleroma/rule.ex
@@ -37,7 +37,10 @@ defmodule Pleroma.Rule do
def get(id), do: Repo.get(__MODULE__, id)
- def exists?(id), do: not is_nil(get(id))
+ def exists?(id) do
+ from(r in __MODULE__, where: r.id == ^id)
+ |> Repo.exists?()
+ end
def create(params) do
{:ok, rule} =
From 6b74a5527410c5e98a6c4906a8e6f9c4d9f5e5d1 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Fri, 1 Jul 2022 15:40:40 -0500
Subject: [PATCH 018/823] InstanceStatic should have reasonable caching
While here fix the naming convention of the old module attribute restricting caching and add a new one for the default cache value
All frontends should be shipped with versioned assets. There be dragons if you don't.
---
lib/pleroma/web/endpoint.ex | 41 +++++++++++++++++++++++--------------
1 file changed, 26 insertions(+), 15 deletions(-)
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index d8d40cceb..17bd956d8 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -19,7 +19,8 @@ defmodule Pleroma.Web.Endpoint do
plug(Pleroma.Web.Plugs.HTTPSecurityPlug)
plug(Pleroma.Web.Plugs.UploadedMedia)
- @static_cache_control "public, no-cache"
+ @static_cache_control "public, max-age=1209600"
+ @static_cache_disabled "public, no-cache"
# InstanceStatic needs to be before Plug.Static to be able to override shipped-static files
# If you're adding new paths to `only:` you'll need to configure them in InstanceStatic as well
@@ -30,22 +31,32 @@ defmodule Pleroma.Web.Endpoint do
from: :pleroma,
only: ["emoji", "images"],
gzip: true,
- cache_control_for_etags: "public, max-age=1209600",
- headers: %{
- "cache-control" => "public, max-age=1209600"
- }
- )
-
- plug(Pleroma.Web.Plugs.InstanceStatic,
- at: "/",
- gzip: true,
cache_control_for_etags: @static_cache_control,
headers: %{
"cache-control" => @static_cache_control
}
)
- # Careful! No `only` restriction here, as we don't know what frontends contain.
+ plug(Pleroma.Web.Plugs.InstanceStatic,
+ at: "/",
+ gzip: true,
+ cache_control_for_etags: @static_cache_disabled,
+ headers: %{
+ "cache-control" => @static_cache_disabled
+ }
+ )
+
+ plug(Pleroma.Web.Plugs.FrontendStatic,
+ at: "/",
+ frontend_type: :primary,
+ only: ["index.html"],
+ gzip: true,
+ cache_control_for_etags: @static_cache_disabled,
+ headers: %{
+ "cache-control" => @static_cache_disabled
+ }
+ )
+
plug(Pleroma.Web.Plugs.FrontendStatic,
at: "/",
frontend_type: :primary,
@@ -62,9 +73,9 @@ defmodule Pleroma.Web.Endpoint do
at: "/pleroma/admin",
frontend_type: :admin,
gzip: true,
- cache_control_for_etags: @static_cache_control,
+ cache_control_for_etags: @static_cache_disabled,
headers: %{
- "cache-control" => @static_cache_control
+ "cache-control" => @static_cache_disabled
}
)
@@ -79,9 +90,9 @@ defmodule Pleroma.Web.Endpoint do
only: Pleroma.Constants.static_only_files(),
# credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
gzip: true,
- cache_control_for_etags: @static_cache_control,
+ cache_control_for_etags: @static_cache_disabled,
headers: %{
- "cache-control" => @static_cache_control
+ "cache-control" => @static_cache_disabled
}
)
From fa2a6d5d6b24657ddbda4ef11d2e6dbcb59545d3 Mon Sep 17 00:00:00 2001
From: Claudio Maradonna
Date: Thu, 7 Apr 2022 18:25:02 +0200
Subject: [PATCH 019/823] feat: simple, but not stupid, uploader for IPFS
fix: format fix with credo
---
config/config.exs | 4 +++
config/description.exs | 24 +++++++++++++
config/dev.exs | 4 +++
lib/pleroma/upload.ex | 13 +++++--
lib/pleroma/uploaders/ipfs.ex | 64 +++++++++++++++++++++++++++++++++++
5 files changed, 107 insertions(+), 2 deletions(-)
create mode 100644 lib/pleroma/uploaders/ipfs.ex
diff --git a/config/config.exs b/config/config.exs
index 6a5acda09..7efad0061 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -82,6 +82,10 @@ config :ex_aws, :s3,
# region: "us-east-1", # may be required for Amazon AWS
scheme: "https://"
+config :pleroma, Pleroma.Uploaders.IPFS,
+ post_gateway_url: nil,
+ get_gateway_url: nil
+
config :pleroma, :emoji,
shortcode_globs: ["/emoji/custom/**/*.png"],
pack_extensions: [".png", ".gif"],
diff --git a/config/description.exs b/config/description.exs
index 7caad18b4..d87bbb9b8 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -136,6 +136,30 @@ config :pleroma, :config_description, [
}
]
},
+ %{
+ group: :pleroma,
+ key: Pleroma.Uploaders.IPFS,
+ type: :group,
+ description: "IPFS uploader-related settings",
+ children: [
+ %{
+ key: :get_gateway_url,
+ type: :string,
+ description: "GET Gateway URL",
+ suggestions: [
+ "get_gateway_url"
+ ]
+ },
+ %{
+ key: :post_gateway_url,
+ type: :string,
+ description: "POST Gateway URL",
+ suggestions: [
+ "post_gateway_url"
+ ]
+ }
+ ]
+ },
%{
group: :pleroma,
key: Pleroma.Uploaders.S3,
diff --git a/config/dev.exs b/config/dev.exs
index ab3e83c12..89a84bf05 100644
--- a/config/dev.exs
+++ b/config/dev.exs
@@ -58,6 +58,10 @@ config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true
# https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects
config :phoenix, :plug_init_mode, :runtime
+config :pleroma, Pleroma.Uploaders.IPFS,
+ post_gateway_url: nil,
+ get_gateway_url: nil
+
if File.exists?("./config/dev.secret.exs") do
import_config "dev.secret.exs"
else
diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
index db2909276..de39bcd6c 100644
--- a/lib/pleroma/upload.ex
+++ b/lib/pleroma/upload.ex
@@ -235,8 +235,14 @@ defmodule Pleroma.Upload do
""
end
- [base_url, path]
- |> Path.join()
+ uploader = Config.get([Pleroma.Upload, :uploader])
+
+ if uploader == Pleroma.Uploaders.IPFS && String.contains?(base_url, "{CID}") do
+ String.replace(base_url, "{CID}", path)
+ else
+ [base_url, path]
+ |> Path.join()
+ end
end
defp url_from_spec(_upload, _base_url, {:url, url}), do: url
@@ -273,6 +279,9 @@ defmodule Pleroma.Upload do
Path.join([upload_base_url, bucket_with_namespace])
end
+ Pleroma.Uploaders.IPFS ->
+ Config.get([Pleroma.Uploaders.IPFS, :get_gateway_url])
+
_ ->
public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
end
diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex
new file mode 100644
index 000000000..b46e9322e
--- /dev/null
+++ b/lib/pleroma/uploaders/ipfs.ex
@@ -0,0 +1,64 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Uploaders.IPFS do
+ @behaviour Pleroma.Uploaders.Uploader
+ require Logger
+
+ alias Pleroma.Config
+ alias Tesla.Multipart
+
+ @impl true
+ def get_file(file) do
+ b_url = Pleroma.Upload.base_url()
+
+ if String.contains?(b_url, "{CID}") do
+ {:ok, {:url, String.replace(b_url, "{CID}", URI.decode(file))}}
+ else
+ {:error, "IPFS Get URL doesn't contain '{CID}' placeholder"}
+ end
+ end
+
+ @impl true
+ def put_file(%Pleroma.Upload{} = upload) do
+ config = Config.get([__MODULE__])
+ post_base_url = Keyword.get(config, :post_gateway_url)
+
+ mp =
+ Multipart.new()
+ |> Multipart.add_content_type_param("charset=utf-8")
+ |> Multipart.add_file(upload.tempfile)
+
+ final_url = Path.join([post_base_url, "/api/v0/add"])
+
+ case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do
+ {:ok, ret} ->
+ case Jason.decode(ret.body) do
+ {:ok, ret} ->
+ {:ok, {:file, ret["Hash"]}}
+
+ error ->
+ Logger.error("#{__MODULE__}: #{inspect(error)}")
+ {:error, "JSON decode failed"}
+ end
+
+ error ->
+ Logger.error("#{__MODULE__}: #{inspect(error)}")
+ {:error, "IPFS Gateway Upload failed"}
+ end
+ end
+
+ @impl true
+ def delete_file(file) do
+ config = Config.get([__MODULE__])
+ post_base_url = Keyword.get(config, :post_gateway_url)
+
+ final_url = Path.join([post_base_url, "/api/v0/files/rm"])
+
+ case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do
+ {:ok, %{status_code: 204}} -> :ok
+ error -> {:error, inspect(error)}
+ end
+ end
+end
From 43dfa58ebda407a0813d398bee8d0ae3e5c9fd5b Mon Sep 17 00:00:00 2001
From: Claudio Maradonna
Date: Mon, 11 Apr 2022 15:10:01 +0200
Subject: [PATCH 020/823] added tests for ipfs uploader. adapted changelog.md
accordingly. improved ipfs uploader with external suggestions
fix lint description.exs
---
CHANGELOG.md | 1 +
config/description.exs | 5 +-
config/dev.exs | 4 --
docs/configuration/cheatsheet.md | 13 ++++
lib/pleroma/upload.ex | 6 +-
lib/pleroma/uploaders/ipfs.ex | 14 +++--
test/pleroma/uploaders/ipfs_test.exs | 88 ++++++++++++++++++++++++++++
7 files changed, 116 insertions(+), 15 deletions(-)
create mode 100644 test/pleroma/uploaders/ipfs_test.exs
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f1beb0cd0..b2185b1ba 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- PleromaAPI: Add `GET /api/v1/pleroma/birthdays` API endpoint
- Make backend-rendered pages translatable. This includes emails. Pages returned as a HTTP response are translated using the language specified in the `userLanguage` cookie, or the `Accept-Language` header. Emails are translated using the `language` field when registering. This language can be changed by `PATCH /api/v1/accounts/update_credentials` with the `language` field.
- Uploadfilter `Pleroma.Upload.Filter.Exiftool.ReadDescription` returns description values to the FE so they can pre fill the image description field
+- Uploader: Add support for uploading attachments using IPFS
### Fixed
- Subscription(Bell) Notifications: Don't create from Pipeline Ingested replies
diff --git a/config/description.exs b/config/description.exs
index d87bbb9b8..1fe5f01f0 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -147,7 +147,8 @@ config :pleroma, :config_description, [
type: :string,
description: "GET Gateway URL",
suggestions: [
- "get_gateway_url"
+ "https://ipfs.mydomain.com/<%= cid %>",
+ "https://<%= cid %>.ipfs.mydomain.com/"
]
},
%{
@@ -155,7 +156,7 @@ config :pleroma, :config_description, [
type: :string,
description: "POST Gateway URL",
suggestions: [
- "post_gateway_url"
+ "http://localhost:5001/"
]
}
]
diff --git a/config/dev.exs b/config/dev.exs
index 89a84bf05..ab3e83c12 100644
--- a/config/dev.exs
+++ b/config/dev.exs
@@ -58,10 +58,6 @@ config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true
# https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects
config :phoenix, :plug_init_mode, :runtime
-config :pleroma, Pleroma.Uploaders.IPFS,
- post_gateway_url: nil,
- get_gateway_url: nil
-
if File.exists?("./config/dev.secret.exs") do
import_config "dev.secret.exs"
else
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 74642397b..7e1f9c934 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -614,6 +614,19 @@ config :ex_aws, :s3,
host: "s3.eu-central-1.amazonaws.com"
```
+#### Pleroma.Uploaders.IPFS
+
+* `post_gateway_url`: URL with port of POST Gateway (unauthenticated)
+* `get_gateway_url`: URL of public GET Gateway
+
+Example:
+
+```elixir
+config :pleroma, Pleroma.Uploaders.IPFS,
+ post_gateway_url: "http://localhost:5001",
+ get_gateway_url: "http://<%= cid %>.ipfs.mydomain.com"
+```
+
### Upload filters
#### Pleroma.Upload.Filter.AnonymizeFilename
diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
index de39bcd6c..b51d23f9e 100644
--- a/lib/pleroma/upload.ex
+++ b/lib/pleroma/upload.ex
@@ -235,10 +235,8 @@ defmodule Pleroma.Upload do
""
end
- uploader = Config.get([Pleroma.Upload, :uploader])
-
- if uploader == Pleroma.Uploaders.IPFS && String.contains?(base_url, "{CID}") do
- String.replace(base_url, "{CID}", path)
+ if String.contains?(base_url, "<%= cid %>") do
+ EEx.eval_string(base_url, cid: path)
else
[base_url, path]
|> Path.join()
diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex
index b46e9322e..722c68fa1 100644
--- a/lib/pleroma/uploaders/ipfs.ex
+++ b/lib/pleroma/uploaders/ipfs.ex
@@ -13,10 +13,10 @@ defmodule Pleroma.Uploaders.IPFS do
def get_file(file) do
b_url = Pleroma.Upload.base_url()
- if String.contains?(b_url, "{CID}") do
- {:ok, {:url, String.replace(b_url, "{CID}", URI.decode(file))}}
+ if String.contains?(b_url, "<%= cid %>") do
+ {:ok, {:url, EEx.eval_string(b_url, cid: URI.decode(file))}}
else
- {:error, "IPFS Get URL doesn't contain '{CID}' placeholder"}
+ {:error, "IPFS Get URL doesn't contain 'cid' placeholder"}
end
end
@@ -36,7 +36,11 @@ defmodule Pleroma.Uploaders.IPFS do
{:ok, ret} ->
case Jason.decode(ret.body) do
{:ok, ret} ->
- {:ok, {:file, ret["Hash"]}}
+ if Map.has_key?(ret, "Hash") do
+ {:ok, {:file, ret["Hash"]}}
+ else
+ {:error, "JSON doesn't contain Hash value"}
+ end
error ->
Logger.error("#{__MODULE__}: #{inspect(error)}")
@@ -45,7 +49,7 @@ defmodule Pleroma.Uploaders.IPFS do
error ->
Logger.error("#{__MODULE__}: #{inspect(error)}")
- {:error, "IPFS Gateway Upload failed"}
+ {:error, "IPFS Gateway upload failed"}
end
end
diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
new file mode 100644
index 000000000..f9ae046cf
--- /dev/null
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -0,0 +1,88 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Uploaders.IPFSTest do
+ use Pleroma.DataCase
+
+ alias Pleroma.Uploaders.IPFS
+ alias Tesla.Multipart
+
+ import Mock
+ import ExUnit.CaptureLog
+
+ setup do
+ clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.IPFS)
+ clear_config([Pleroma.Uploaders.IPFS])
+
+ clear_config(
+ [Pleroma.Uploaders.IPFS, :get_gateway_url],
+ "https://<%= cid %>.ipfs.mydomain.com"
+ )
+
+ clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001")
+ end
+
+ describe "get_file/1" do
+ test "it returns path to ipfs file with cid as subdomain" do
+ assert IPFS.get_file("testcid") == {
+ :ok,
+ {:url, "https://testcid.ipfs.mydomain.com"}
+ }
+ end
+
+ test "it returns path to ipfs file with cid as path" do
+ clear_config(
+ [Pleroma.Uploaders.IPFS, :get_gateway_url],
+ "https://ipfs.mydomain.com/ipfs/<%= cid %>"
+ )
+
+ assert IPFS.get_file("testcid") == {
+ :ok,
+ {:url, "https://ipfs.mydomain.com/ipfs/testcid"}
+ }
+ end
+ end
+
+ describe "put_file/1" do
+ setup do
+ file_upload = %Pleroma.Upload{
+ name: "image-tet.jpg",
+ content_type: "image/jpeg",
+ path: "test_folder/image-tet.jpg",
+ tempfile: Path.absname("test/instance_static/add/shortcode.png")
+ }
+
+ [file_upload: file_upload]
+ end
+
+ test "save file", %{file_upload: file_upload} do
+ with_mock Pleroma.HTTP,
+ post: fn _, _, _, _ ->
+ {:ok,
+ %Tesla.Env{
+ status: 200,
+ body: "{\"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}"
+ }}
+ end do
+ assert IPFS.put_file(file_upload) ==
+ {:ok, {:file, "bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi"}}
+ end
+ end
+
+ test "returns error", %{file_upload: file_upload} do
+ with_mock Pleroma.HTTP, post: fn _, _, _, _ -> {:error, "IPFS Gateway upload failed"} end do
+ assert capture_log(fn ->
+ assert IPFS.put_file(file_upload) == {:error, "IPFS Gateway upload failed"}
+ end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}"
+ end
+ end
+ end
+
+ describe "delete_file/1" do
+ test_with_mock "deletes file", Pleroma.HTTP,
+ post: fn _, _, _, _ -> {:ok, %{status_code: 204}} end do
+ assert :ok = IPFS.delete_file("image.jpg")
+ end
+ end
+end
From 44659ecd65fb2251f9130fcecf1732b8931104c1 Mon Sep 17 00:00:00 2001
From: Claudio Maradonna
Date: Sat, 16 Apr 2022 09:38:49 +0200
Subject: [PATCH 021/823] ipfs: revert to String.replace for cid placeholder
ipfs: fix lint
---
config/description.exs | 4 ++--
docs/configuration/cheatsheet.md | 2 +-
lib/pleroma/upload.ex | 4 ++--
lib/pleroma/uploaders/ipfs.ex | 7 +++++--
test/pleroma/uploaders/ipfs_test.exs | 4 ++--
5 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/config/description.exs b/config/description.exs
index 1fe5f01f0..b180e7308 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -147,8 +147,8 @@ config :pleroma, :config_description, [
type: :string,
description: "GET Gateway URL",
suggestions: [
- "https://ipfs.mydomain.com/<%= cid %>",
- "https://<%= cid %>.ipfs.mydomain.com/"
+ "https://ipfs.mydomain.com/{CID}",
+ "https://{CID}.ipfs.mydomain.com/"
]
},
%{
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 7e1f9c934..d35b33574 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -624,7 +624,7 @@ Example:
```elixir
config :pleroma, Pleroma.Uploaders.IPFS,
post_gateway_url: "http://localhost:5001",
- get_gateway_url: "http://<%= cid %>.ipfs.mydomain.com"
+ get_gateway_url: "http://{CID}.ipfs.mydomain.com"
```
### Upload filters
diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
index b51d23f9e..8a01cf613 100644
--- a/lib/pleroma/upload.ex
+++ b/lib/pleroma/upload.ex
@@ -235,8 +235,8 @@ defmodule Pleroma.Upload do
""
end
- if String.contains?(base_url, "<%= cid %>") do
- EEx.eval_string(base_url, cid: path)
+ if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do
+ String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path)
else
[base_url, path]
|> Path.join()
diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex
index 722c68fa1..dde520d8e 100644
--- a/lib/pleroma/uploaders/ipfs.ex
+++ b/lib/pleroma/uploaders/ipfs.ex
@@ -9,12 +9,15 @@ defmodule Pleroma.Uploaders.IPFS do
alias Pleroma.Config
alias Tesla.Multipart
+ @placeholder "{CID}"
+ def placeholder, do: @placeholder
+
@impl true
def get_file(file) do
b_url = Pleroma.Upload.base_url()
- if String.contains?(b_url, "<%= cid %>") do
- {:ok, {:url, EEx.eval_string(b_url, cid: URI.decode(file))}}
+ if String.contains?(b_url, @placeholder) do
+ {:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}}
else
{:error, "IPFS Get URL doesn't contain 'cid' placeholder"}
end
diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
index f9ae046cf..fc87fa378 100644
--- a/test/pleroma/uploaders/ipfs_test.exs
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -17,7 +17,7 @@ defmodule Pleroma.Uploaders.IPFSTest do
clear_config(
[Pleroma.Uploaders.IPFS, :get_gateway_url],
- "https://<%= cid %>.ipfs.mydomain.com"
+ "https://{CID}.ipfs.mydomain.com"
)
clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001")
@@ -34,7 +34,7 @@ defmodule Pleroma.Uploaders.IPFSTest do
test "it returns path to ipfs file with cid as path" do
clear_config(
[Pleroma.Uploaders.IPFS, :get_gateway_url],
- "https://ipfs.mydomain.com/ipfs/<%= cid %>"
+ "https://ipfs.mydomain.com/ipfs/{CID}"
)
assert IPFS.get_file("testcid") == {
From 7c1af86f979ecebcd38995e5278fe2d59a36eda5 Mon Sep 17 00:00:00 2001
From: Claudio Maradonna
Date: Mon, 9 May 2022 12:15:40 +0200
Subject: [PATCH 022/823] ipfs: refactor final_url generation. add tests for
final_url
fix lint
---
lib/pleroma/uploaders/ipfs.ex | 19 ++++++++++---------
test/pleroma/uploaders/ipfs_test.exs | 13 ++++++++++++-
2 files changed, 22 insertions(+), 10 deletions(-)
diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex
index dde520d8e..7a7481d81 100644
--- a/lib/pleroma/uploaders/ipfs.ex
+++ b/lib/pleroma/uploaders/ipfs.ex
@@ -12,6 +12,13 @@ defmodule Pleroma.Uploaders.IPFS do
@placeholder "{CID}"
def placeholder, do: @placeholder
+ def get_final_url(method) do
+ config = Config.get([__MODULE__])
+ post_base_url = Keyword.get(config, :post_gateway_url)
+
+ Path.join([post_base_url, method])
+ end
+
@impl true
def get_file(file) do
b_url = Pleroma.Upload.base_url()
@@ -25,15 +32,12 @@ defmodule Pleroma.Uploaders.IPFS do
@impl true
def put_file(%Pleroma.Upload{} = upload) do
- config = Config.get([__MODULE__])
- post_base_url = Keyword.get(config, :post_gateway_url)
-
mp =
Multipart.new()
|> Multipart.add_content_type_param("charset=utf-8")
|> Multipart.add_file(upload.tempfile)
- final_url = Path.join([post_base_url, "/api/v0/add"])
+ final_url = get_final_url("/api/v0/add")
case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do
{:ok, ret} ->
@@ -42,7 +46,7 @@ defmodule Pleroma.Uploaders.IPFS do
if Map.has_key?(ret, "Hash") do
{:ok, {:file, ret["Hash"]}}
else
- {:error, "JSON doesn't contain Hash value"}
+ {:error, "JSON doesn't contain Hash key"}
end
error ->
@@ -58,10 +62,7 @@ defmodule Pleroma.Uploaders.IPFS do
@impl true
def delete_file(file) do
- config = Config.get([__MODULE__])
- post_base_url = Keyword.get(config, :post_gateway_url)
-
- final_url = Path.join([post_base_url, "/api/v0/files/rm"])
+ final_url = get_final_url("/api/v0/files/rm")
case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do
{:ok, %{status_code: 204}} -> :ok
diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
index fc87fa378..d567272d2 100644
--- a/test/pleroma/uploaders/ipfs_test.exs
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -23,6 +23,16 @@ defmodule Pleroma.Uploaders.IPFSTest do
clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001")
end
+ describe "get_final_url" do
+ test "it returns the final url for put_file" do
+ assert IPFS.get_final_url("/api/v0/add") == "http://localhost:5001/api/v0/add"
+ end
+
+ test "it returns the final url for delete_file" do
+ assert IPFS.get_final_url("/api/v0/files/rm") == "http://localhost:5001/api/v0/files/rm"
+ end
+ end
+
describe "get_file/1" do
test "it returns path to ipfs file with cid as subdomain" do
assert IPFS.get_file("testcid") == {
@@ -62,7 +72,8 @@ defmodule Pleroma.Uploaders.IPFSTest do
{:ok,
%Tesla.Env{
status: 200,
- body: "{\"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}"
+ body:
+ "{\"Name\":\"image-tet.jpg\",\"Size\":\"5000\", \"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}"
}}
end do
assert IPFS.put_file(file_upload) ==
From 98f268e5ecc5bab98c98270a582f8b3f0e3be4e8 Mon Sep 17 00:00:00 2001
From: Claudio Maradonna
Date: Thu, 9 Jun 2022 19:24:13 +0200
Subject: [PATCH 023/823] ipfs: small refactor and more tests
---
lib/pleroma/uploaders/ipfs.ex | 24 ++++++++++++++----------
test/pleroma/uploaders/ipfs_test.exs | 24 ++++++++++++++++++++++--
2 files changed, 36 insertions(+), 12 deletions(-)
diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex
index 7a7481d81..9f6f26e2e 100644
--- a/lib/pleroma/uploaders/ipfs.ex
+++ b/lib/pleroma/uploaders/ipfs.ex
@@ -9,16 +9,24 @@ defmodule Pleroma.Uploaders.IPFS do
alias Pleroma.Config
alias Tesla.Multipart
- @placeholder "{CID}"
- def placeholder, do: @placeholder
-
- def get_final_url(method) do
+ defp get_final_url(method) do
config = Config.get([__MODULE__])
post_base_url = Keyword.get(config, :post_gateway_url)
Path.join([post_base_url, method])
end
+ def put_file_endpoint() do
+ get_final_url("/api/v0/add")
+ end
+
+ def delete_file_endpoint() do
+ get_final_url("/api/v0/files/rm")
+ end
+
+ @placeholder "{CID}"
+ def placeholder, do: @placeholder
+
@impl true
def get_file(file) do
b_url = Pleroma.Upload.base_url()
@@ -37,9 +45,7 @@ defmodule Pleroma.Uploaders.IPFS do
|> Multipart.add_content_type_param("charset=utf-8")
|> Multipart.add_file(upload.tempfile)
- final_url = get_final_url("/api/v0/add")
-
- case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do
+ case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do
{:ok, ret} ->
case Jason.decode(ret.body) do
{:ok, ret} ->
@@ -62,9 +68,7 @@ defmodule Pleroma.Uploaders.IPFS do
@impl true
def delete_file(file) do
- final_url = get_final_url("/api/v0/files/rm")
-
- case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do
+ case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do
{:ok, %{status_code: 204}} -> :ok
error -> {:error, inspect(error)}
end
diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
index d567272d2..f2b880d9f 100644
--- a/test/pleroma/uploaders/ipfs_test.exs
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -25,11 +25,11 @@ defmodule Pleroma.Uploaders.IPFSTest do
describe "get_final_url" do
test "it returns the final url for put_file" do
- assert IPFS.get_final_url("/api/v0/add") == "http://localhost:5001/api/v0/add"
+ assert IPFS.put_file_endpoint() == "http://localhost:5001/api/v0/add"
end
test "it returns the final url for delete_file" do
- assert IPFS.get_final_url("/api/v0/files/rm") == "http://localhost:5001/api/v0/files/rm"
+ assert IPFS.delete_file_endpoint() == "http://localhost:5001/api/v0/files/rm"
end
end
@@ -88,6 +88,26 @@ defmodule Pleroma.Uploaders.IPFSTest do
end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}"
end
end
+
+ test "returns error if JSON decode fails", %{file_upload: file_upload} do
+ with_mocks([
+ {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]},
+ {Jason, [], [decode: fn _ -> {:error, "JSON decode failed"} end]}
+ ]) do
+ assert capture_log(fn ->
+ assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"}
+ end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"JSON decode failed\"}"
+ end
+ end
+
+ test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do
+ with_mocks([
+ {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]},
+ {Jason, [], [decode: fn _ -> {:ok, %{}} end]}
+ ]) do
+ assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"}
+ end
+ end
end
describe "delete_file/1" do
From 254f2ea85400ebd692fc4a45f5ac22fedd49ec09 Mon Sep 17 00:00:00 2001
From: Claudio Maradonna
Date: Thu, 9 Jun 2022 23:38:50 +0200
Subject: [PATCH 024/823] ipfs: remove unused alias
fix analysis job
---
lib/pleroma/uploaders/ipfs.ex | 4 ++--
test/pleroma/uploaders/ipfs_test.exs | 1 -
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex
index 9f6f26e2e..32e06c5cf 100644
--- a/lib/pleroma/uploaders/ipfs.ex
+++ b/lib/pleroma/uploaders/ipfs.ex
@@ -16,11 +16,11 @@ defmodule Pleroma.Uploaders.IPFS do
Path.join([post_base_url, method])
end
- def put_file_endpoint() do
+ def put_file_endpoint do
get_final_url("/api/v0/add")
end
- def delete_file_endpoint() do
+ def delete_file_endpoint do
get_final_url("/api/v0/files/rm")
end
diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
index f2b880d9f..5edb6266b 100644
--- a/test/pleroma/uploaders/ipfs_test.exs
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -6,7 +6,6 @@ defmodule Pleroma.Uploaders.IPFSTest do
use Pleroma.DataCase
alias Pleroma.Uploaders.IPFS
- alias Tesla.Multipart
import Mock
import ExUnit.CaptureLog
From 5e097eb91def0efd3cd0008309fd524fcfd88e15 Mon Sep 17 00:00:00 2001
From: Claudio Maradonna
Date: Tue, 28 Jun 2022 17:53:44 +0200
Subject: [PATCH 025/823] ipfs: better tests with @ilja suggestions
---
test/pleroma/uploaders/ipfs_test.exs | 38 ++++++++++++++++++----------
1 file changed, 25 insertions(+), 13 deletions(-)
diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
index 5edb6266b..9df3f239c 100644
--- a/test/pleroma/uploaders/ipfs_test.exs
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -6,6 +6,7 @@ defmodule Pleroma.Uploaders.IPFSTest do
use Pleroma.DataCase
alias Pleroma.Uploaders.IPFS
+ alias Tesla.Multipart
import Mock
import ExUnit.CaptureLog
@@ -62,12 +63,17 @@ defmodule Pleroma.Uploaders.IPFSTest do
tempfile: Path.absname("test/instance_static/add/shortcode.png")
}
- [file_upload: file_upload]
+ mp =
+ Multipart.new()
+ |> Multipart.add_content_type_param("charset=utf-8")
+ |> Multipart.add_file(file_upload.tempfile)
+
+ [file_upload: file_upload, mp: mp]
end
test "save file", %{file_upload: file_upload} do
with_mock Pleroma.HTTP,
- post: fn _, _, _, _ ->
+ post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
{:ok,
%Tesla.Env{
status: 200,
@@ -81,7 +87,10 @@ defmodule Pleroma.Uploaders.IPFSTest do
end
test "returns error", %{file_upload: file_upload} do
- with_mock Pleroma.HTTP, post: fn _, _, _, _ -> {:error, "IPFS Gateway upload failed"} end do
+ with_mock Pleroma.HTTP,
+ post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
+ {:error, "IPFS Gateway upload failed"}
+ end do
assert capture_log(fn ->
assert IPFS.put_file(file_upload) == {:error, "IPFS Gateway upload failed"}
end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}"
@@ -89,21 +98,22 @@ defmodule Pleroma.Uploaders.IPFSTest do
end
test "returns error if JSON decode fails", %{file_upload: file_upload} do
- with_mocks([
- {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]},
- {Jason, [], [decode: fn _ -> {:error, "JSON decode failed"} end]}
- ]) do
+ with_mock Pleroma.HTTP, [],
+ post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
+ {:ok, %Tesla.Env{status: 200, body: 'invalid'}}
+ end do
assert capture_log(fn ->
assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"}
- end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"JSON decode failed\"}"
+ end) =~
+ "Elixir.Pleroma.Uploaders.IPFS: {:error, %Jason.DecodeError{data: \"invalid\", position: 0, token: nil}}"
end
end
test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do
- with_mocks([
- {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]},
- {Jason, [], [decode: fn _ -> {:ok, %{}} end]}
- ]) do
+ with_mock Pleroma.HTTP, [],
+ post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
+ {:ok, %Tesla.Env{status: 200, body: '{"key": "value"}'}}
+ end do
assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"}
end
end
@@ -111,7 +121,9 @@ defmodule Pleroma.Uploaders.IPFSTest do
describe "delete_file/1" do
test_with_mock "deletes file", Pleroma.HTTP,
- post: fn _, _, _, _ -> {:ok, %{status_code: 204}} end do
+ post: fn "http://localhost:5001/api/v0/files/rm", "", [], params: [arg: "image.jpg"] ->
+ {:ok, %{status_code: 204}}
+ end do
assert :ok = IPFS.delete_file("image.jpg")
end
end
From 21d9091f5e422493ff69fe59db9c965e0d511369 Mon Sep 17 00:00:00 2001
From: Claudio Maradonna
Date: Fri, 8 Jul 2022 10:06:46 +0200
Subject: [PATCH 026/823] ipfs: replacing single quotes with double quotes
---
test/pleroma/uploaders/ipfs_test.exs | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
index 9df3f239c..853d185e5 100644
--- a/test/pleroma/uploaders/ipfs_test.exs
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -100,7 +100,7 @@ defmodule Pleroma.Uploaders.IPFSTest do
test "returns error if JSON decode fails", %{file_upload: file_upload} do
with_mock Pleroma.HTTP, [],
post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
- {:ok, %Tesla.Env{status: 200, body: 'invalid'}}
+ {:ok, %Tesla.Env{status: 200, body: "invalid"}}
end do
assert capture_log(fn ->
assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"}
@@ -112,7 +112,7 @@ defmodule Pleroma.Uploaders.IPFSTest do
test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do
with_mock Pleroma.HTTP, [],
post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
- {:ok, %Tesla.Env{status: 200, body: '{"key": "value"}'}}
+ {:ok, %Tesla.Env{status: 200, body: "{\"key\": \"value\"}"}}
end do
assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"}
end
From 3ed39e310939d90ddbad7bd7ffa1ebd8aca6e74c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Fri, 8 Jul 2022 21:28:23 +0200
Subject: [PATCH 027/823] Add test
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
test/pleroma/notification_test.exs | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs
index eea2fcb67..c43502eb5 100644
--- a/test/pleroma/notification_test.exs
+++ b/test/pleroma/notification_test.exs
@@ -128,6 +128,21 @@ defmodule Pleroma.NotificationTest do
subscriber_notifications = Notification.for_user(subscriber)
assert Enum.empty?(subscriber_notifications)
end
+
+ test "does not create subscriber notification if mentioned" do
+ user = insert(:user)
+ subscriber = insert(:user)
+
+ User.subscribe(subscriber, user)
+
+ {:ok, status} = CommonAPI.post(user, %{status: "mentioning @#{subscriber.nickname}"})
+ {:ok, [notification] = notifications} = Notification.create_notifications(status)
+
+ assert length(notifications) == 1
+
+ assert notification.user_id == subscriber.id
+ assert notification.type == "mention"
+ end
end
test "create_poll_notifications/1" do
From e154ebbf7933123e91d5b5c6f5070e78eb3e383b Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sun, 15 Aug 2021 21:53:04 +0300
Subject: [PATCH 028/823] Initial meilisearch implementation, doesn't delete
posts yet
---
config/config.exs | 7 ++-
config/test.exs | 2 +
lib/mix/tasks/pleroma/search/meilisearch.ex | 38 ++++++++++++
lib/pleroma/activity.ex | 1 +
lib/pleroma/activity/search.ex | 4 +-
lib/pleroma/application.ex | 6 +-
lib/pleroma/search/meilisearch.ex | 60 +++++++++++++++++++
lib/pleroma/web/activity_pub/activity_pub.ex | 6 ++
.../controllers/search_controller.ex | 5 +-
9 files changed, 123 insertions(+), 6 deletions(-)
create mode 100644 lib/mix/tasks/pleroma/search/meilisearch.ex
create mode 100644 lib/pleroma/search/meilisearch.ex
diff --git a/config/config.exs b/config/config.exs
index 4e21ce457..1df7dd44b 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -866,9 +866,14 @@ config :pleroma, Pleroma.User.Backup,
config :pleroma, ConcurrentLimiter, [
{Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},
- {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}
+ {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]},
+ {Pleroma.Search, [max_running: 20, max_waiting: 50]}
]
+config :pleroma, Pleroma.Search, module: Pleroma.Activity.Search
+
+config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/"
+
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"
diff --git a/config/test.exs b/config/test.exs
index d5c25f65e..d1c356f14 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -133,6 +133,8 @@ config :pleroma, :side_effects,
ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock,
logger: Pleroma.LoggerMock
+config :pleroma, Pleroma.Search, module: Pleroma.Activity.Search
+
# Reduce recompilation time
# https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects
config :phoenix, :plug_init_mode, :runtime
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
new file mode 100644
index 000000000..2af8e5853
--- /dev/null
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -0,0 +1,38 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
+ import Mix.Pleroma
+
+ import Ecto.Query
+
+ def run(["index"]) do
+ start_pleroma()
+
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ Pleroma.Repo.chunk_stream(
+ from(Pleroma.Object,
+ limit: 200,
+ where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0")
+ ),
+ 100,
+ :batches
+ )
+ |> Stream.map(fn objects ->
+ Enum.map(objects, fn object ->
+ data = object.data
+ %{id: object.id, source: data["source"], ap: data["id"]}
+ end)
+ end)
+ |> Stream.each(fn activities ->
+ {:ok, _} =
+ Pleroma.HTTP.post(
+ "#{endpoint}/indexes/objects/documents",
+ Jason.encode!(activities)
+ )
+ end)
+ |> Stream.run()
+ end
+end
diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index ebfd4ed45..9563136f9 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -367,6 +367,7 @@ defmodule Pleroma.Activity do
end
defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search
+ def add_to_index(_activity), do: nil
def direct_conversation_id(activity, for_user) do
alias Pleroma.Conversation.Participation
diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex
index 0b9b24aa4..3dce9d355 100644
--- a/lib/pleroma/activity/search.ex
+++ b/lib/pleroma/activity/search.ex
@@ -136,7 +136,7 @@ defmodule Pleroma.Activity.Search do
)
end
- defp maybe_restrict_local(q, user) do
+ def maybe_restrict_local(q, user) do
limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
case {limit, user} do
@@ -149,7 +149,7 @@ defmodule Pleroma.Activity.Search do
defp restrict_local(q), do: where(q, local: true)
- defp maybe_fetch(activities, user, search_query) do
+ def maybe_fetch(activities, user, search_query) do
with true <- Regex.match?(~r/https?:/, search_query),
{:ok, object} <- Fetcher.fetch_object_from_id(search_query),
%Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 1c1db8c10..62d1b8b39 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -321,7 +321,11 @@ defmodule Pleroma.Application do
def limiters_setup do
config = Config.get(ConcurrentLimiter, [])
- [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy]
+ [
+ Pleroma.Web.RichMedia.Helpers,
+ Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy,
+ Pleroma.Search
+ ]
|> Enum.each(fn module ->
mod_config = Keyword.get(config, module, [])
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
new file mode 100644
index 000000000..92e0d3429
--- /dev/null
+++ b/lib/pleroma/search/meilisearch.ex
@@ -0,0 +1,60 @@
+defmodule Pleroma.Search.Meilisearch do
+ require Logger
+
+ alias Pleroma.Activity
+
+ import Pleroma.Activity.Search
+ import Ecto.Query
+
+ def search(user, query, options \\ []) do
+ limit = Enum.min([Keyword.get(options, :limit), 40])
+ offset = Keyword.get(options, :offset, 0)
+ author = Keyword.get(options, :author)
+
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ {:ok, result} =
+ Pleroma.HTTP.post(
+ "#{endpoint}/indexes/objects/search",
+ Jason.encode!(%{q: query, offset: offset, limit: limit})
+ )
+
+ hits = Jason.decode!(result.body)["hits"] |> Enum.map(& &1["ap"])
+
+ try do
+ hits
+ |> Activity.create_by_object_ap_id()
+ |> Activity.with_preloaded_object()
+ |> Activity.with_preloaded_object()
+ |> Activity.restrict_deactivated_users()
+ |> maybe_restrict_local(user)
+ |> maybe_restrict_author(author)
+ |> maybe_restrict_blocked(user)
+ |> maybe_fetch(user, query)
+ |> order_by([activity], desc: activity.id)
+ |> Pleroma.Repo.all()
+ rescue
+ _ -> maybe_fetch([], user, query)
+ end
+ end
+
+ def add_to_index(activity) do
+ object = activity.object
+
+ if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" do
+ data = object.data
+
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ {:ok, result} =
+ Pleroma.HTTP.post(
+ "#{endpoint}/indexes/objects/documents",
+ Jason.encode!([%{id: object.id, source: data["source"], ap: data["id"]}])
+ )
+
+ if not Map.has_key?(Jason.decode!(result.body), "updateId") do
+ Logger.error("Failed to add activity #{activity.id} to index: #{result.body}")
+ end
+ end
+ end
+end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index a5d7036d9..034c3b185 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -140,6 +140,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
end)
+ search_module = Pleroma.Config.get([Pleroma.Search, :module])
+
+ ConcurrentLimiter.limit(Pleroma.Search, fn ->
+ Task.start(fn -> search_module.add_to_index(activity) end)
+ end)
+
{:ok, activity}
else
%Activity{} = activity ->
diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
index 5e6e04734..99c33eba6 100644
--- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
@@ -5,7 +5,6 @@
defmodule Pleroma.Web.MastodonAPI.SearchController do
use Pleroma.Web, :controller
- alias Pleroma.Activity
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.ControllerHelper
@@ -100,7 +99,9 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
end
defp resource_search(_, "statuses", query, options) do
- statuses = with_fallback(fn -> Activity.search(options[:for_user], query, options) end)
+ search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
+
+ statuses = with_fallback(fn -> search_module.search(options[:for_user], query, options) end)
StatusView.render("index.json",
activities: statuses,
From 0318e9a59945d7a5625111157867f0f9ebaffd91 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 16 Aug 2021 10:18:01 +0300
Subject: [PATCH 029/823] Add logging to milisiearch index and make it use
desc(id)
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 26 +++++++++++++++++----
1 file changed, 21 insertions(+), 5 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 2af8e5853..1fece96e5 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -3,8 +3,9 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
- import Mix.Pleroma
+ require Logger
+ import Mix.Pleroma
import Ecto.Query
def run(["index"]) do
@@ -12,12 +13,25 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+ {:ok, _} =
+ Pleroma.HTTP.post(
+ "#{endpoint}/indexes/objects/settings/ranking-rules",
+ Jason.encode!([
+ "desc(id)",
+ "typo",
+ "words",
+ "proximity",
+ "attribute",
+ "wordsPosition",
+ "exactness"
+ ])
+ )
+
Pleroma.Repo.chunk_stream(
from(Pleroma.Object,
- limit: 200,
where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0")
),
- 100,
+ 200,
:batches
)
|> Stream.map(fn objects ->
@@ -26,12 +40,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
%{id: object.id, source: data["source"], ap: data["id"]}
end)
end)
- |> Stream.each(fn activities ->
+ |> Stream.each(fn objects ->
{:ok, _} =
Pleroma.HTTP.post(
"#{endpoint}/indexes/objects/documents",
- Jason.encode!(activities)
+ Jason.encode!(objects)
)
+
+ IO.puts("Indexed #{Enum.count(objects)} entries")
end)
|> Stream.run()
end
From 365024abec905e427babb5403f0fccbde65f4bcd Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 16 Aug 2021 22:24:31 +0300
Subject: [PATCH 030/823] Ensure only indexing public posts and implement
clearing and delete
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 15 ++++++++++++++-
lib/pleroma/activity.ex | 1 +
lib/pleroma/search/meilisearch.ex | 17 ++++++++++++++++-
lib/pleroma/web/common_api.ex | 7 +++++++
4 files changed, 38 insertions(+), 2 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 1fece96e5..0b86fdece 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -4,6 +4,7 @@
defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
require Logger
+ require Pleroma.Constants
import Mix.Pleroma
import Ecto.Query
@@ -29,7 +30,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
Pleroma.Repo.chunk_stream(
from(Pleroma.Object,
- where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0")
+ # Only index public posts which are notes and have some text
+ where:
+ fragment("data->>'type' = 'Note'") and
+ fragment("LENGTH(data->>'source') > 0") and
+ fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public())
),
200,
:batches
@@ -51,4 +56,12 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
end)
|> Stream.run()
end
+
+ def run(["clear"]) do
+ start_pleroma()
+
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], [])
+ end
end
diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 9563136f9..2c168fd41 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -368,6 +368,7 @@ defmodule Pleroma.Activity do
defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search
def add_to_index(_activity), do: nil
+ def remove_from_index(_object), do: nil
def direct_conversation_id(activity, for_user) do
alias Pleroma.Conversation.Participation
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 92e0d3429..dbe6b2d67 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -1,5 +1,6 @@
defmodule Pleroma.Search.Meilisearch do
require Logger
+ require Pleroma.Constants
alias Pleroma.Activity
@@ -41,7 +42,8 @@ defmodule Pleroma.Search.Meilisearch do
def add_to_index(activity) do
object = activity.object
- if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" do
+ if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" and
+ Pleroma.Constants.as_public() in object.data["to"] do
data = object.data
endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
@@ -57,4 +59,17 @@ defmodule Pleroma.Search.Meilisearch do
end
end
end
+
+ def remove_from_index(object) do
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ {:ok, _} =
+ Pleroma.HTTP.request(
+ :delete,
+ "#{endpoint}/indexes/objects/documents/#{object.id}",
+ "",
+ [],
+ []
+ )
+ end
end
diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex
index 89f5dd606..54a8aa213 100644
--- a/lib/pleroma/web/common_api.ex
+++ b/lib/pleroma/web/common_api.ex
@@ -147,6 +147,13 @@ defmodule Pleroma.Web.CommonAPI do
true <- User.superuser?(user) || user.ap_id == object.data["actor"],
{:ok, delete_data, _} <- Builder.delete(user, object.data["id"]),
{:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do
+ # Also delete from search index
+ search_module = Pleroma.Config.get([Pleroma.Search, :module])
+
+ ConcurrentLimiter.limit(Pleroma.Search, fn ->
+ Task.start(fn -> search_module.remove_from_index(object) end)
+ end)
+
{:ok, delete}
else
{:find_activity, _} ->
From ea6a6a128712e81c4f298b2bb2cedfadf2295cff Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 16 Aug 2021 22:30:56 +0300
Subject: [PATCH 031/823] Make the indexing batch differently and more, show
number indexed
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 65 ++++++++++++---------
1 file changed, 39 insertions(+), 26 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 0b86fdece..2a6438528 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -28,33 +28,46 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
])
)
- Pleroma.Repo.chunk_stream(
- from(Pleroma.Object,
- # Only index public posts which are notes and have some text
- where:
- fragment("data->>'type' = 'Note'") and
- fragment("LENGTH(data->>'source') > 0") and
- fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public())
- ),
- 200,
- :batches
- )
- |> Stream.map(fn objects ->
- Enum.map(objects, fn object ->
- data = object.data
- %{id: object.id, source: data["source"], ap: data["id"]}
- end)
- end)
- |> Stream.each(fn objects ->
- {:ok, _} =
- Pleroma.HTTP.post(
- "#{endpoint}/indexes/objects/documents",
- Jason.encode!(objects)
- )
+ chunk_size = 100_000
- IO.puts("Indexed #{Enum.count(objects)} entries")
- end)
- |> Stream.run()
+ Pleroma.Repo.transaction(
+ fn ->
+ Pleroma.Repo.stream(
+ from(Pleroma.Object,
+ # Only index public posts which are notes and have some text
+ where:
+ fragment("data->>'type' = 'Note'") and
+ fragment("LENGTH(data->>'source') > 0") and
+ fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()),
+ order_by: fragment("data->'published' DESC")
+ ),
+ timeout: :infinity
+ )
+ |> Stream.chunk_every(chunk_size)
+ |> Stream.transform(0, fn objects, acc ->
+ new_acc = acc + Enum.count(objects)
+
+ IO.puts("Indexed #{new_acc} entries")
+
+ {[objects], new_acc}
+ end)
+ |> Stream.map(fn objects ->
+ Enum.map(objects, fn object ->
+ data = object.data
+ %{id: object.id, source: data["source"], ap: data["id"]}
+ end)
+ end)
+ |> Stream.each(fn objects ->
+ {:ok, _} =
+ Pleroma.HTTP.post(
+ "#{endpoint}/indexes/objects/documents",
+ Jason.encode!(objects)
+ )
+ end)
+ |> Stream.run()
+ end,
+ timeout: :infinity
+ )
end
def run(["clear"]) do
From 38996f551a4ec014e9f4cb4a691d31beecab43ba Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Tue, 17 Aug 2021 00:06:32 +0300
Subject: [PATCH 032/823] Make meilisearch sort on publish date converted to
unix time
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 2a6438528..2dd9c0a62 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -18,7 +18,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
Pleroma.HTTP.post(
"#{endpoint}/indexes/objects/settings/ranking-rules",
Jason.encode!([
- "desc(id)",
+ "desc(published)",
"typo",
"words",
"proximity",
@@ -54,7 +54,15 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
|> Stream.map(fn objects ->
Enum.map(objects, fn object ->
data = object.data
- %{id: object.id, source: data["source"], ap: data["id"]}
+
+ {:ok, published, _} = DateTime.from_iso8601(data["published"])
+
+ %{
+ id: object.id,
+ source: data["source"],
+ ap: data["id"],
+ published: published |> DateTime.to_unix()
+ }
end)
end)
|> Stream.each(fn objects ->
From 9beaebd97e1746df010aecfcc01d9e2e9a4c60ac Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Tue, 17 Aug 2021 00:30:14 +0300
Subject: [PATCH 033/823] Tweak search ordering to hopefully return newer
results
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 15 ++++++++++++---
lib/pleroma/search/meilisearch.ex | 13 +++++++++++--
2 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 2dd9c0a62..dcecbd7cf 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -39,7 +39,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
fragment("data->>'type' = 'Note'") and
fragment("LENGTH(data->>'source') > 0") and
fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()),
- order_by: fragment("data->'published' DESC")
+ order_by: [desc: fragment("data->'published'")]
),
timeout: :infinity
)
@@ -66,11 +66,15 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
end)
end)
|> Stream.each(fn objects ->
- {:ok, _} =
+ {:ok, result} =
Pleroma.HTTP.post(
"#{endpoint}/indexes/objects/documents",
Jason.encode!(objects)
)
+
+ if not Map.has_key?(Jason.decode!(result.body), "updateId") do
+ IO.puts("Failed to index: #{result}")
+ end
end)
|> Stream.run()
end,
@@ -83,6 +87,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
- {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], [])
+ {:ok, result} =
+ Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], [])
+
+ if not Map.has_key?(Jason.decode!(result.body), "updateId") do
+ IO.puts("Failed to clear: #{result}")
+ end
end
end
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index dbe6b2d67..9fdb0a07f 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -32,7 +32,7 @@ defmodule Pleroma.Search.Meilisearch do
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
|> maybe_fetch(user, query)
- |> order_by([activity], desc: activity.id)
+ |> order_by([object: obj], desc: obj.data["published"])
|> Pleroma.Repo.all()
rescue
_ -> maybe_fetch([], user, query)
@@ -48,10 +48,19 @@ defmodule Pleroma.Search.Meilisearch do
endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+ {:ok, published, _} = DateTime.from_iso8601(data["published"])
+
{:ok, result} =
Pleroma.HTTP.post(
"#{endpoint}/indexes/objects/documents",
- Jason.encode!([%{id: object.id, source: data["source"], ap: data["id"]}])
+ Jason.encode!([
+ %{
+ id: object.id,
+ source: data["source"],
+ ap: data["id"],
+ published: published |> DateTime.to_unix()
+ }
+ ])
)
if not Map.has_key?(Jason.decode!(result.body), "updateId") do
From 00c48a33acf0bd59fa7e7b58a67b049e4f4adc31 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Tue, 17 Aug 2021 00:57:53 +0300
Subject: [PATCH 034/823] Use content instead of source and scrub it
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 12 ++++--------
lib/pleroma/search/meilisearch.ex | 2 +-
2 files changed, 5 insertions(+), 9 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index dcecbd7cf..5270de255 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -37,7 +37,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
# Only index public posts which are notes and have some text
where:
fragment("data->>'type' = 'Note'") and
- fragment("LENGTH(data->>'source') > 0") and
+ fragment("LENGTH(data->>'content') > 0") and
fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()),
order_by: [desc: fragment("data->'published'")]
),
@@ -56,10 +56,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
data = object.data
{:ok, published, _} = DateTime.from_iso8601(data["published"])
+ {:ok, content} = FastSanitize.strip_tags(data["content"])
%{
id: object.id,
- source: data["source"],
+ content: content,
ap: data["id"],
published: published |> DateTime.to_unix()
}
@@ -87,11 +88,6 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
- {:ok, result} =
- Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], [])
-
- if not Map.has_key?(Jason.decode!(result.body), "updateId") do
- IO.puts("Failed to clear: #{result}")
- end
+ {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects", "", [], [])
end
end
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 9fdb0a07f..87fdeaf5e 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -56,7 +56,7 @@ defmodule Pleroma.Search.Meilisearch do
Jason.encode!([
%{
id: object.id,
- source: data["source"],
+ content: data["content"] |> Pleroma.HTML.filter_tags(),
ap: data["id"],
published: published |> DateTime.to_unix()
}
From e35d87ea54f70a39206f6103ef0e7334e2a428cc Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Tue, 17 Aug 2021 01:37:43 +0300
Subject: [PATCH 035/823] Make the chunk size smaller
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 5270de255..44af25f3e 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -28,7 +28,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
])
)
- chunk_size = 100_000
+ chunk_size = 10_000
Pleroma.Repo.transaction(
fn ->
From 2b2e409ad72862967cabf06344874ae9bff9860f Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sun, 22 Aug 2021 16:37:52 +0300
Subject: [PATCH 036/823] Also index incoming federated posts
---
lib/pleroma/search/search.ex | 18 ++++++++++++++++++
lib/pleroma/web/activity_pub/activity_pub.ex | 7 ++-----
lib/pleroma/web/activity_pub/side_effects.ex | 7 +++++++
lib/pleroma/web/common_api.ex | 8 ++------
4 files changed, 29 insertions(+), 11 deletions(-)
create mode 100644 lib/pleroma/search/search.ex
diff --git a/lib/pleroma/search/search.ex b/lib/pleroma/search/search.ex
new file mode 100644
index 000000000..e363abf19
--- /dev/null
+++ b/lib/pleroma/search/search.ex
@@ -0,0 +1,18 @@
+defmodule Pleroma.Search do
+ def add_to_index(activity) do
+ search_module = Pleroma.Config.get([Pleroma.Search, :module])
+
+ ConcurrentLimiter.limit(Pleroma.Search, fn ->
+ Task.start(fn -> search_module.add_to_index(activity) end)
+ end)
+ end
+
+ def remove_from_index(object) do
+ # Also delete from search index
+ search_module = Pleroma.Config.get([Pleroma.Search, :module])
+
+ ConcurrentLimiter.limit(Pleroma.Search, fn ->
+ Task.start(fn -> search_module.remove_from_index(object) end)
+ end)
+ end
+end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 034c3b185..7178cf9eb 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -140,11 +140,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
end)
- search_module = Pleroma.Config.get([Pleroma.Search, :module])
-
- ConcurrentLimiter.limit(Pleroma.Search, fn ->
- Task.start(fn -> search_module.add_to_index(activity) end)
- end)
+ # Add local posts to search index
+ Pleroma.Search.add_to_index(activity)
{:ok, activity}
else
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 5eefd2824..15e006b18 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -197,6 +197,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
# - Increase replies count
# - Set up ActivityExpiration
# - Set up notifications
+ # - Index incoming posts for search (if needed)
@impl true
def handle(%{data: %{"type" => "Create"}} = activity, meta) do
with {:ok, object, meta} <- handle_object_creation(meta[:object_data], activity, meta),
@@ -226,6 +227,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
end)
+ Pleroma.Search.add_to_index(Map.put(activity, :object, object))
+
meta =
meta
|> add_notifications(notifications)
@@ -286,6 +289,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
# - Reduce the user note count
# - Reduce the reply count
# - Stream out the activity
+ # - Removes posts from search index (if needed)
@impl true
def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, meta) do
deleted_object =
@@ -325,6 +329,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
if result == :ok do
Notification.create_notifications(object)
+
+ Pleroma.Search.remove_from_index(object)
+
{:ok, object, meta}
else
{:error, result}
diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex
index 54a8aa213..ba6c07975 100644
--- a/lib/pleroma/web/common_api.ex
+++ b/lib/pleroma/web/common_api.ex
@@ -147,12 +147,8 @@ defmodule Pleroma.Web.CommonAPI do
true <- User.superuser?(user) || user.ap_id == object.data["actor"],
{:ok, delete_data, _} <- Builder.delete(user, object.data["id"]),
{:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do
- # Also delete from search index
- search_module = Pleroma.Config.get([Pleroma.Search, :module])
-
- ConcurrentLimiter.limit(Pleroma.Search, fn ->
- Task.start(fn -> search_module.remove_from_index(object) end)
- end)
+ # Remove from search index for local posts
+ Pleroma.Search.remove_from_index(object)
{:ok, delete}
else
From 9f16ca80e0fe60b8b0e3e8ddb9b06ca0bec31002 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sun, 22 Aug 2021 18:47:41 +0300
Subject: [PATCH 037/823] Mark only content as searchable for meilisearch
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 44af25f3e..ebd3cc81f 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -28,6 +28,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
])
)
+ {:ok, _} =
+ Pleroma.HTTP.post(
+ "#{endpoint}/indexes/objects/settings/searchable-attributes",
+ Jason.encode!([
+ "content"
+ ])
+ )
+
chunk_size = 10_000
Pleroma.Repo.transaction(
@@ -55,8 +63,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
Enum.map(objects, fn object ->
data = object.data
+ content_str =
+ case data["content"] do
+ [nil | rest] -> to_string(rest)
+ str -> str
+ end
+
{:ok, published, _} = DateTime.from_iso8601(data["published"])
- {:ok, content} = FastSanitize.strip_tags(data["content"])
+ {:ok, content} = FastSanitize.strip_tags(content_str)
%{
id: object.id,
From 3dedadf192a3acd0c1dfc2b11eba5a247ae7f61c Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sun, 22 Aug 2021 19:38:03 +0300
Subject: [PATCH 038/823] Adjust content indexing to skip more unneeded stuff
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 45 ++++++++++++++-------
1 file changed, 30 insertions(+), 15 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index ebd3cc81f..3704e0bdc 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -52,13 +52,6 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
timeout: :infinity
)
|> Stream.chunk_every(chunk_size)
- |> Stream.transform(0, fn objects, acc ->
- new_acc = acc + Enum.count(objects)
-
- IO.puts("Indexed #{new_acc} entries")
-
- {[objects], new_acc}
- end)
|> Stream.map(fn objects ->
Enum.map(objects, fn object ->
data = object.data
@@ -70,15 +63,34 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
end
{:ok, published, _} = DateTime.from_iso8601(data["published"])
- {:ok, content} = FastSanitize.strip_tags(content_str)
- %{
- id: object.id,
- content: content,
- ap: data["id"],
- published: published |> DateTime.to_unix()
- }
+ content =
+ with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str),
+ trimmed <- String.trim(scrubbed) do
+ trimmed
+ end
+
+ # Only index if there is anything in the string. If there is a single symbol,
+ # it's probably a dot from mastodon posts with just the picture
+ if String.length(content) > 1 do
+ %{
+ id: object.id,
+ content: content,
+ ap: data["id"],
+ published: published |> DateTime.to_unix()
+ }
+ else
+ nil
+ end
end)
+ |> Enum.filter(fn o -> not is_nil(o) end)
+ end)
+ |> Stream.transform(0, fn objects, acc ->
+ new_acc = acc + Enum.count(objects)
+
+ IO.puts("Indexed #{new_acc} entries")
+
+ {[objects], new_acc}
end)
|> Stream.each(fn objects ->
{:ok, result} =
@@ -102,6 +114,9 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
- {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects", "", [], [])
+ {:ok, _} =
+ Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [],
+ timeout: :infinity
+ )
end
end
From 35e9192cedcbc56fb07c9933e2988bf900256b53 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sun, 22 Aug 2021 22:53:18 +0300
Subject: [PATCH 039/823] Rework task indexing to share code with the main
module
The code in the main module now scrubs new posts too
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 35 +---------------
lib/pleroma/search/meilisearch.ex | 46 ++++++++++++++-------
2 files changed, 34 insertions(+), 47 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 3704e0bdc..b5a394e34 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -51,40 +51,9 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
),
timeout: :infinity
)
+ |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1)
+ |> Stream.filter(fn o -> not is_nil(o) end)
|> Stream.chunk_every(chunk_size)
- |> Stream.map(fn objects ->
- Enum.map(objects, fn object ->
- data = object.data
-
- content_str =
- case data["content"] do
- [nil | rest] -> to_string(rest)
- str -> str
- end
-
- {:ok, published, _} = DateTime.from_iso8601(data["published"])
-
- content =
- with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str),
- trimmed <- String.trim(scrubbed) do
- trimmed
- end
-
- # Only index if there is anything in the string. If there is a single symbol,
- # it's probably a dot from mastodon posts with just the picture
- if String.length(content) > 1 do
- %{
- id: object.id,
- content: content,
- ap: data["id"],
- published: published |> DateTime.to_unix()
- }
- else
- nil
- end
- end)
- |> Enum.filter(fn o -> not is_nil(o) end)
- end)
|> Stream.transform(0, fn objects, acc ->
new_acc = acc + Enum.count(objects)
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 87fdeaf5e..10468e36c 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -39,28 +39,46 @@ defmodule Pleroma.Search.Meilisearch do
end
end
- def add_to_index(activity) do
- object = activity.object
-
- if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" and
+ def object_to_search_data(object) do
+ if not is_nil(object) and object.data["type"] == "Note" and
Pleroma.Constants.as_public() in object.data["to"] do
data = object.data
- endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+ content_str =
+ case data["content"] do
+ [nil | rest] -> to_string(rest)
+ str -> str
+ end
- {:ok, published, _} = DateTime.from_iso8601(data["published"])
+ content =
+ with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str),
+ trimmed <- String.trim(scrubbed) do
+ trimmed
+ end
+
+ if String.length(content) > 1 do
+ {:ok, published, _} = DateTime.from_iso8601(data["published"])
+
+ %{
+ id: object.id,
+ content: content,
+ ap: data["id"],
+ published: published |> DateTime.to_unix()
+ }
+ end
+ end
+ end
+
+ def add_to_index(activity) do
+ maybe_search_data = object_to_search_data(activity)
+
+ if activity.data["type"] == "Create" and maybe_search_data do
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
{:ok, result} =
Pleroma.HTTP.post(
"#{endpoint}/indexes/objects/documents",
- Jason.encode!([
- %{
- id: object.id,
- content: data["content"] |> Pleroma.HTML.filter_tags(),
- ap: data["id"],
- published: published |> DateTime.to_unix()
- }
- ])
+ Jason.encode!([maybe_search_data])
)
if not Map.has_key?(Jason.decode!(result.body), "updateId") do
From 410c8cb765bbec1014cb2bbdbcc44d3a25f834e1 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sun, 22 Aug 2021 23:47:43 +0300
Subject: [PATCH 040/823] Make indexing logs rewrite themselves
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index b5a394e34..2485a441d 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -57,7 +57,9 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
|> Stream.transform(0, fn objects, acc ->
new_acc = acc + Enum.count(objects)
- IO.puts("Indexed #{new_acc} entries")
+ # Reset to the beginning of the line and rewrite it
+ IO.write("\r")
+ IO.write("Indexed #{new_acc} entries")
{[objects], new_acc}
end)
@@ -76,6 +78,8 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
end,
timeout: :infinity
)
+
+ IO.write("\n")
end
def run(["clear"]) do
From 2c7d973af7797ae860829c1764ade521a17e7263 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 23 Aug 2021 19:35:21 +0300
Subject: [PATCH 041/823] Implement meilisearch auth
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 75 ++++++++++++---------
lib/pleroma/search/meilisearch.ex | 69 ++++++++++++-------
2 files changed, 88 insertions(+), 56 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 2485a441d..230be5aa1 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -9,32 +9,30 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
import Mix.Pleroma
import Ecto.Query
+ import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1]
+
def run(["index"]) do
start_pleroma()
- endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+ meili_post!(
+ "/indexes/objects/settings/ranking-rules",
+ [
+ "desc(published)",
+ "typo",
+ "words",
+ "proximity",
+ "attribute",
+ "wordsPosition",
+ "exactness"
+ ]
+ )
- {:ok, _} =
- Pleroma.HTTP.post(
- "#{endpoint}/indexes/objects/settings/ranking-rules",
- Jason.encode!([
- "desc(published)",
- "typo",
- "words",
- "proximity",
- "attribute",
- "wordsPosition",
- "exactness"
- ])
- )
-
- {:ok, _} =
- Pleroma.HTTP.post(
- "#{endpoint}/indexes/objects/settings/searchable-attributes",
- Jason.encode!([
- "content"
- ])
- )
+ meili_post!(
+ "/indexes/objects/settings/searchable-attributes",
+ [
+ "content"
+ ]
+ )
chunk_size = 10_000
@@ -64,14 +62,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
{[objects], new_acc}
end)
|> Stream.each(fn objects ->
- {:ok, result} =
- Pleroma.HTTP.post(
- "#{endpoint}/indexes/objects/documents",
- Jason.encode!(objects)
+ result =
+ meili_post!(
+ "/indexes/objects/documents",
+ objects
)
- if not Map.has_key?(Jason.decode!(result.body), "updateId") do
- IO.puts("Failed to index: #{result}")
+ if not Map.has_key?(result, "updateId") do
+ IO.puts("Failed to index: #{inspect(result)}")
end
end)
|> Stream.run()
@@ -85,11 +83,26 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
def run(["clear"]) do
start_pleroma()
+ meili_delete!("/indexes/objects/documents")
+ end
+
+ def run(["show-private-key", master_key]) do
+ start_pleroma()
+
endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
- {:ok, _} =
- Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [],
- timeout: :infinity
+ {:ok, result} =
+ Pleroma.HTTP.get(
+ Path.join(endpoint, "/keys"),
+ [{"X-Meili-API-Key", master_key}]
)
+
+ decoded = Jason.decode!(result.body)
+
+ if decoded["private"] do
+ IO.puts(decoded["private"])
+ else
+ IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}")
+ end
end
end
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 10468e36c..8745d539d 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -7,20 +7,50 @@ defmodule Pleroma.Search.Meilisearch do
import Pleroma.Activity.Search
import Ecto.Query
+ defp meili_headers() do
+ private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key])
+
+ if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}]
+ end
+
+ def meili_post!(path, params) do
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ {:ok, result} =
+ Pleroma.HTTP.post(
+ Path.join(endpoint, path),
+ Jason.encode!(params),
+ meili_headers()
+ )
+
+ Jason.decode!(result.body)
+ end
+
+ def meili_delete!(path) do
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ {:ok, _} =
+ Pleroma.HTTP.request(
+ :delete,
+ Path.join(endpoint, path),
+ "",
+ meili_headers(),
+ timeout: :infinity
+ )
+ end
+
def search(user, query, options \\ []) do
limit = Enum.min([Keyword.get(options, :limit), 40])
offset = Keyword.get(options, :offset, 0)
author = Keyword.get(options, :author)
- endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
-
- {:ok, result} =
- Pleroma.HTTP.post(
- "#{endpoint}/indexes/objects/search",
- Jason.encode!(%{q: query, offset: offset, limit: limit})
+ result =
+ meili_post!(
+ "/indexes/objects/search",
+ %{q: query, offset: offset, limit: limit}
)
- hits = Jason.decode!(result.body)["hits"] |> Enum.map(& &1["ap"])
+ hits = result["hits"] |> Enum.map(& &1["ap"])
try do
hits
@@ -73,30 +103,19 @@ defmodule Pleroma.Search.Meilisearch do
maybe_search_data = object_to_search_data(activity)
if activity.data["type"] == "Create" and maybe_search_data do
- endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
-
- {:ok, result} =
- Pleroma.HTTP.post(
- "#{endpoint}/indexes/objects/documents",
- Jason.encode!([maybe_search_data])
+ result =
+ meili_post!(
+ "/indexes/objects/documents",
+ [maybe_search_data]
)
- if not Map.has_key?(Jason.decode!(result.body), "updateId") do
- Logger.error("Failed to add activity #{activity.id} to index: #{result.body}")
+ if not Map.has_key?(result, "updateId") do
+ Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}")
end
end
end
def remove_from_index(object) do
- endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
-
- {:ok, _} =
- Pleroma.HTTP.request(
- :delete,
- "#{endpoint}/indexes/objects/documents/#{object.id}",
- "",
- [],
- []
- )
+ meili_delete!("/indexes/objects/documents/#{object.id}")
end
end
From a67f9da5cc46b4e184aa1afe3dd1bd1df31de15b Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 23 Aug 2021 20:02:34 +0300
Subject: [PATCH 042/823] Add a message with a count of posts to index
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 230be5aa1..557b06182 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -38,7 +38,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
Pleroma.Repo.transaction(
fn ->
- Pleroma.Repo.stream(
+ query =
from(Pleroma.Object,
# Only index public posts which are notes and have some text
where:
@@ -46,7 +46,13 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
fragment("LENGTH(data->>'content') > 0") and
fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()),
order_by: [desc: fragment("data->'published'")]
- ),
+ )
+
+ count = query |> Pleroma.Repo.aggregate(:count, :data)
+ IO.puts("Entries to index: #{count}")
+
+ Pleroma.Repo.stream(
+ query,
timeout: :infinity
)
|> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1)
From 09a1ae1b6eca4efbb935aa1c0da950009d110fb2 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 23 Aug 2021 20:21:46 +0300
Subject: [PATCH 043/823] Add the meilisearch.stats command
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 10 +++++++++-
lib/pleroma/search/meilisearch.ex | 12 ++++++++++++
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 557b06182..f2d9fe312 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -9,7 +9,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
import Mix.Pleroma
import Ecto.Query
- import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1]
+ import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1]
def run(["index"]) do
start_pleroma()
@@ -111,4 +111,12 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}")
end
end
+
+ def run(["stats"]) do
+ start_pleroma()
+
+ result = meili_get!("/indexes/objects/stats")
+ IO.puts("Number of entries: #{result["numberOfDocuments"]}")
+ IO.puts("Indexing? #{result["isIndexing"]}")
+ end
end
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 8745d539d..1ad17bf9f 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -13,6 +13,18 @@ defmodule Pleroma.Search.Meilisearch do
if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}]
end
+ def meili_get!(path) do
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ {:ok, result} =
+ Pleroma.HTTP.get(
+ Path.join(endpoint, path),
+ meili_headers()
+ )
+
+ Jason.decode!(result.body)
+ end
+
def meili_post!(path, params) do
endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
From 07ccab9766a6289326676a4814537564f25f35fa Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 23 Aug 2021 20:27:16 +0300
Subject: [PATCH 044/823] Add search/meilisearch documentation
---
docs/configuration/search.md | 99 ++++++++++++++++++++++++++++++++++++
1 file changed, 99 insertions(+)
create mode 100644 docs/configuration/search.md
diff --git a/docs/configuration/search.md b/docs/configuration/search.md
new file mode 100644
index 000000000..14ec2bc63
--- /dev/null
+++ b/docs/configuration/search.md
@@ -0,0 +1,99 @@
+# Configuring search
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+## Built-in search
+
+To use built-in search that has no external dependencies, set the search module to `Pleroma.Activity`:
+
+> config :pleroma, Pleroma.Search, module: Pleroma.Activity
+
+While it has no external dependencies, it has problems with performance and relevancy.
+
+## Meilisearch
+
+To use [meilisearch](https://www.meilisearch.com/), set the search module to `Pleroma.Search.Meilisearch`:
+
+> config :pleroma, Pleroma.Search, module: Pleroma.Search.Meilisearch
+
+You then need to set the address of the meilisearch instance, and optionally the private key for authentication.
+
+> config :pleroma, Pleroma.Search.Meilisearch,
+> url: "http://127.0.0.1:7700/",
+> private_key: "private key"
+
+Information about setting up meilisearch can be found in the
+[official documentation](https://docs.meilisearch.com/learn/getting_started/installation.html).
+You probably want to start it with `MEILI_NO_ANALYTICS=true` and `MEILI_NO_CENTRY=true` environment variables,
+to disable analytics.
+
+### Private key authentication (optional)
+
+To set the private key, use the `MEILI_MASTER_KEY` environment variable when starting. After setting the _master key_,
+you have to get the _private key_, which is actually used for authentication.
+
+=== "OTP"
+ ```sh
+ ./bin/pleroma_ctl search.meilisearch show-private-key
+ ```
+
+=== "From Source"
+ ```sh
+ mix pleroma.search.meilisearch show-private-key
+ ```
+
+This is the key you actually put into your configuration file.
+
+### Initial indexing
+
+After setting up the configuration, you'll want to index all of your already existsing posts. Only public posts are indexed. You'll only
+have to do it one time, but it might take a while, depending on the amount of posts your instance has seen. This is also a fairly RAM
+consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2
+million posts while idle and up to 7G while indexing initially, but your experience may be different).
+
+To start te initial indexing, run the `index` command:
+
+=== "OTP"
+ ```sh
+ ./bin/pleroma_ctl search.meilisearch index
+ ```
+
+=== "From Source"
+ ```sh
+ mix pleroma.search.meilisearch index
+ ```
+
+This will show you the total amount of posts to index, and then show you the amount of posts indexed currently, until the numbers eventually
+become the same. The posts are indexed in big batches and meilisearch will take some time to actually index them, even after you have
+inserted all the posts into it. Depending on the amount of posts, this may be as long as several hours. To get information about the status
+of indexing and how many posts have actually been indexed, use the `stats` command:
+
+=== "OTP"
+ ```sh
+ ./bin/pleroma_ctl search.meilisearch stats
+ ```
+
+=== "From Source"
+ ```sh
+ mix pleroma.search.meilisearch stats
+ ```
+
+### Clearing the index
+
+In case you need to clear the index (for example, to re-index from scratch, if that needs to happen for some reason), you can
+use the `clear` command:
+
+=== "OTP"
+ ```sh
+ ./bin/pleroma_ctl search.meilisearch clear
+ ```
+
+=== "From Source"
+ ```sh
+ mix pleroma.search.meilisearch clear
+ ```
+
+This will clear **all** the posts from the search index. Note, that deleted posts are also removed from index by the instance itself, so
+there is no need to actually clear the whole index, unless you want **all** of it gone. That said, the index does not hold any information
+that cannot be re-created from the database, it should also generally be a lot smaller than the size of your database. Still, the size
+depends on the amount of text in posts.
From d9ef7e075880ba39dd4ca8e21566c680070faa42 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 23 Aug 2021 21:15:15 +0300
Subject: [PATCH 045/823] Fix activity being passed to objec_to_search_data
---
lib/pleroma/search/meilisearch.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 1ad17bf9f..212bdd473 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -112,7 +112,7 @@ defmodule Pleroma.Search.Meilisearch do
end
def add_to_index(activity) do
- maybe_search_data = object_to_search_data(activity)
+ maybe_search_data = object_to_search_data(activity.object)
if activity.data["type"] == "Create" and maybe_search_data do
result =
From 005947e9f77b40d1b6dd6c05f952df6ecb2aa1fc Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 23 Aug 2021 23:52:21 +0300
Subject: [PATCH 046/823] Add tests for local post indexing for meilisearch
---
config/test.exs | 4 +-
test/pleroma/search/meilisearch_test.exs | 108 +++++++++++++++++++++++
2 files changed, 111 insertions(+), 1 deletion(-)
create mode 100644 test/pleroma/search/meilisearch_test.exs
diff --git a/config/test.exs b/config/test.exs
index d1c356f14..c9b2b51ba 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -133,7 +133,9 @@ config :pleroma, :side_effects,
ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock,
logger: Pleroma.LoggerMock
-config :pleroma, Pleroma.Search, module: Pleroma.Activity.Search
+config :pleroma, Pleroma.Search, module: Pleroma.Activity
+
+config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil
# Reduce recompilation time
# https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects
diff --git a/test/pleroma/search/meilisearch_test.exs b/test/pleroma/search/meilisearch_test.exs
new file mode 100644
index 000000000..6e13c8edf
--- /dev/null
+++ b/test/pleroma/search/meilisearch_test.exs
@@ -0,0 +1,108 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Search.MeilisearchTest do
+ require Pleroma.Constants
+
+ use Pleroma.DataCase
+
+ import Pleroma.Factory
+ import Tesla.Mock
+ import Mock
+
+ alias Pleroma.Web.CommonAPI
+ alias Pleroma.Search.Meilisearch
+
+ setup_all do
+ Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+ :ok
+ end
+
+ describe "meilisearch" do
+ setup do: clear_config([Pleroma.Search, :module], Meilisearch)
+
+ setup_with_mocks(
+ [
+ {Meilisearch, [:passthrough],
+ [
+ add_to_index: fn a -> passthrough([a]) end,
+ remove_from_index: fn a -> passthrough([a]) end
+ ]}
+ ],
+ context,
+ do: {:ok, context}
+ )
+
+ test "indexes a local post on creation" do
+ user = insert(:user)
+
+ mock_global(fn
+ %{method: :post, url: "http://127.0.0.1:7700/indexes/objects/documents", body: body} ->
+ assert match?(
+ [%{"content" => "guys i just don't wanna leave the swamp"}],
+ Jason.decode!(body)
+ )
+
+ json(%{updateId: 1})
+ end)
+
+ {:ok, activity} =
+ CommonAPI.post(user, %{
+ status: "guys i just don't wanna leave the swamp",
+ visibility: "public"
+ })
+
+ assert_called(Meilisearch.add_to_index(activity))
+ end
+
+ test "doesn't index posts that are not public" do
+ user = insert(:user)
+
+ Enum.each(["unlisted", "private", "direct"], fn visiblity ->
+ {:ok, _} =
+ CommonAPI.post(user, %{
+ status: "guys i just don't wanna leave the swamp",
+ visibility: visiblity
+ })
+ end)
+
+ history = call_history(Meilisearch)
+ assert Enum.count(history) == 3
+
+ Enum.each(history, fn {_, _, return} ->
+ assert is_nil(return)
+ end)
+ end
+
+ test "deletes posts from index when deleted locally" do
+ user = insert(:user)
+
+ mock_global(fn
+ %{method: :post, url: "http://127.0.0.1:7700/indexes/objects/documents", body: body} ->
+ assert match?(
+ [%{"content" => "guys i just don't wanna leave the swamp"}],
+ Jason.decode!(body)
+ )
+
+ json(%{updateId: 1})
+
+ %{method: :delete, url: "http://127.0.0.1:7700/indexes/objects/documents/" <> id} ->
+ assert String.length(id) > 1
+ json(%{updateId: 2})
+ end)
+
+ {:ok, activity} =
+ CommonAPI.post(user, %{
+ status: "guys i just don't wanna leave the swamp",
+ visibility: "public"
+ })
+
+ assert_called(Meilisearch.add_to_index(activity))
+
+ {:ok, _} = CommonAPI.delete(activity.id, user)
+
+ assert_called(Meilisearch.remove_from_index(:_))
+ end
+ end
+end
From a5bb7f9345ff73469c0d776bce5455ec4f27b4ee Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 23 Aug 2021 23:52:37 +0300
Subject: [PATCH 047/823] Add private_key: nil to default meilisearch options
---
config/config.exs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/config/config.exs b/config/config.exs
index 1df7dd44b..711775982 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -872,7 +872,7 @@ config :pleroma, ConcurrentLimiter, [
config :pleroma, Pleroma.Search, module: Pleroma.Activity.Search
-config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/"
+config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
From 40280cc273ad7f2b355846e2f41b9873a8d5ff2c Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sat, 28 Aug 2021 15:59:13 +0300
Subject: [PATCH 048/823] Reorder ranking rules for (maybe) better results
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index f2d9fe312..cdf9ab0bd 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -18,12 +18,12 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
"/indexes/objects/settings/ranking-rules",
[
"desc(published)",
- "typo",
"words",
+ "exactness",
"proximity",
- "attribute",
"wordsPosition",
- "exactness"
+ "typo",
+ "attribute"
]
)
From 6beef2d1179ab9a377e87872b7fbe2997bbbbebd Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Fri, 8 Oct 2021 12:24:37 +0300
Subject: [PATCH 049/823] Move add_to_index / remove_from_index to
Pleroma.Actitivy.Search
---
lib/pleroma/activity.ex | 2 --
lib/pleroma/activity/search.ex | 3 +++
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 2c168fd41..ebfd4ed45 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -367,8 +367,6 @@ defmodule Pleroma.Activity do
end
defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search
- def add_to_index(_activity), do: nil
- def remove_from_index(_object), do: nil
def direct_conversation_id(activity, for_user) do
alias Pleroma.Conversation.Participation
diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex
index 3dce9d355..47ab5208c 100644
--- a/lib/pleroma/activity/search.ex
+++ b/lib/pleroma/activity/search.ex
@@ -45,6 +45,9 @@ defmodule Pleroma.Activity.Search do
end
end
+ def add_to_index(_activity), do: nil
+ def remove_from_index(_object), do: nil
+
def maybe_restrict_author(query, %User{} = author) do
Activity.Queries.by_author(query, author)
end
From c569ad05b3d812c87171e68eac79eec749321033 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Tue, 12 Oct 2021 19:14:39 +0300
Subject: [PATCH 050/823] Add more documentation about rum to meilisearch docs
---
docs/configuration/search.md | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index 14ec2bc63..e9743f1a4 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -12,6 +12,15 @@ While it has no external dependencies, it has problems with performance and rele
## Meilisearch
+Note that it's quite a bit more memory hungry than PostgreSQL (around 4-5G for ~1.2 million
+posts while idle and up to 7G while indexing initially). The disk usage for this additional index is also
+around 4 gigabytes. Like [RUM](./cheatsheet.md#rum-indexing-for-full-text-search) indexes, it offers considerably
+higher performance and ordering by timestamp in a reasonable amount of time.
+Additionally, the search results seem to be more accurate.
+
+Due to high memory usage, it may be best to set it up on a different machine, if running pleroma on a low-resource
+computer, and use private key authentication to secure the remote search instance.
+
To use [meilisearch](https://www.meilisearch.com/), set the search module to `Pleroma.Search.Meilisearch`:
> config :pleroma, Pleroma.Search, module: Pleroma.Search.Meilisearch
From 95cb2bb694e3f8857895b21331b02b9277d65d9b Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Tue, 12 Oct 2021 19:17:37 +0300
Subject: [PATCH 051/823] Don't try removing from index again in common_api
It's already removed in the side effects of the pipeline
---
lib/pleroma/web/common_api.ex | 3 ---
1 file changed, 3 deletions(-)
diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex
index ba6c07975..89f5dd606 100644
--- a/lib/pleroma/web/common_api.ex
+++ b/lib/pleroma/web/common_api.ex
@@ -147,9 +147,6 @@ defmodule Pleroma.Web.CommonAPI do
true <- User.superuser?(user) || user.ap_id == object.data["actor"],
{:ok, delete_data, _} <- Builder.delete(user, object.data["id"]),
{:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do
- # Remove from search index for local posts
- Pleroma.Search.remove_from_index(object)
-
{:ok, delete}
else
{:find_activity, _} ->
From cf558208c202d5188954e26077d35bcc1ae02fce Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Tue, 12 Oct 2021 19:34:57 +0300
Subject: [PATCH 052/823] Use proper deleted object for removing from index
---
lib/pleroma/web/activity_pub/side_effects.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 15e006b18..4762b5ac6 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -330,7 +330,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
if result == :ok do
Notification.create_notifications(object)
- Pleroma.Search.remove_from_index(object)
+ Pleroma.Search.remove_from_index(deleted_object)
{:ok, object, meta}
else
From e4b7a3f51f270f468c15cc4ce850c847633c030b Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Fri, 29 Oct 2021 00:38:00 +0300
Subject: [PATCH 053/823] Modify some meilisearch variables
---
config/config.exs | 2 +-
lib/pleroma/search/meilisearch.ex | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/config/config.exs b/config/config.exs
index 711775982..b55dd94cb 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -867,7 +867,7 @@ config :pleroma, Pleroma.User.Backup,
config :pleroma, ConcurrentLimiter, [
{Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},
{Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]},
- {Pleroma.Search, [max_running: 20, max_waiting: 50]}
+ {Pleroma.Search, [max_running: 30, max_waiting: 50]}
]
config :pleroma, Pleroma.Search, module: Pleroma.Activity.Search
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 212bdd473..b8248e40c 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -47,7 +47,7 @@ defmodule Pleroma.Search.Meilisearch do
Path.join(endpoint, path),
"",
meili_headers(),
- timeout: :infinity
+ []
)
end
From 0b4fd0d342e3ced073e82355b380cbfee5478c60 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Fri, 29 Oct 2021 13:58:24 +0300
Subject: [PATCH 054/823] Set content-type to application/json
---
lib/pleroma/search/meilisearch.ex | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index b8248e40c..d94ab8b64 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -10,7 +10,8 @@ defmodule Pleroma.Search.Meilisearch do
defp meili_headers() do
private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key])
- if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}]
+ [{"Content-Type", "application/json"}] ++
+ if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}]
end
def meili_get!(path) do
From 4445421297f4a4375ce9df4857a66ad08e984507 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Fri, 29 Oct 2021 21:04:59 +0300
Subject: [PATCH 055/823] Only add local posts to index in activity_pub
Remote ones are already added in another place
---
lib/pleroma/web/activity_pub/activity_pub.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 7178cf9eb..cdc70aacf 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -141,7 +141,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end)
# Add local posts to search index
- Pleroma.Search.add_to_index(activity)
+ if local, do: Pleroma.Search.add_to_index(activity)
{:ok, activity}
else
From e928e307f34542b0a0af8b615c986aeac478b637 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Wed, 10 Nov 2021 21:25:12 +0300
Subject: [PATCH 056/823] Add a reindex option
Signed-off-by: Ekaterina Vaartis
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index cdf9ab0bd..2a3c3a8b9 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -11,9 +11,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1]
- def run(["index"]) do
+ def run(["index" | args]) do
start_pleroma()
+ is_reindex = "--reindex" in args
+
meili_post!(
"/indexes/objects/settings/ranking-rules",
[
@@ -68,6 +70,19 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
{[objects], new_acc}
end)
|> Stream.each(fn objects ->
+ objects =
+ objects
+ |> Enum.filter(fn o ->
+ if is_reindex do
+ result = meili_get!("/indexes/objects/documents/#{o.id}")
+
+ # Filter out the already indexed documents. This is true when the document does not exist
+ result["errorCode"] == "document_not_found"
+ else
+ true
+ end
+ end)
+
result =
meili_post!(
"/indexes/objects/documents",
From 9c1a9307079c8d007ae7cbf3e089d2bc5ea6b733 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sat, 13 Nov 2021 15:07:51 +0300
Subject: [PATCH 057/823] Support reindexing meilisearch >=0.24.0
It has has a different error code key
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 2a3c3a8b9..3b134ad3f 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -76,8 +76,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
if is_reindex do
result = meili_get!("/indexes/objects/documents/#{o.id}")
+ # With >= 0.24.0 the name for "errorCode" is just "code"
+ error_code_key =
+ if meili_get!("/version")["pkgVersion"] |> Version.match?(">= 0.24.0"),
+ do: "code",
+ else: "errorCode"
+
# Filter out the already indexed documents. This is true when the document does not exist
- result["errorCode"] == "document_not_found"
+ result[error_code_key] == "document_not_found"
else
true
end
From 8898b5e927bae27a521e4eadd0faf970ad27c5bc Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sun, 14 Nov 2021 20:15:12 +0300
Subject: [PATCH 058/823] Fix a typo in search docs
---
docs/configuration/search.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index e9743f1a4..9adc7884f 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -60,7 +60,7 @@ have to do it one time, but it might take a while, depending on the amount of po
consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2
million posts while idle and up to 7G while indexing initially, but your experience may be different).
-To start te initial indexing, run the `index` command:
+To start the initial indexing, run the `index` command:
=== "OTP"
```sh
From 7009ef5672ad20f92374d218cd614a38cd70515e Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sun, 14 Nov 2021 20:24:05 +0300
Subject: [PATCH 059/823] Move the search.ex file so credo doesn't complain
---
lib/pleroma/{search => }/search.ex | 0
1 file changed, 0 insertions(+), 0 deletions(-)
rename lib/pleroma/{search => }/search.ex (100%)
diff --git a/lib/pleroma/search/search.ex b/lib/pleroma/search.ex
similarity index 100%
rename from lib/pleroma/search/search.ex
rename to lib/pleroma/search.ex
From 39e596a5b51c0c86b6d6bd5f23177a1e6a64cf0b Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sun, 14 Nov 2021 21:42:18 +0300
Subject: [PATCH 060/823] Style fixes
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 3 ++-
lib/pleroma/search/meilisearch.ex | 2 +-
test/pleroma/search/meilisearch_test.exs | 2 +-
3 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 3b134ad3f..62ace7e39 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -82,7 +82,8 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
do: "code",
else: "errorCode"
- # Filter out the already indexed documents. This is true when the document does not exist
+ # Filter out the already indexed documents.
+ # This is true when the document does not exist
result[error_code_key] == "document_not_found"
else
true
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index d94ab8b64..41f99ad9f 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -7,7 +7,7 @@ defmodule Pleroma.Search.Meilisearch do
import Pleroma.Activity.Search
import Ecto.Query
- defp meili_headers() do
+ defp meili_headers do
private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key])
[{"Content-Type", "application/json"}] ++
diff --git a/test/pleroma/search/meilisearch_test.exs b/test/pleroma/search/meilisearch_test.exs
index 6e13c8edf..251388ea2 100644
--- a/test/pleroma/search/meilisearch_test.exs
+++ b/test/pleroma/search/meilisearch_test.exs
@@ -11,8 +11,8 @@ defmodule Pleroma.Search.MeilisearchTest do
import Tesla.Mock
import Mock
- alias Pleroma.Web.CommonAPI
alias Pleroma.Search.Meilisearch
+ alias Pleroma.Web.CommonAPI
setup_all do
Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
From 0fae71f88d142f64ec18a49ff4292db816dacdc8 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Tue, 16 Nov 2021 21:54:26 +0300
Subject: [PATCH 061/823] Rename search.ex to database_search.ex and add
search/2
---
lib/pleroma/{search.ex => search/database_search.ex} | 8 +++++++-
lib/pleroma/web/activity_pub/activity_pub.ex | 2 +-
lib/pleroma/web/activity_pub/side_effects.ex | 4 ++--
.../web/mastodon_api/controllers/search_controller.ex | 4 +---
4 files changed, 11 insertions(+), 7 deletions(-)
rename lib/pleroma/{search.ex => search/database_search.ex} (68%)
diff --git a/lib/pleroma/search.ex b/lib/pleroma/search/database_search.ex
similarity index 68%
rename from lib/pleroma/search.ex
rename to lib/pleroma/search/database_search.ex
index e363abf19..be0e19be0 100644
--- a/lib/pleroma/search.ex
+++ b/lib/pleroma/search/database_search.ex
@@ -1,4 +1,4 @@
-defmodule Pleroma.Search do
+defmodule Pleroma.Search.DatabaseSearch do
def add_to_index(activity) do
search_module = Pleroma.Config.get([Pleroma.Search, :module])
@@ -15,4 +15,10 @@ defmodule Pleroma.Search do
Task.start(fn -> search_module.remove_from_index(object) end)
end)
end
+
+ def search(query, options) do
+ search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
+
+ search_module.search(options[:for_user], query, options)
+ end
end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index cdc70aacf..7e3444676 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -141,7 +141,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end)
# Add local posts to search index
- if local, do: Pleroma.Search.add_to_index(activity)
+ if local, do: Pleroma.Search.DatabaseSearch.add_to_index(activity)
{:ok, activity}
else
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 4762b5ac6..fa57eab69 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -227,7 +227,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
end)
- Pleroma.Search.add_to_index(Map.put(activity, :object, object))
+ Pleroma.Search.DatabaseSearch.add_to_index(Map.put(activity, :object, object))
meta =
meta
@@ -330,7 +330,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
if result == :ok do
Notification.create_notifications(object)
- Pleroma.Search.remove_from_index(deleted_object)
+ Pleroma.Search.DatabaseSearch.remove_from_index(deleted_object)
{:ok, object, meta}
else
diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
index 99c33eba6..10f1aa532 100644
--- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
@@ -99,9 +99,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
end
defp resource_search(_, "statuses", query, options) do
- search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
-
- statuses = with_fallback(fn -> search_module.search(options[:for_user], query, options) end)
+ statuses = with_fallback(fn -> Pleroma.Search.DatabaseSearch.search(query, options) end)
StatusView.render("index.json",
activities: statuses,
From a6946048fbe049aa223d094d36eb767739ab5ff2 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Wed, 17 Nov 2021 22:29:49 +0300
Subject: [PATCH 062/823] Rename Activity.Search to Search.DatabaseSearch
---
config/config.exs | 2 +-
config/test.exs | 2 +-
docs/configuration/search.md | 2 +-
lib/pleroma/activity.ex | 2 +-
lib/pleroma/activity/search.ex | 165 ------------------
lib/pleroma/search.ex | 24 +++
lib/pleroma/search/database_search.ex | 157 +++++++++++++++--
lib/pleroma/search/meilisearch.ex | 2 +-
lib/pleroma/web/activity_pub/activity_pub.ex | 2 +-
lib/pleroma/web/activity_pub/side_effects.ex | 4 +-
.../controllers/search_controller.ex | 2 +-
.../database_search_test.ex} | 10 +-
12 files changed, 181 insertions(+), 193 deletions(-)
delete mode 100644 lib/pleroma/activity/search.ex
create mode 100644 lib/pleroma/search.ex
rename test/pleroma/{activity/search_test.exs => search/database_search_test.ex} (86%)
diff --git a/config/config.exs b/config/config.exs
index b55dd94cb..f7f3a1454 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -870,7 +870,7 @@ config :pleroma, ConcurrentLimiter, [
{Pleroma.Search, [max_running: 30, max_waiting: 50]}
]
-config :pleroma, Pleroma.Search, module: Pleroma.Activity.Search
+config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil
diff --git a/config/test.exs b/config/test.exs
index c9b2b51ba..ea29be638 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -133,7 +133,7 @@ config :pleroma, :side_effects,
ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock,
logger: Pleroma.LoggerMock
-config :pleroma, Pleroma.Search, module: Pleroma.Activity
+config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil
diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index 9adc7884f..c7e77d9c2 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -6,7 +6,7 @@
To use built-in search that has no external dependencies, set the search module to `Pleroma.Activity`:
-> config :pleroma, Pleroma.Search, module: Pleroma.Activity
+> config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
While it has no external dependencies, it has problems with performance and relevancy.
diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index ebfd4ed45..389c80691 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -366,7 +366,7 @@ defmodule Pleroma.Activity do
from(activity in query, where: activity.actor not in subquery(deactivated_users_query))
end
- defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search
+ defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch
def direct_conversation_id(activity, for_user) do
alias Pleroma.Conversation.Participation
diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex
deleted file mode 100644
index 47ab5208c..000000000
--- a/lib/pleroma/activity/search.ex
+++ /dev/null
@@ -1,165 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2022 Pleroma Authors
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Activity.Search do
- alias Pleroma.Activity
- alias Pleroma.Object.Fetcher
- alias Pleroma.Pagination
- alias Pleroma.User
- alias Pleroma.Web.ActivityPub.Visibility
-
- require Pleroma.Constants
-
- import Ecto.Query
-
- def search(user, search_query, options \\ []) do
- index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin
- limit = Enum.min([Keyword.get(options, :limit), 40])
- offset = Keyword.get(options, :offset, 0)
- author = Keyword.get(options, :author)
-
- search_function =
- if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do
- :websearch
- else
- :plain
- end
-
- try do
- Activity
- |> Activity.with_preloaded_object()
- |> Activity.restrict_deactivated_users()
- |> restrict_public(user)
- |> query_with(index_type, search_query, search_function)
- |> maybe_restrict_local(user)
- |> maybe_restrict_author(author)
- |> maybe_restrict_blocked(user)
- |> Pagination.fetch_paginated(
- %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
- :offset
- )
- |> maybe_fetch(user, search_query)
- rescue
- _ -> maybe_fetch([], user, search_query)
- end
- end
-
- def add_to_index(_activity), do: nil
- def remove_from_index(_object), do: nil
-
- def maybe_restrict_author(query, %User{} = author) do
- Activity.Queries.by_author(query, author)
- end
-
- def maybe_restrict_author(query, _), do: query
-
- def maybe_restrict_blocked(query, %User{} = user) do
- Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user))
- end
-
- def maybe_restrict_blocked(query, _), do: query
-
- defp restrict_public(q, user) when not is_nil(user) do
- intended_recipients = [
- Pleroma.Constants.as_public(),
- Pleroma.Web.ActivityPub.Utils.as_local_public()
- ]
-
- from([a, o] in q,
- where: fragment("?->>'type' = 'Create'", a.data),
- where: fragment("? && ?", ^intended_recipients, a.recipients)
- )
- end
-
- defp restrict_public(q, _user) do
- from([a, o] in q,
- where: fragment("?->>'type' = 'Create'", a.data),
- where: ^Pleroma.Constants.as_public() in a.recipients
- )
- end
-
- defp query_with(q, :gin, search_query, :plain) do
- %{rows: [[tsc]]} =
- Ecto.Adapters.SQL.query!(
- Pleroma.Repo,
- "select current_setting('default_text_search_config')::regconfig::oid;"
- )
-
- from([a, o] in q,
- where:
- fragment(
- "to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)",
- ^tsc,
- o.data,
- ^search_query
- )
- )
- end
-
- defp query_with(q, :gin, search_query, :websearch) do
- %{rows: [[tsc]]} =
- Ecto.Adapters.SQL.query!(
- Pleroma.Repo,
- "select current_setting('default_text_search_config')::regconfig::oid;"
- )
-
- from([a, o] in q,
- where:
- fragment(
- "to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)",
- ^tsc,
- o.data,
- ^search_query
- )
- )
- end
-
- defp query_with(q, :rum, search_query, :plain) do
- from([a, o] in q,
- where:
- fragment(
- "? @@ plainto_tsquery(?)",
- o.fts_content,
- ^search_query
- ),
- order_by: [fragment("? <=> now()::date", o.inserted_at)]
- )
- end
-
- defp query_with(q, :rum, search_query, :websearch) do
- from([a, o] in q,
- where:
- fragment(
- "? @@ websearch_to_tsquery(?)",
- o.fts_content,
- ^search_query
- ),
- order_by: [fragment("? <=> now()::date", o.inserted_at)]
- )
- end
-
- def maybe_restrict_local(q, user) do
- limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
-
- case {limit, user} do
- {:all, _} -> restrict_local(q)
- {:unauthenticated, %User{}} -> q
- {:unauthenticated, _} -> restrict_local(q)
- {false, _} -> q
- end
- end
-
- defp restrict_local(q), do: where(q, local: true)
-
- def maybe_fetch(activities, user, search_query) do
- with true <- Regex.match?(~r/https?:/, search_query),
- {:ok, object} <- Fetcher.fetch_object_from_id(search_query),
- %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
- true <- Visibility.visible_for_user?(activity, user) do
- [activity | activities]
- else
- _ -> activities
- end
- end
-end
diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex
new file mode 100644
index 000000000..ae0b28c54
--- /dev/null
+++ b/lib/pleroma/search.ex
@@ -0,0 +1,24 @@
+defmodule Pleroma.Search do
+ def add_to_index(activity) do
+ search_module = Pleroma.Config.get([Pleroma.Search, :module])
+
+ ConcurrentLimiter.limit(Pleroma.Search, fn ->
+ Task.start(fn -> search_module.add_to_index(activity) end)
+ end)
+ end
+
+ def remove_from_index(object) do
+ # Also delete from search index
+ search_module = Pleroma.Config.get([Pleroma.Search, :module])
+
+ ConcurrentLimiter.limit(Pleroma.Search, fn ->
+ Task.start(fn -> search_module.remove_from_index(object) end)
+ end)
+ end
+
+ def search(query, options) do
+ search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
+
+ search_module.search(options[:for_user], query, options)
+ end
+end
diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex
index be0e19be0..5a8b8ca67 100644
--- a/lib/pleroma/search/database_search.ex
+++ b/lib/pleroma/search/database_search.ex
@@ -1,24 +1,153 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
defmodule Pleroma.Search.DatabaseSearch do
- def add_to_index(activity) do
- search_module = Pleroma.Config.get([Pleroma.Search, :module])
+ alias Pleroma.Activity
+ alias Pleroma.Object.Fetcher
+ alias Pleroma.Pagination
+ alias Pleroma.User
+ alias Pleroma.Web.ActivityPub.Visibility
- ConcurrentLimiter.limit(Pleroma.Search, fn ->
- Task.start(fn -> search_module.add_to_index(activity) end)
- end)
+ require Pleroma.Constants
+
+ import Ecto.Query
+
+ def search(user, search_query, options \\ []) do
+ index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin
+ limit = Enum.min([Keyword.get(options, :limit), 40])
+ offset = Keyword.get(options, :offset, 0)
+ author = Keyword.get(options, :author)
+
+ search_function =
+ if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do
+ :websearch
+ else
+ :plain
+ end
+
+ try do
+ Activity
+ |> Activity.with_preloaded_object()
+ |> Activity.restrict_deactivated_users()
+ |> restrict_public()
+ |> query_with(index_type, search_query, search_function)
+ |> maybe_restrict_local(user)
+ |> maybe_restrict_author(author)
+ |> maybe_restrict_blocked(user)
+ |> Pagination.fetch_paginated(
+ %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
+ :offset
+ )
+ |> maybe_fetch(user, search_query)
+ rescue
+ _ -> maybe_fetch([], user, search_query)
+ end
end
- def remove_from_index(object) do
- # Also delete from search index
- search_module = Pleroma.Config.get([Pleroma.Search, :module])
+ def add_to_index(_activity), do: nil
+ def remove_from_index(_object), do: nil
- ConcurrentLimiter.limit(Pleroma.Search, fn ->
- Task.start(fn -> search_module.remove_from_index(object) end)
- end)
+ def maybe_restrict_author(query, %User{} = author) do
+ Activity.Queries.by_author(query, author)
end
- def search(query, options) do
- search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
+ def maybe_restrict_author(query, _), do: query
- search_module.search(options[:for_user], query, options)
+ def maybe_restrict_blocked(query, %User{} = user) do
+ Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user))
+ end
+
+ def maybe_restrict_blocked(query, _), do: query
+
+ def restrict_public(q) do
+ from([a, o] in q,
+ where: fragment("?->>'type' = 'Create'", a.data),
+ where: ^Pleroma.Constants.as_public() in a.recipients
+ )
+ end
+
+ defp query_with(q, :gin, search_query, :plain) do
+ %{rows: [[tsc]]} =
+ Ecto.Adapters.SQL.query!(
+ Pleroma.Repo,
+ "select current_setting('default_text_search_config')::regconfig::oid;"
+ )
+
+ from([a, o] in q,
+ where:
+ fragment(
+ "to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)",
+ ^tsc,
+ o.data,
+ ^search_query
+ )
+ )
+ end
+
+ defp query_with(q, :gin, search_query, :websearch) do
+ %{rows: [[tsc]]} =
+ Ecto.Adapters.SQL.query!(
+ Pleroma.Repo,
+ "select current_setting('default_text_search_config')::regconfig::oid;"
+ )
+
+ from([a, o] in q,
+ where:
+ fragment(
+ "to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)",
+ ^tsc,
+ o.data,
+ ^search_query
+ )
+ )
+ end
+
+ defp query_with(q, :rum, search_query, :plain) do
+ from([a, o] in q,
+ where:
+ fragment(
+ "? @@ plainto_tsquery(?)",
+ o.fts_content,
+ ^search_query
+ ),
+ order_by: [fragment("? <=> now()::date", o.inserted_at)]
+ )
+ end
+
+ defp query_with(q, :rum, search_query, :websearch) do
+ from([a, o] in q,
+ where:
+ fragment(
+ "? @@ websearch_to_tsquery(?)",
+ o.fts_content,
+ ^search_query
+ ),
+ order_by: [fragment("? <=> now()::date", o.inserted_at)]
+ )
+ end
+
+ def maybe_restrict_local(q, user) do
+ limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
+
+ case {limit, user} do
+ {:all, _} -> restrict_local(q)
+ {:unauthenticated, %User{}} -> q
+ {:unauthenticated, _} -> restrict_local(q)
+ {false, _} -> q
+ end
+ end
+
+ defp restrict_local(q), do: where(q, local: true)
+
+ def maybe_fetch(activities, user, search_query) do
+ with true <- Regex.match?(~r/https?:/, search_query),
+ {:ok, object} <- Fetcher.fetch_object_from_id(search_query),
+ %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
+ true <- Visibility.visible_for_user?(activity, user) do
+ [activity | activities]
+ else
+ _ -> activities
+ end
end
end
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 41f99ad9f..fa9e27b03 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -4,7 +4,7 @@ defmodule Pleroma.Search.Meilisearch do
alias Pleroma.Activity
- import Pleroma.Activity.Search
+ import Pleroma.Search.DatabaseSearch
import Ecto.Query
defp meili_headers do
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 7e3444676..cdc70aacf 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -141,7 +141,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end)
# Add local posts to search index
- if local, do: Pleroma.Search.DatabaseSearch.add_to_index(activity)
+ if local, do: Pleroma.Search.add_to_index(activity)
{:ok, activity}
else
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index fa57eab69..4762b5ac6 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -227,7 +227,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
end)
- Pleroma.Search.DatabaseSearch.add_to_index(Map.put(activity, :object, object))
+ Pleroma.Search.add_to_index(Map.put(activity, :object, object))
meta =
meta
@@ -330,7 +330,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
if result == :ok do
Notification.create_notifications(object)
- Pleroma.Search.DatabaseSearch.remove_from_index(deleted_object)
+ Pleroma.Search.remove_from_index(deleted_object)
{:ok, object, meta}
else
diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
index 10f1aa532..e4acba226 100644
--- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
@@ -99,7 +99,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
end
defp resource_search(_, "statuses", query, options) do
- statuses = with_fallback(fn -> Pleroma.Search.DatabaseSearch.search(query, options) end)
+ statuses = with_fallback(fn -> Pleroma.Search.search(query, options) end)
StatusView.render("index.json",
activities: statuses,
diff --git a/test/pleroma/activity/search_test.exs b/test/pleroma/search/database_search_test.ex
similarity index 86%
rename from test/pleroma/activity/search_test.exs
rename to test/pleroma/search/database_search_test.ex
index 3b5fd2c3c..c123d0b84 100644
--- a/test/pleroma/activity/search_test.exs
+++ b/test/pleroma/search/database_search_test.ex
@@ -2,8 +2,8 @@
# Copyright © 2017-2022 Pleroma Authors
# SPDX-License-Identifier: AGPL-3.0-only
-defmodule Pleroma.Activity.SearchTest do
- alias Pleroma.Activity.Search
+defmodule Pleroma.Search.DatabaseSearchTest do
+ alias Pleroma.Search.DatabaseSearch
alias Pleroma.Web.CommonAPI
import Pleroma.Factory
@@ -13,7 +13,7 @@ defmodule Pleroma.Activity.SearchTest do
user = insert(:user)
{:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
- [result] = Search.search(nil, "wednesday")
+ [result] = DatabaseSearch.search(nil, "wednesday")
assert result.id == post.id
end
@@ -45,7 +45,7 @@ defmodule Pleroma.Activity.SearchTest do
{:ok, _post2} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
# plainto doesn't understand complex queries
- assert [result] = Search.search(nil, "wednesday -dudes")
+ assert [result] = DatabaseSearch.search(nil, "wednesday -dudes")
assert result.id == post.id
end
@@ -55,7 +55,7 @@ defmodule Pleroma.Activity.SearchTest do
{:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
{:ok, other_post} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
- assert [result] = Search.search(nil, "wednesday -dudes")
+ assert [result] = DatabaseSearch.search(nil, "wednesday -dudes")
assert result.id == other_post.id
end
From a12f63bc81481e3f852934e8cc1269e16a57cf0a Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 22 Nov 2021 21:39:54 +0300
Subject: [PATCH 063/823] Implement suggestions from the Meilisearch MR
- Index unlisted posts
- Move version check outside of the streaming and only do it once
- Use a PUT request instead of checking manually if there is need to insert
- Add error handling, sort of
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 84 +++++++++-----------
lib/pleroma/search/meilisearch.ex | 85 ++++++++++++++-------
2 files changed, 93 insertions(+), 76 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 62ace7e39..6730a99a9 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -3,38 +3,40 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
- require Logger
require Pleroma.Constants
import Mix.Pleroma
import Ecto.Query
- import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1]
+ import Pleroma.Search.Meilisearch,
+ only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete!: 1]
- def run(["index" | args]) do
+ def run(["index"]) do
start_pleroma()
- is_reindex = "--reindex" in args
+ {:ok, _} =
+ meili_post(
+ "/indexes/objects/settings/ranking-rules",
+ [
+ "desc(published)",
+ "words",
+ "exactness",
+ "proximity",
+ "wordsPosition",
+ "typo",
+ "attribute"
+ ]
+ )
- meili_post!(
- "/indexes/objects/settings/ranking-rules",
- [
- "desc(published)",
- "words",
- "exactness",
- "proximity",
- "wordsPosition",
- "typo",
- "attribute"
- ]
- )
+ {:ok, _} =
+ meili_post(
+ "/indexes/objects/settings/searchable-attributes",
+ [
+ "content"
+ ]
+ )
- meili_post!(
- "/indexes/objects/settings/searchable-attributes",
- [
- "content"
- ]
- )
+ IO.puts("Created indices. Starting to insert posts.")
chunk_size = 10_000
@@ -42,11 +44,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
fn ->
query =
from(Pleroma.Object,
- # Only index public posts which are notes and have some text
+ # Only index public and unlisted posts which are notes and have some text
where:
fragment("data->>'type' = 'Note'") and
- fragment("LENGTH(data->>'content') > 0") and
- fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()),
+ (fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) or
+ fragment("data->'cc' \\? ?", ^Pleroma.Constants.as_public())),
order_by: [desc: fragment("data->'published'")]
)
@@ -70,34 +72,18 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
{[objects], new_acc}
end)
|> Stream.each(fn objects ->
- objects =
- objects
- |> Enum.filter(fn o ->
- if is_reindex do
- result = meili_get!("/indexes/objects/documents/#{o.id}")
-
- # With >= 0.24.0 the name for "errorCode" is just "code"
- error_code_key =
- if meili_get!("/version")["pkgVersion"] |> Version.match?(">= 0.24.0"),
- do: "code",
- else: "errorCode"
-
- # Filter out the already indexed documents.
- # This is true when the document does not exist
- result[error_code_key] == "document_not_found"
- else
- true
- end
- end)
-
result =
- meili_post!(
+ meili_put(
"/indexes/objects/documents",
objects
)
- if not Map.has_key?(result, "updateId") do
- IO.puts("Failed to index: #{inspect(result)}")
+ with {:ok, res} <- result do
+ if not Map.has_key?(res, "updateId") do
+ IO.puts("\nFailed to index: #{inspect(result)}")
+ end
+ else
+ e -> IO.puts("\nFailed to index due to network error: #{inspect(e)}")
end
end)
|> Stream.run()
@@ -137,7 +123,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
def run(["stats"]) do
start_pleroma()
- result = meili_get!("/indexes/objects/stats")
+ {:ok, result} = meili_get("/indexes/objects/stats")
IO.puts("Number of entries: #{result["numberOfDocuments"]}")
IO.puts("Indexing? #{result["isIndexing"]}")
end
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index fa9e27b03..21b44de86 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -14,29 +14,50 @@ defmodule Pleroma.Search.Meilisearch do
if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}]
end
- def meili_get!(path) do
+ def meili_get(path) do
endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
- {:ok, result} =
+ result =
Pleroma.HTTP.get(
Path.join(endpoint, path),
meili_headers()
)
- Jason.decode!(result.body)
+ with {:ok, res} <- result do
+ {:ok, Jason.decode!(res.body)}
+ end
end
- def meili_post!(path, params) do
+ def meili_post(path, params) do
endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
- {:ok, result} =
+ result =
Pleroma.HTTP.post(
Path.join(endpoint, path),
Jason.encode!(params),
meili_headers()
)
- Jason.decode!(result.body)
+ with {:ok, res} <- result do
+ {:ok, Jason.decode!(res.body)}
+ end
+ end
+
+ def meili_put(path, params) do
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ result =
+ Pleroma.HTTP.request(
+ :put,
+ Path.join(endpoint, path),
+ Jason.encode!(params),
+ meili_headers(),
+ []
+ )
+
+ with {:ok, res} <- result do
+ {:ok, Jason.decode!(res.body)}
+ end
end
def meili_delete!(path) do
@@ -57,34 +78,40 @@ defmodule Pleroma.Search.Meilisearch do
offset = Keyword.get(options, :offset, 0)
author = Keyword.get(options, :author)
- result =
- meili_post!(
+ res =
+ meili_post(
"/indexes/objects/search",
%{q: query, offset: offset, limit: limit}
)
- hits = result["hits"] |> Enum.map(& &1["ap"])
+ with {:ok, result} <- res do
+ hits = result["hits"] |> Enum.map(& &1["ap"])
- try do
- hits
- |> Activity.create_by_object_ap_id()
- |> Activity.with_preloaded_object()
- |> Activity.with_preloaded_object()
- |> Activity.restrict_deactivated_users()
- |> maybe_restrict_local(user)
- |> maybe_restrict_author(author)
- |> maybe_restrict_blocked(user)
- |> maybe_fetch(user, query)
- |> order_by([object: obj], desc: obj.data["published"])
- |> Pleroma.Repo.all()
- rescue
- _ -> maybe_fetch([], user, query)
+ try do
+ hits
+ |> Activity.create_by_object_ap_id()
+ |> Activity.with_preloaded_object()
+ |> Activity.with_preloaded_object()
+ |> Activity.restrict_deactivated_users()
+ |> maybe_restrict_local(user)
+ |> maybe_restrict_author(author)
+ |> maybe_restrict_blocked(user)
+ |> maybe_fetch(user, query)
+ |> order_by([object: obj], desc: obj.data["published"])
+ |> Pleroma.Repo.all()
+ rescue
+ _ -> maybe_fetch([], user, query)
+ end
end
end
def object_to_search_data(object) do
+ # Only index public or unlisted Notes
if not is_nil(object) and object.data["type"] == "Note" and
- Pleroma.Constants.as_public() in object.data["to"] do
+ not is_nil(object.data["content"]) and
+ (Pleroma.Constants.as_public() in object.data["to"] or
+ Pleroma.Constants.as_public() in object.data["cc"]) and
+ String.length(object.data["content"]) > 1 do
data = object.data
content_str =
@@ -117,13 +144,17 @@ defmodule Pleroma.Search.Meilisearch do
if activity.data["type"] == "Create" and maybe_search_data do
result =
- meili_post!(
+ meili_put(
"/indexes/objects/documents",
[maybe_search_data]
)
- if not Map.has_key?(result, "updateId") do
- Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}")
+ with {:ok, res} <- result,
+ true <- Map.has_key?(res, "updateId") do
+ # Do nothing
+ else
+ _ ->
+ Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}")
end
end
end
From 3a11e79de0c7092bf4fe0649e4ab1fcb53eb14a3 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 20 Dec 2021 17:46:23 +0300
Subject: [PATCH 064/823] Add config description for meilisearch
---
config/description.exs | 35 +++++++++++++++++++++++++++++++++++
1 file changed, 35 insertions(+)
diff --git a/config/description.exs b/config/description.exs
index 3a2a65272..81fefaf89 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -3475,5 +3475,40 @@ config :pleroma, :config_description, [
]
}
]
+ },
+ %{
+ group: :pleroma,
+ key: Pleroma.Search,
+ type: :group,
+ description: "General search settings.",
+ children: [
+ %{
+ key: :module,
+ type: :keyword,
+ description: "Selected search module.",
+ suggestion: [Pleroma.Search.DatabaseSearch, Pleroma.Search.Meilisearch]
+ }
+ ]
+ },
+ %{
+ group: :pleroma,
+ key: Pleroma.Search.Meilisearch,
+ type: :group,
+ description: "Meilisearch settings.",
+ children: [
+ %{
+ key: :url,
+ type: :string,
+ description: "Meilisearch URL.",
+ suggestion: ["http://127.0.0.1:7700/"]
+ },
+ %{
+ key: :private_key,
+ type: :string,
+ description:
+ "Private key for meilisearch authentication, or `nil` to disable private key authentication.",
+ suggestion: [nil]
+ }
+ ]
}
]
From 3412713c5b2fd24605b18933ef70de164ee14f2d Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 20 Dec 2021 18:16:33 +0300
Subject: [PATCH 065/823] Update search.md documentation with meilisearch
indexing steps
---
docs/configuration/search.md | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index c7e77d9c2..7dbbd3e17 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -60,6 +60,15 @@ have to do it one time, but it might take a while, depending on the amount of po
consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2
million posts while idle and up to 7G while indexing initially, but your experience may be different).
+The sequence of actions is as follows:
+
+1. First, change the configuration to use `Pleroma.Search.Meilisearch` as the search backend
+2. Restart your instance, at this point it can be used while the search indexing is running, though search won't return anything
+3. Start the initial indexing process (as described below with `index`),
+ and wait until the task says it sent everything from the database to index
+4. Wait until everything is actually indexed (by checking with `stats` as described below),
+ at this point you don't have to do anything, just wait a while.
+
To start the initial indexing, run the `index` command:
=== "OTP"
From 3179ed0921197a8a8f32a519c7d41dc09011024d Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 20 Dec 2021 18:48:52 +0300
Subject: [PATCH 066/823] Make chunk size configurable
---
config/config.exs | 5 ++++-
lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/config/config.exs b/config/config.exs
index f7f3a1454..bcbc59b83 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -872,7 +872,10 @@ config :pleroma, ConcurrentLimiter, [
config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
-config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil
+config :pleroma, Pleroma.Search.Meilisearch,
+ url: "http://127.0.0.1:7700/",
+ private_key: nil,
+ initial_indexing_chunk_size: 100_000
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 6730a99a9..021552f7b 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -38,7 +38,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
IO.puts("Created indices. Starting to insert posts.")
- chunk_size = 10_000
+ chunk_size = Pleroma.Config.get([Pleroma.Search.Meilisearch, :initial_indexing_chunk_size])
Pleroma.Repo.transaction(
fn ->
From 571533ae2618478f26db312e52265e143356debd Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 20 Dec 2021 19:05:59 +0300
Subject: [PATCH 067/823] Don't support meilisearch < 0.24.0, since it breaks
things
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 18 +++++++++++++++---
1 file changed, 15 insertions(+), 3 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 021552f7b..5098668ad 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -14,17 +14,29 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
def run(["index"]) do
start_pleroma()
+ meili_version =
+ (
+ {:ok, result} = meili_get("/version")
+
+ result["pkgVersion"]
+ )
+
+ # The ranking rule syntax was changed but nothing about that is mentioned in the changelog
+ if not Version.match?(meili_version, ">= 0.24.0") do
+ raise "Meilisearch <0.24.0 not supported"
+ end
+
{:ok, _} =
meili_post(
"/indexes/objects/settings/ranking-rules",
[
- "desc(published)",
+ "published:desc",
"words",
"exactness",
"proximity",
- "wordsPosition",
"typo",
- "attribute"
+ "attribute",
+ "sort"
]
)
From 4f2637acc6c46ea39ae38e869903e7ffcc38b34d Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 20 Dec 2021 19:27:22 +0300
Subject: [PATCH 068/823] Add description for initial_indexing_chunk_size
---
config/description.exs | 8 ++++++++
docs/configuration/search.md | 8 ++++++--
2 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/config/description.exs b/config/description.exs
index 81fefaf89..cea4401ba 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -3508,6 +3508,14 @@ config :pleroma, :config_description, [
description:
"Private key for meilisearch authentication, or `nil` to disable private key authentication.",
suggestion: [nil]
+ },
+ %{
+ key: :initial_indexing_chunk_size,
+ type: :int,
+ description:
+ "Amount of posts in a batch when running the initial indexing operation. Should probably not be more than 100000" <>
+ " since there's a limit on maximum insert size",
+ suggestion: [100_000]
}
]
}
diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index 7dbbd3e17..a785a18ad 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -25,11 +25,15 @@ To use [meilisearch](https://www.meilisearch.com/), set the search module to `Pl
> config :pleroma, Pleroma.Search, module: Pleroma.Search.Meilisearch
-You then need to set the address of the meilisearch instance, and optionally the private key for authentication.
+You then need to set the address of the meilisearch instance, and optionally the private key for authentication. You might
+also want to change the `initial_indexing_chunk_size` to be smaller if you're server is not very powerful, but not higher than `100_000`,
+because meilisearch will refuse to process it if it's too big. However, in general you want this to be as big as possible, because meilisearch
+indexes faster when it can process many posts in a single batch.
> config :pleroma, Pleroma.Search.Meilisearch,
> url: "http://127.0.0.1:7700/",
-> private_key: "private key"
+> private_key: "private key",
+> initial_indexing_chunk_size: 100_000
Information about setting up meilisearch can be found in the
[official documentation](https://docs.meilisearch.com/learn/getting_started/installation.html).
From 6f2f457751ea09507045e6dd5d5869a14befd3d1 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Mon, 20 Dec 2021 22:38:50 +0300
Subject: [PATCH 069/823] Add a search backend behaviour
---
lib/pleroma/search/database_search.ex | 5 +++++
lib/pleroma/search/meilisearch.ex | 4 ++++
lib/pleroma/search/search_backend.ex | 17 +++++++++++++++++
3 files changed, 26 insertions(+)
create mode 100644 lib/pleroma/search/search_backend.ex
diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex
index 5a8b8ca67..3735a5fab 100644
--- a/lib/pleroma/search/database_search.ex
+++ b/lib/pleroma/search/database_search.ex
@@ -13,6 +13,8 @@ defmodule Pleroma.Search.DatabaseSearch do
import Ecto.Query
+ @behaviour Pleroma.Search.SearchBackend
+
def search(user, search_query, options \\ []) do
index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin
limit = Enum.min([Keyword.get(options, :limit), 40])
@@ -45,7 +47,10 @@ defmodule Pleroma.Search.DatabaseSearch do
end
end
+ @impl true
def add_to_index(_activity), do: nil
+
+ @impl true
def remove_from_index(_object), do: nil
def maybe_restrict_author(query, %User{} = author) do
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 21b44de86..33bbf8392 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -7,6 +7,8 @@ defmodule Pleroma.Search.Meilisearch do
import Pleroma.Search.DatabaseSearch
import Ecto.Query
+ @behaviour Pleroma.Search.SearchBackend
+
defp meili_headers do
private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key])
@@ -139,6 +141,7 @@ defmodule Pleroma.Search.Meilisearch do
end
end
+ @impl true
def add_to_index(activity) do
maybe_search_data = object_to_search_data(activity.object)
@@ -159,6 +162,7 @@ defmodule Pleroma.Search.Meilisearch do
end
end
+ @impl true
def remove_from_index(object) do
meili_delete!("/indexes/objects/documents/#{object.id}")
end
diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex
new file mode 100644
index 000000000..ed6bfd329
--- /dev/null
+++ b/lib/pleroma/search/search_backend.ex
@@ -0,0 +1,17 @@
+defmodule Pleroma.Search.SearchBackend do
+ @doc """
+ Add the object associated with the activity to the search index.
+
+ The whole activity is passed, to allow filtering on things such as scope.
+ """
+ @callback add_to_index(activity :: Pleroma.Activity.t()) :: nil
+
+ @doc """
+ Remove the object from the index.
+
+ Just the object, as opposed to the whole activity, is passed, since the object
+ is what contains the actual content and there is no need for fitlering when removing
+ from index.
+ """
+ @callback remove_from_index(object :: Pleroma.Object.t()) :: nil
+end
From 2bc21c6f1884bae3226f760ed1da39dd9c5f2958 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sat, 22 Jan 2022 15:23:11 +0300
Subject: [PATCH 070/823] Use oban for search indexing
---
config/config.exs | 3 ++-
lib/pleroma/search.ex | 15 ++++---------
lib/pleroma/workers/search_indexing_worker.ex | 21 +++++++++++++++++++
3 files changed, 27 insertions(+), 12 deletions(-)
create mode 100644 lib/pleroma/workers/search_indexing_worker.ex
diff --git a/config/config.exs b/config/config.exs
index bcbc59b83..3e2f0da3f 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -571,7 +571,8 @@ config :pleroma, Oban,
remote_fetcher: 2,
attachments_cleanup: 1,
new_users_digest: 1,
- mute_expire: 5
+ mute_expire: 5,
+ search_indexing: 1
],
plugins: [Oban.Plugins.Pruner],
crontab: [
diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex
index ae0b28c54..af858fc46 100644
--- a/lib/pleroma/search.ex
+++ b/lib/pleroma/search.ex
@@ -1,19 +1,12 @@
defmodule Pleroma.Search do
- def add_to_index(activity) do
- search_module = Pleroma.Config.get([Pleroma.Search, :module])
+ alias Pleroma.Workers.SearchIndexingWorker
- ConcurrentLimiter.limit(Pleroma.Search, fn ->
- Task.start(fn -> search_module.add_to_index(activity) end)
- end)
+ def add_to_index(activity) do
+ SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity.id})
end
def remove_from_index(object) do
- # Also delete from search index
- search_module = Pleroma.Config.get([Pleroma.Search, :module])
-
- ConcurrentLimiter.limit(Pleroma.Search, fn ->
- Task.start(fn -> search_module.remove_from_index(object) end)
- end)
+ SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object.id})
end
def search(query, options) do
diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex
new file mode 100644
index 000000000..43b7bad1e
--- /dev/null
+++ b/lib/pleroma/workers/search_indexing_worker.ex
@@ -0,0 +1,21 @@
+defmodule Pleroma.Workers.SearchIndexingWorker do
+ use Pleroma.Workers.WorkerHelper, queue: "search_indexing"
+
+ @impl Oban.Worker
+
+ def perform(%Job{args: %{"op" => "add_to_index", "activity" => activity_id}}) do
+ activity = Pleroma.Activity.get_by_id_with_object(activity_id)
+
+ search_module = Pleroma.Config.get([Pleroma.Search, :module])
+
+ search_module.add_to_index(activity)
+ end
+
+ def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do
+ object = Pleroma.Object.get_by_id(object_id)
+
+ search_module = Pleroma.Config.get([Pleroma.Search, :module])
+
+ search_module.remove_from_index(object)
+ end
+end
From d89dc5518b5c0eb232e7ac85ddd538f89c32606d Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sat, 22 Jan 2022 16:31:32 +0300
Subject: [PATCH 071/823] Fix meilisearch tests and jobs for oban
---
lib/pleroma/workers/search_indexing_worker.ex | 4 +++
test/pleroma/search/meilisearch_test.exs | 35 ++++++++++++++-----
2 files changed, 30 insertions(+), 9 deletions(-)
diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex
index 43b7bad1e..70a8d42d0 100644
--- a/lib/pleroma/workers/search_indexing_worker.ex
+++ b/lib/pleroma/workers/search_indexing_worker.ex
@@ -9,6 +9,8 @@ defmodule Pleroma.Workers.SearchIndexingWorker do
search_module = Pleroma.Config.get([Pleroma.Search, :module])
search_module.add_to_index(activity)
+
+ :ok
end
def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do
@@ -17,5 +19,7 @@ defmodule Pleroma.Workers.SearchIndexingWorker do
search_module = Pleroma.Config.get([Pleroma.Search, :module])
search_module.remove_from_index(object)
+
+ :ok
end
end
diff --git a/test/pleroma/search/meilisearch_test.exs b/test/pleroma/search/meilisearch_test.exs
index 251388ea2..da614577f 100644
--- a/test/pleroma/search/meilisearch_test.exs
+++ b/test/pleroma/search/meilisearch_test.exs
@@ -6,6 +6,7 @@ defmodule Pleroma.Search.MeilisearchTest do
require Pleroma.Constants
use Pleroma.DataCase
+ use Oban.Testing, repo: Pleroma.Repo
import Pleroma.Factory
import Tesla.Mock
@@ -13,6 +14,7 @@ defmodule Pleroma.Search.MeilisearchTest do
alias Pleroma.Search.Meilisearch
alias Pleroma.Web.CommonAPI
+ alias Pleroma.Workers.SearchIndexingWorker
setup_all do
Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
@@ -27,7 +29,8 @@ defmodule Pleroma.Search.MeilisearchTest do
{Meilisearch, [:passthrough],
[
add_to_index: fn a -> passthrough([a]) end,
- remove_from_index: fn a -> passthrough([a]) end
+ remove_from_index: fn a -> passthrough([a]) end,
+ meili_put: fn u, a -> passthrough([u, a]) end
]}
],
context,
@@ -38,7 +41,7 @@ defmodule Pleroma.Search.MeilisearchTest do
user = insert(:user)
mock_global(fn
- %{method: :post, url: "http://127.0.0.1:7700/indexes/objects/documents", body: body} ->
+ %{method: :put, url: "http://127.0.0.1:7700/indexes/objects/documents", body: body} ->
assert match?(
[%{"content" => "guys i just don't wanna leave the swamp"}],
Jason.decode!(body)
@@ -53,6 +56,15 @@ defmodule Pleroma.Search.MeilisearchTest do
visibility: "public"
})
+ args = %{"op" => "add_to_index", "activity" => activity.id}
+
+ assert_enqueued(
+ worker: SearchIndexingWorker,
+ args: args
+ )
+
+ assert :ok = perform_job(SearchIndexingWorker, args)
+
assert_called(Meilisearch.add_to_index(activity))
end
@@ -60,26 +72,25 @@ defmodule Pleroma.Search.MeilisearchTest do
user = insert(:user)
Enum.each(["unlisted", "private", "direct"], fn visiblity ->
- {:ok, _} =
+ {:ok, activity} =
CommonAPI.post(user, %{
status: "guys i just don't wanna leave the swamp",
visibility: visiblity
})
+
+ Meilisearch.add_to_index(activity)
+ assert_not_called(Meilisearch.meili_put(:_))
end)
history = call_history(Meilisearch)
assert Enum.count(history) == 3
-
- Enum.each(history, fn {_, _, return} ->
- assert is_nil(return)
- end)
end
test "deletes posts from index when deleted locally" do
user = insert(:user)
mock_global(fn
- %{method: :post, url: "http://127.0.0.1:7700/indexes/objects/documents", body: body} ->
+ %{method: :put, url: "http://127.0.0.1:7700/indexes/objects/documents", body: body} ->
assert match?(
[%{"content" => "guys i just don't wanna leave the swamp"}],
Jason.decode!(body)
@@ -98,10 +109,16 @@ defmodule Pleroma.Search.MeilisearchTest do
visibility: "public"
})
- assert_called(Meilisearch.add_to_index(activity))
+ args = %{"op" => "add_to_index", "activity" => activity.id}
+ assert_enqueued(worker: SearchIndexingWorker, args: args)
+ assert :ok = perform_job(SearchIndexingWorker, args)
{:ok, _} = CommonAPI.delete(activity.id, user)
+ delete_args = %{"op" => "remove_from_index", "object" => activity.object.id}
+ assert_enqueued(worker: SearchIndexingWorker, args: delete_args)
+ assert :ok = perform_job(SearchIndexingWorker, delete_args)
+
assert_called(Meilisearch.remove_from_index(:_))
end
end
From 3387935e8354e32171fe6e28a8f96f49154acbb3 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sat, 22 Jan 2022 16:52:06 +0300
Subject: [PATCH 072/823] Don't try removing deleted users and such from index
as posts
---
lib/pleroma/search.ex | 8 ++++----
lib/pleroma/web/activity_pub/side_effects.ex | 5 ++++-
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex
index af858fc46..3b266e59b 100644
--- a/lib/pleroma/search.ex
+++ b/lib/pleroma/search.ex
@@ -1,12 +1,12 @@
defmodule Pleroma.Search do
alias Pleroma.Workers.SearchIndexingWorker
- def add_to_index(activity) do
- SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity.id})
+ def add_to_index(%Pleroma.Activity{id: activity_id}) do
+ SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity_id})
end
- def remove_from_index(object) do
- SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object.id})
+ def remove_from_index(%Pleroma.Object{id: object_id}) do
+ SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object_id})
end
def search(query, options) do
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 4762b5ac6..644e62630 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -330,7 +330,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
if result == :ok do
Notification.create_notifications(object)
- Pleroma.Search.remove_from_index(deleted_object)
+ # Only remove from index when deleting actual objects, not users or anything else
+ with %Pleroma.Object{} <- deleted_object do
+ Pleroma.Search.remove_from_index(deleted_object)
+ end
{:ok, object, meta}
else
From fd2cfc80d2853c27f4d0c07631849da9b8d73e85 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sat, 22 Jan 2022 17:17:43 +0300
Subject: [PATCH 073/823] Change search_indexing = 10 and retries for indexing
= 2
---
config/config.exs | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/config/config.exs b/config/config.exs
index 3e2f0da3f..ce9338014 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -572,7 +572,7 @@ config :pleroma, Oban,
attachments_cleanup: 1,
new_users_digest: 1,
mute_expire: 5,
- search_indexing: 1
+ search_indexing: 10
],
plugins: [Oban.Plugins.Pruner],
crontab: [
@@ -583,7 +583,8 @@ config :pleroma, Oban,
config :pleroma, :workers,
retries: [
federator_incoming: 5,
- federator_outgoing: 5
+ federator_outgoing: 5,
+ search_indexing: 2
]
config :pleroma, Pleroma.Formatter,
From 79225d9b0adcd848502e5ba0bbbb295855a30ba0 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sat, 22 Jan 2022 21:09:53 +0300
Subject: [PATCH 074/823] Actually, unlisted posts are indexed
---
test/pleroma/search/meilisearch_test.exs | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/test/pleroma/search/meilisearch_test.exs b/test/pleroma/search/meilisearch_test.exs
index da614577f..04a2d75d9 100644
--- a/test/pleroma/search/meilisearch_test.exs
+++ b/test/pleroma/search/meilisearch_test.exs
@@ -71,19 +71,23 @@ defmodule Pleroma.Search.MeilisearchTest do
test "doesn't index posts that are not public" do
user = insert(:user)
- Enum.each(["unlisted", "private", "direct"], fn visiblity ->
+ Enum.each(["private", "direct"], fn visibility ->
{:ok, activity} =
CommonAPI.post(user, %{
status: "guys i just don't wanna leave the swamp",
- visibility: visiblity
+ visibility: visibility
})
- Meilisearch.add_to_index(activity)
+ args = %{"op" => "add_to_index", "activity" => activity.id}
+
+ assert_enqueued(worker: SearchIndexingWorker, args: args)
+ assert :ok = perform_job(SearchIndexingWorker, args)
+
assert_not_called(Meilisearch.meili_put(:_))
end)
history = call_history(Meilisearch)
- assert Enum.count(history) == 3
+ assert Enum.count(history) == 2
end
test "deletes posts from index when deleted locally" do
From 1e23f527e3e22108b402552a0766e488048ed3f4 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Tue, 22 Mar 2022 20:29:17 +0300
Subject: [PATCH 075/823] Change the meilisearch key auth to conform to 0.25.0
---
docs/configuration/search.md | 6 +++---
lib/mix/tasks/pleroma/search/meilisearch.ex | 14 ++++++++------
lib/pleroma/search/meilisearch.ex | 2 +-
3 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index a785a18ad..82217e5ee 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -47,15 +47,15 @@ you have to get the _private key_, which is actually used for authentication.
=== "OTP"
```sh
- ./bin/pleroma_ctl search.meilisearch show-private-key
+ ./bin/pleroma_ctl search.meilisearch show-keys
```
=== "From Source"
```sh
- mix pleroma.search.meilisearch show-private-key
+ mix pleroma.search.meilisearch show-keys
```
-This is the key you actually put into your configuration file.
+You will see a "Default Admin API Key", this is the key you actually put into your configuration file.
### Initial indexing
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 5098668ad..db56876fa 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -22,7 +22,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
)
# The ranking rule syntax was changed but nothing about that is mentioned in the changelog
- if not Version.match?(meili_version, ">= 0.24.0") do
+ if not Version.match?(meili_version, ">= 0.25.0") do
raise "Meilisearch <0.24.0 not supported"
end
@@ -112,7 +112,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
meili_delete!("/indexes/objects/documents")
end
- def run(["show-private-key", master_key]) do
+ def run(["show-keys", master_key]) do
start_pleroma()
endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
@@ -120,15 +120,17 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
{:ok, result} =
Pleroma.HTTP.get(
Path.join(endpoint, "/keys"),
- [{"X-Meili-API-Key", master_key}]
+ [{"Authorization", "Bearer #{master_key}"}]
)
decoded = Jason.decode!(result.body)
- if decoded["private"] do
- IO.puts(decoded["private"])
+ if decoded["results"] do
+ Enum.each(decoded["results"], fn %{"description" => desc, "key" => key} ->
+ IO.puts("#{desc}: #{key}")
+ end)
else
- IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}")
+ IO.puts("Error fetching the keys, check the master key is correct: #{inspect(decoded)}")
end
end
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 33bbf8392..0f9182ffc 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -13,7 +13,7 @@ defmodule Pleroma.Search.Meilisearch do
private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key])
[{"Content-Type", "application/json"}] ++
- if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}]
+ if is_nil(private_key), do: [], else: [{"Authorization", "Bearer #{private_key}"}]
end
def meili_get(path) do
From 84608be87e2c5961a4deb9030307c978bf1168e5 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Tue, 22 Mar 2022 20:45:49 +0300
Subject: [PATCH 076/823] Change updateId to uid because apparently that's the
new name
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +-
lib/pleroma/search/meilisearch.ex | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index db56876fa..d4a83c3cd 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -91,7 +91,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
)
with {:ok, res} <- result do
- if not Map.has_key?(res, "updateId") do
+ if not Map.has_key?(res, "uid") do
IO.puts("\nFailed to index: #{inspect(result)}")
end
else
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 0f9182ffc..3db65f261 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -153,7 +153,7 @@ defmodule Pleroma.Search.Meilisearch do
)
with {:ok, res} <- result,
- true <- Map.has_key?(res, "updateId") do
+ true <- Map.has_key?(res, "uid") do
# Do nothing
else
_ ->
From b150e6f15e0f06c8e23c0ac66aeaf80eb2f8c31a Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Wed, 23 Mar 2022 11:36:01 +0300
Subject: [PATCH 077/823] Update meilisearch docs
---
docs/configuration/search.md | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index 82217e5ee..f131948a7 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -37,8 +37,10 @@ indexes faster when it can process many posts in a single batch.
Information about setting up meilisearch can be found in the
[official documentation](https://docs.meilisearch.com/learn/getting_started/installation.html).
-You probably want to start it with `MEILI_NO_ANALYTICS=true` and `MEILI_NO_CENTRY=true` environment variables,
-to disable analytics.
+You probably want to start it with `MEILI_NO_ANALYTICS=true` environment variable to disable analytics.
+At least version 0.25.0 is required, but you are strongly adviced to use at least 0.26.0, as it introduces
+the `--enable-auto-batching` option which drastically improves performance. Without this option, the search
+is hardly usable on a somewhat big instance.
### Private key authentication (optional)
From e20f74c71b078d706bc93632773f9b590d2fb018 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Fri, 26 Aug 2022 23:39:58 +0300
Subject: [PATCH 078/823] Remove duplicate function call
---
lib/pleroma/search/meilisearch.ex | 1 -
1 file changed, 1 deletion(-)
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 3db65f261..53f8a2544 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -93,7 +93,6 @@ defmodule Pleroma.Search.Meilisearch do
hits
|> Activity.create_by_object_ap_id()
|> Activity.with_preloaded_object()
- |> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
From 119b2b847b76c7300bd71699d9f2e5676bdb0bb4 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sat, 27 Aug 2022 00:09:37 +0300
Subject: [PATCH 079/823] Instead of checking string length, explicitly check
for "" and "."
---
lib/pleroma/search/meilisearch.ex | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 53f8a2544..2e13b8407 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -112,7 +112,7 @@ defmodule Pleroma.Search.Meilisearch do
not is_nil(object.data["content"]) and
(Pleroma.Constants.as_public() in object.data["to"] or
Pleroma.Constants.as_public() in object.data["cc"]) and
- String.length(object.data["content"]) > 1 do
+ object.data["content"] not in ["", "."] do
data = object.data
content_str =
@@ -127,7 +127,8 @@ defmodule Pleroma.Search.Meilisearch do
trimmed
end
- if String.length(content) > 1 do
+ # Make sure we have a non-empty string
+ if content != "" do
{:ok, published, _} = DateTime.from_iso8601(data["published"])
%{
From 102ebb42bdba1673da39a8fa8ed1662bc8565aa4 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sat, 27 Aug 2022 00:19:08 +0300
Subject: [PATCH 080/823] Make search a callback
---
lib/pleroma/search/database_search.ex | 1 +
lib/pleroma/search/meilisearch.ex | 1 +
lib/pleroma/search/search_backend.ex | 11 +++++++++--
3 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex
index 3735a5fab..9a340abf1 100644
--- a/lib/pleroma/search/database_search.ex
+++ b/lib/pleroma/search/database_search.ex
@@ -15,6 +15,7 @@ defmodule Pleroma.Search.DatabaseSearch do
@behaviour Pleroma.Search.SearchBackend
+ @impl true
def search(user, search_query, options \\ []) do
index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin
limit = Enum.min([Keyword.get(options, :limit), 40])
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 2e13b8407..4e88169d2 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -75,6 +75,7 @@ defmodule Pleroma.Search.Meilisearch do
)
end
+ @impl true
def search(user, query, options \\ []) do
limit = Enum.min([Keyword.get(options, :limit), 40])
offset = Keyword.get(options, :offset, 0)
diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex
index ed6bfd329..a42e2f5f6 100644
--- a/lib/pleroma/search/search_backend.ex
+++ b/lib/pleroma/search/search_backend.ex
@@ -1,10 +1,17 @@
defmodule Pleroma.Search.SearchBackend do
+ @doc """
+ Search statuses with a query, restricting to only those the user should have access to.
+ """
+ @callback search(user :: Pleroma.User.t(), query :: String.t(), options :: [any()]) :: [
+ Pleroma.Activity.t()
+ ]
+
@doc """
Add the object associated with the activity to the search index.
The whole activity is passed, to allow filtering on things such as scope.
"""
- @callback add_to_index(activity :: Pleroma.Activity.t()) :: nil
+ @callback add_to_index(activity :: Pleroma.Activity.t()) :: :ok | {:error, any()}
@doc """
Remove the object from the index.
@@ -13,5 +20,5 @@ defmodule Pleroma.Search.SearchBackend do
is what contains the actual content and there is no need for fitlering when removing
from index.
"""
- @callback remove_from_index(object :: Pleroma.Object.t()) :: nil
+ @callback remove_from_index(object :: Pleroma.Object.t()) :: {:ok, any()} | {:error, any()}
end
From 5ac67632384bfb284ac51f2a450d41cf3913378a Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sat, 27 Aug 2022 00:31:36 +0300
Subject: [PATCH 081/823] Make add_to_index and remove_from_index report errors
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 4 +--
lib/pleroma/search/meilisearch.ex | 27 +++++++++++--------
lib/pleroma/workers/search_indexing_worker.ex | 4 ---
3 files changed, 18 insertions(+), 17 deletions(-)
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index d4a83c3cd..72a558228 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -9,7 +9,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
import Ecto.Query
import Pleroma.Search.Meilisearch,
- only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete!: 1]
+ only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete: 1]
def run(["index"]) do
start_pleroma()
@@ -109,7 +109,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
def run(["clear"]) do
start_pleroma()
- meili_delete!("/indexes/objects/documents")
+ meili_delete("/indexes/objects/documents")
end
def run(["show-keys", master_key]) do
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 4e88169d2..24789b00c 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -62,17 +62,16 @@ defmodule Pleroma.Search.Meilisearch do
end
end
- def meili_delete!(path) do
+ def meili_delete(path) do
endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
- {:ok, _} =
- Pleroma.HTTP.request(
- :delete,
- Path.join(endpoint, path),
- "",
- meili_headers(),
- []
- )
+ Pleroma.HTTP.request(
+ :delete,
+ Path.join(endpoint, path),
+ "",
+ meili_headers(),
+ []
+ )
end
@impl true
@@ -155,16 +154,22 @@ defmodule Pleroma.Search.Meilisearch do
with {:ok, res} <- result,
true <- Map.has_key?(res, "uid") do
- # Do nothing
+ # Added successfully
+ :ok
else
_ ->
+ # There was an error, report it
Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}")
+ {:error, result}
end
+ else
+ # The post isn't something we can search, that's ok
+ :ok
end
end
@impl true
def remove_from_index(object) do
- meili_delete!("/indexes/objects/documents/#{object.id}")
+ meili_delete("/indexes/objects/documents/#{object.id}")
end
end
diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex
index 70a8d42d0..43b7bad1e 100644
--- a/lib/pleroma/workers/search_indexing_worker.ex
+++ b/lib/pleroma/workers/search_indexing_worker.ex
@@ -9,8 +9,6 @@ defmodule Pleroma.Workers.SearchIndexingWorker do
search_module = Pleroma.Config.get([Pleroma.Search, :module])
search_module.add_to_index(activity)
-
- :ok
end
def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do
@@ -19,7 +17,5 @@ defmodule Pleroma.Workers.SearchIndexingWorker do
search_module = Pleroma.Config.get([Pleroma.Search, :module])
search_module.remove_from_index(object)
-
- :ok
end
end
From 6256822afd368e5f6b410d47c5ff9b584e50a461 Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sat, 27 Aug 2022 01:11:50 +0300
Subject: [PATCH 082/823] Check for updateId, not uid
---
lib/pleroma/search/meilisearch.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 24789b00c..0b90971b1 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -153,7 +153,7 @@ defmodule Pleroma.Search.Meilisearch do
)
with {:ok, res} <- result,
- true <- Map.has_key?(res, "uid") do
+ true <- Map.has_key?(res, "updateId") do
# Added successfully
:ok
else
From 5a39866388c411f2bcee9848352f8c420513f34f Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis
Date: Sat, 27 Aug 2022 01:43:59 +0300
Subject: [PATCH 083/823] Specifically strip mentions for search indexing
---
lib/mix/tasks/pleroma/search/meilisearch.ex | 1 +
lib/pleroma/search/meilisearch.ex | 3 ++-
priv/scrubbers/search_indexing.ex | 24 +++++++++++++++++++++
3 files changed, 27 insertions(+), 1 deletion(-)
create mode 100644 priv/scrubbers/search_indexing.ex
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
index 72a558228..8379a0c25 100644
--- a/lib/mix/tasks/pleroma/search/meilisearch.ex
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -13,6 +13,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
def run(["index"]) do
start_pleroma()
+ Pleroma.HTML.compile_scrubbers()
meili_version =
(
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 0b90971b1..7af7f460a 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -122,7 +122,8 @@ defmodule Pleroma.Search.Meilisearch do
end
content =
- with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str),
+ with {:ok, scrubbed} <-
+ FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing),
trimmed <- String.trim(scrubbed) do
trimmed
end
diff --git a/priv/scrubbers/search_indexing.ex b/priv/scrubbers/search_indexing.ex
new file mode 100644
index 000000000..02756ab79
--- /dev/null
+++ b/priv/scrubbers/search_indexing.ex
@@ -0,0 +1,24 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTML.Scrubber.SearchIndexing do
+ @moduledoc """
+ An HTML scrubbing policy that scrubs things for searching.
+ """
+
+ require FastSanitize.Sanitizer.Meta
+ alias FastSanitize.Sanitizer.Meta
+
+ # Explicitly remove mentions
+ def scrub({:a, attrs, children}) do
+ if(Enum.any?(attrs, fn {att, val} -> att == "class" and String.contains?(val, "mention") end),
+ do: nil,
+ # Strip the tag itself, leave only children (text, presumably)
+ else: children
+ )
+ end
+
+ Meta.strip_comments()
+ Meta.strip_everything_not_covered()
+end
From 78d1105bffee7ece8a2b972d3cb58a6e41d86828 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Sun, 19 Feb 2023 22:02:38 +0100
Subject: [PATCH 084/823] Fix down migration
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
.../20220319000000_add_status_to_notifications_enum.exs | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs b/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs
index 62c0afb63..c3bc85894 100644
--- a/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs
+++ b/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs
@@ -36,7 +36,8 @@ defmodule Pleroma.Repo.Migrations.AddStatusToNotificationsEnum do
'reblog',
'favourite',
'pleroma:report',
- 'poll
+ 'poll',
+ 'update'
)
"""
|> execute()
From 675639225a905f5b0b2650cd3f20a4758fc3f868 Mon Sep 17 00:00:00 2001
From: HJ <30-hj@users.noreply.git.pleroma.social>
Date: Fri, 28 Apr 2023 11:13:42 +0000
Subject: [PATCH 085/823] allow https: so that flash works across instances
without need for media proxy
---
lib/pleroma/web/plugs/http_security_plug.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex
index 34895c8d5..045384e08 100644
--- a/lib/pleroma/web/plugs/http_security_plug.ex
+++ b/lib/pleroma/web/plugs/http_security_plug.ex
@@ -104,7 +104,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
{[img_src, " https:"], [media_src, " https:"]}
end
- connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url]
+ connect_src = ["connect-src 'self' blob: https: ", static_url, ?\s, websocket_url]
connect_src =
if Config.get(:env) == :dev do
From cd20d15bb8d2f97f8dd0850993041f15865cdda9 Mon Sep 17 00:00:00 2001
From: HJ <30-hj@users.noreply.git.pleroma.social>
Date: Fri, 28 Apr 2023 11:19:14 +0000
Subject: [PATCH 086/823] changelog
---
changelog.d/3879.fix | 1 +
1 file changed, 1 insertion(+)
create mode 100644 changelog.d/3879.fix
diff --git a/changelog.d/3879.fix b/changelog.d/3879.fix
new file mode 100644
index 000000000..7c58cc3c2
--- /dev/null
+++ b/changelog.d/3879.fix
@@ -0,0 +1 @@
+fix not being able to fetch flash file from remote instance
\ No newline at end of file
From c0d11da2d8edc57ef88163c06a19aad3e28d14db Mon Sep 17 00:00:00 2001
From: Henry Jameson
Date: Sun, 7 May 2023 15:16:30 +0300
Subject: [PATCH 087/823] conditionally set csp depnding on media-proxy state
---
lib/pleroma/web/plugs/http_security_plug.ex | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex
index 045384e08..df46cfa0c 100644
--- a/lib/pleroma/web/plugs/http_security_plug.ex
+++ b/lib/pleroma/web/plugs/http_security_plug.ex
@@ -93,18 +93,26 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
img_src = "img-src 'self' data: blob:"
media_src = "media-src 'self'"
+ connect_src = ["connect-src 'self' blob:", static_url, ?\s, websocket_url]
# Strict multimedia CSP enforcement only when MediaProxy is enabled
- {img_src, media_src} =
+ {img_src, media_src, connect_src} =
if Config.get([:media_proxy, :enabled]) &&
!Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
sources = build_csp_multimedia_source_list()
- {[img_src, sources], [media_src, sources]}
+ {
+ [img_src, sources],
+ [media_src, sources],
+ [connect_src, sources]
+ }
else
- {[img_src, " https:"], [media_src, " https:"]}
+ {
+ [img_src, " https:"],
+ [media_src, " https:"],
+ [connect_src, " https:"]
+ }
end
- connect_src = ["connect-src 'self' blob: https: ", static_url, ?\s, websocket_url]
connect_src =
if Config.get(:env) == :dev do
From f8ef4924ecab5ba6851eee82845624bc15f868de Mon Sep 17 00:00:00 2001
From: Henry Jameson
Date: Sun, 7 May 2023 15:24:09 +0300
Subject: [PATCH 088/823] fix whitespace
---
lib/pleroma/web/plugs/http_security_plug.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex
index df46cfa0c..a3166bc96 100644
--- a/lib/pleroma/web/plugs/http_security_plug.ex
+++ b/lib/pleroma/web/plugs/http_security_plug.ex
@@ -93,7 +93,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
img_src = "img-src 'self' data: blob:"
media_src = "media-src 'self'"
- connect_src = ["connect-src 'self' blob:", static_url, ?\s, websocket_url]
+ connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url]
# Strict multimedia CSP enforcement only when MediaProxy is enabled
{img_src, media_src, connect_src} =
From f50fd9278fd36e6bd3ae36bb7f5033d9fd8a84ac Mon Sep 17 00:00:00 2001
From: Henry Jameson
Date: Sun, 7 May 2023 15:29:19 +0300
Subject: [PATCH 089/823] reduce redundant reduntancy reduction
---
lib/pleroma/web/plugs/http_security_plug.ex | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex
index a3166bc96..b189d5bfd 100644
--- a/lib/pleroma/web/plugs/http_security_plug.ex
+++ b/lib/pleroma/web/plugs/http_security_plug.ex
@@ -93,7 +93,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
img_src = "img-src 'self' data: blob:"
media_src = "media-src 'self'"
- connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url]
+ connect_src = "connect-src 'self' blob:"
# Strict multimedia CSP enforcement only when MediaProxy is enabled
{img_src, media_src, connect_src} =
@@ -103,7 +103,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
{
[img_src, sources],
[media_src, sources],
- [connect_src, sources]
+ [connect_src, sources, ?\s, websocket_url]
}
else
{
From 2a07411b0cb14ea26966659605d95074b02a8538 Mon Sep 17 00:00:00 2001
From: Henry Jameson
Date: Sun, 7 May 2023 15:34:17 +0300
Subject: [PATCH 090/823] keep the websocket url for all modes
---
lib/pleroma/web/plugs/http_security_plug.ex | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex
index b189d5bfd..b3dc8a3a6 100644
--- a/lib/pleroma/web/plugs/http_security_plug.ex
+++ b/lib/pleroma/web/plugs/http_security_plug.ex
@@ -93,7 +93,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
img_src = "img-src 'self' data: blob:"
media_src = "media-src 'self'"
- connect_src = "connect-src 'self' blob:"
+ connect_src = ["connect-src 'self' blob: ", ?\s, websocket_url]
# Strict multimedia CSP enforcement only when MediaProxy is enabled
{img_src, media_src, connect_src} =
@@ -103,7 +103,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
{
[img_src, sources],
[media_src, sources],
- [connect_src, sources, ?\s, websocket_url]
+ [connect_src, sources]
}
else
{
From 9363ef53a34c9d96191bccaece76dd4e01f493b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Sun, 14 May 2023 15:02:58 +0200
Subject: [PATCH 091/823] Add test for 'status' notification type for
NotificationView
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
.../views/notification_view_test.exs | 27 +++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs
index 6ea894691..92de6c6a7 100644
--- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs
@@ -286,4 +286,31 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do
test_notifications_rendering([notification], user, [expected])
end
+
+ test "Subscribed status notification" do
+ user = insert(:user)
+ subscriber = insert(:user)
+
+ User.subscribe(subscriber, user)
+
+ {:ok, activity} = CommonAPI.post(user, %{status: "hi"})
+ {:ok, [notification]} = Notification.create_notifications(activity)
+
+ user = User.get_cached_by_id(user.id)
+
+ expected = %{
+ id: to_string(notification.id),
+ pleroma: %{is_seen: false, is_muted: false},
+ type: "status",
+ account:
+ AccountView.render("show.json", %{
+ user: user,
+ for: subscriber
+ }),
+ status: StatusView.render("show.json", %{activity: activity, for: subscriber}),
+ created_at: Utils.to_masto_date(notification.inserted_at)
+ }
+
+ test_notifications_rendering([notification], subscriber, [expected])
+ end
end
From bed44f9f628c3e184ba367765c87c47656060f28 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Wed, 31 May 2023 13:10:18 -0400
Subject: [PATCH 092/823] Update to Phoenix 1.7
Chase the other deps to match versions you get from a current "mix phx.new" project creation
---
mix.exs | 15 ++++++++-------
mix.lock | 24 +++++++++++++-----------
2 files changed, 21 insertions(+), 18 deletions(-)
diff --git a/mix.exs b/mix.exs
index bb16f0a9c..3fcbfffaf 100644
--- a/mix.exs
+++ b/mix.exs
@@ -113,18 +113,19 @@ defmodule Pleroma.Mixfile do
# Type `mix help deps` for examples and options.
defp deps do
[
- {:phoenix, "~> 1.6.0"},
- {:phoenix_ecto, "~> 4.4.0"},
+ {:phoenix, "~> 1.7.3"},
+ {:phoenix_ecto, "~> 4.4"},
+ {:ecto_sql, "~> 3.10"},
{:ecto_enum, "~> 1.4"},
{:postgrex, ">= 0.0.0"},
- {:phoenix_html, "~> 3.1"},
+ {:phoenix_html, "~> 3.3"},
{:phoenix_live_reload, "~> 1.3.3", only: :dev},
- {:phoenix_live_view, "~> 0.17.1"},
- {:phoenix_live_dashboard, "~> 0.6.2"},
- {:telemetry_metrics, "~> 0.6.1"},
+ {:phoenix_live_view, "~> 0.19.0"},
+ {:phoenix_live_dashboard, "~> 0.8.0"},
+ {:telemetry_metrics, "~> 0.6"},
{:telemetry_poller, "~> 1.0"},
{:tzdata, "~> 1.0.3"},
- {:plug_cowboy, "~> 2.3"},
+ {:plug_cowboy, "~> 2.5"},
# oban 2.14 requires Elixir 1.12+
{:oban, "~> 2.13.4"},
{:gettext,
diff --git a/mix.lock b/mix.lock
index 43012a58e..fd8e02f95 100644
--- a/mix.lock
+++ b/mix.lock
@@ -16,22 +16,22 @@
"connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"},
"cors_plug": {:hex, :cors_plug, "2.0.3", "316f806d10316e6d10f09473f19052d20ba0a0ce2a1d910ddf57d663dac402ae", [:mix], [{:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ee4ae1418e6ce117fc42c2ba3e6cbdca4e95ecd2fe59a05ec6884ca16d469aea"},
"covertool": {:hex, :covertool, "2.0.4", "54acff6cddd88d28dea663cd2e1fe20dd32fcf5f5d3aff7d59031ce44ce39efa", [:rebar3], [], "hexpm", "5c9568ba4308fda2082172737c80c31d991ea83961eb10791f06106a870d0cdc"},
- "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"},
+ "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"},
"cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
"cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"},
"credo": {:hex, :credo, "1.7.0", "6119bee47272e85995598ee04f2ebbed3e947678dee048d10b5feca139435f75", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "6839fcf63d1f0d1c0f450abc8564a57c43d644077ab96f2934563e68b8a769d7"},
"crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
"custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"},
- "db_connection": {:hex, :db_connection, "2.4.3", "3b9aac9f27347ec65b271847e6baeb4443d8474289bd18c1d6f4de655b70c94d", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c127c15b0fa6cfb32eed07465e05da6c815b032508d4ed7c116122871df73c12"},
- "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"},
+ "db_connection": {:hex, :db_connection, "2.5.0", "bb6d4f30d35ded97b29fe80d8bd6f928a1912ca1ff110831edcd238a1973652c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c92d5ba26cd69ead1ff7582dbb860adeedfff39774105a4f1c92cbb654b55aa2"},
+ "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
"deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
"earmark": {:hex, :earmark, "1.4.22", "ea3e45c6359446dc308be0a64ce82a03260d973de7d0625a762e6d352ff57958", [:mix], [{:earmark_parser, "~> 1.4.23", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "1caf5145665a42fd76d5317286b0c171861fb1c04f86ab103dde76868814fdfb"},
"earmark_parser": {:hex, :earmark_parser, "1.4.31", "a93921cdc6b9b869f519213d5bc79d9e218ba768d7270d46fdcf1c01bacff9e2", [:mix], [], "hexpm", "317d367ee0335ef037a87e46c91a2269fef6306413f731e8ec11fc45a7efd059"},
"eblurhash": {:hex, :eblurhash, "1.2.2", "7da4255aaea984b31bb71155f673257353b0e0554d0d30dcf859547e74602582", [:rebar3], [], "hexpm", "8c20ca00904de023a835a9dcb7b7762fed32264c85a80c3cafa85288e405044c"},
- "ecto": {:hex, :ecto, "3.9.5", "9f0aa7ae44a1577b651c98791c6988cd1b69b21bc724e3fd67090b97f7604263", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d4f3115d8cbacdc0bfa4b742865459fb1371d0715515842a1fb17fe31920b74c"},
+ "ecto": {:hex, :ecto, "3.10.1", "c6757101880e90acc6125b095853176a02da8f1afe056f91f1f90b80c9389822", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d2ac4255f1601bdf7ac74c0ed971102c6829dc158719b94bd30041bbad77f87a"},
"ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},
- "ecto_psql_extras": {:hex, :ecto_psql_extras, "0.7.10", "e14d400930f401ca9f541b3349212634e44027d7f919bbb71224d7ac0d0e8acd", [:mix], [{:ecto_sql, "~> 3.4", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.15.7 or ~> 0.16.0", [hex: :postgrex, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "505e8cd81e4f17c090be0f99e92b1b3f0fd915f98e76965130b8ccfb891e7088"},
- "ecto_sql": {:hex, :ecto_sql, "3.9.2", "34227501abe92dba10d9c3495ab6770e75e79b836d114c41108a4bf2ce200ad5", [:mix], [{:db_connection, "~> 2.5 or ~> 2.4.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1eb5eeb4358fdbcd42eac11c1fbd87e3affd7904e639d77903c1358b2abd3f70"},
+ "ecto_psql_extras": {:hex, :ecto_psql_extras, "0.7.11", "6e20144c1446dcccfcdb4c142c9d8b7992a90a569b1d5958cbea5458550b25f0", [:mix], [{:ecto_sql, "~> 3.4", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.15.7 or ~> 0.16.0 or ~> 0.17.0", [hex: :postgrex, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "def61f1f92d4f40d51c80bbae2157212d6c0a459eb604be446e47369cbd40b23"},
+ "ecto_sql": {:hex, :ecto_sql, "3.10.1", "6ea6b3036a0b0ca94c2a02613fd9f742614b5cfe494c41af2e6571bb034dd94c", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f6a25bdbbd695f12c8171eaff0851fa4c8e72eec1e98c7364402dda9ce11c56b"},
"eimp": {:hex, :eimp, "1.0.14", "fc297f0c7e2700457a95a60c7010a5f1dcb768a083b6d53f49cd94ab95a28f22", [:rebar3], [{:p1_utils, "1.0.18", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "501133f3112079b92d9e22da8b88bf4f0e13d4d67ae9c15c42c30bd25ceb83b6"},
"elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"},
"esbuild": {:hex, :esbuild, "0.5.0", "d5bb08ff049d7880ee3609ed5c4b864bd2f46445ea40b16b4acead724fb4c4a3", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "f183a0b332d963c4cfaf585477695ea59eef9a6f2204fdd0efa00e099694ffe5"},
@@ -84,13 +84,13 @@
"open_api_spex": {:hex, :open_api_spex, "3.16.1", "8137c338129d63060b4b04947c6c57429f86267045c479c703a38a6d3f98dee1", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "ef6fd778ac121af866b48b75ad4ad256b6ff33949113ea4aa1629af8bfdfdbfb"},
"parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"},
"pbkdf2_elixir": {:hex, :pbkdf2_elixir, "1.2.1", "9cbe354b58121075bd20eb83076900a3832324b7dd171a6895fab57b6bb2752c", [:mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}], "hexpm", "d3b40a4a4630f0b442f19eca891fcfeeee4c40871936fed2f68e1c4faa30481f"},
- "phoenix": {:hex, :phoenix, "1.6.16", "e5bdd18c7a06da5852a25c7befb72246de4ddc289182285f8685a40b7b5f5451", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 1.0 or ~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e15989ff34f670a96b95ef6d1d25bad0d9c50df5df40b671d8f4a669e050ac39"},
+ "phoenix": {:hex, :phoenix, "1.7.3", "4d8eca2c020c9ed81a28e7a8c60e0a4f6f9f7f6e12eb91dfd01301eac07424c1", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.4", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "6b1bc308758f95ecf3e0d795389440a2ca88a903e0fda1f921c780918c16d640"},
"phoenix_ecto": {:hex, :phoenix_ecto, "4.4.0", "0672ed4e4808b3fbed494dded89958e22fb882de47a97634c0b13e7b0b5f7720", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "09864e558ed31ee00bd48fcc1d4fc58ae9678c9e81649075431e69dbabb43cc1"},
"phoenix_html": {:hex, :phoenix_html, "3.3.1", "4788757e804a30baac6b3fc9695bf5562465dd3f1da8eb8460ad5b404d9a2178", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "bed1906edd4906a15fd7b412b85b05e521e1f67c9a85418c55999277e553d0d3"},
- "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.6.5", "1495bb014be12c9a9252eca04b9af54246f6b5c1e4cd1f30210cd00ec540cf8e", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.3", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.17.7", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "ef4fa50dd78364409039c99cf6f98ab5209b4c5f8796c17f4db118324f0db852"},
+ "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.0", "0b3158b5b198aa444473c91d23d79f52fb077e807ffad80dacf88ce078fa8df2", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "87785a54474fed91a67a1227a741097eb1a42c2e49d3c0d098b588af65cd410d"},
"phoenix_live_reload": {:hex, :phoenix_live_reload, "1.3.3", "3a53772a6118d5679bf50fc1670505a290e32a1d195df9e069d8c53ab040c054", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "766796676e5f558dbae5d1bdb066849673e956005e3730dfd5affd7a6da4abac"},
- "phoenix_live_view": {:hex, :phoenix_live_view, "0.17.14", "5ec615d4d61bf9d4755f158bd6c80372b715533fe6d6219e12d74fb5eedbeac1", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.0 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.1", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "afeb6ba43ce329a6f7fc1c9acdfc6d3039995345f025febb7f409a92f6faebd3"},
- "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.1", "ba04e489ef03763bf28a17eb2eaddc2c20c6d217e2150a61e3298b0f4c2012b5", [:mix], [], "hexpm", "81367c6d1eea5878ad726be80808eb5a787a23dee699f96e72b1109c57cdd8d9"},
+ "phoenix_live_view": {:hex, :phoenix_live_view, "0.19.0", "de5643d03e3cdf5ff19cd45b5d14543a3b1ad8551d529f6b24246e88a6c6f1b8", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a650b6f814c4f386314b98f1aebf92f8652649166612f84ef2e60a20894addfa"},
+ "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.2", "a4950b63ace57720b0fc1c6da083c53346b36f99021de89595cc4f026288ff51", [:mix], [], "hexpm", "45741676a94c71f9afdfed9d22d49b6856c026ff504db04e3dc03a1d86f8201c"},
"phoenix_swoosh": {:hex, :phoenix_swoosh, "1.2.0", "a544d83fde4a767efb78f45404a74c9e37b2a9c5ea3339692e65a6966731f935", [:mix], [{:finch, "~> 0.8", [hex: :finch, repo: "hexpm", optional: true]}, {:hackney, "~> 1.10", [hex: :hackney, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_view, "~> 1.0 or ~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: false]}, {:swoosh, "~> 1.5", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm", "e88d117251e89a16b92222415a6d87b99a96747ddf674fc5c7631de734811dba"},
"phoenix_template": {:hex, :phoenix_template, "1.0.1", "85f79e3ad1b0180abb43f9725973e3b8c2c3354a87245f91431eec60553ed3ef", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "157dc078f6226334c91cb32c1865bf3911686f8bcd6bcff86736f6253e6993ee"},
"phoenix_view": {:hex, :phoenix_view, "2.0.2", "6bd4d2fd595ef80d33b439ede6a19326b78f0f1d8d62b9a318e3d9c1af351098", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "a929e7230ea5c7ee0e149ffcf44ce7cf7f4b6d2bfe1752dd7c084cdff152d36f"},
@@ -100,7 +100,7 @@
"plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"},
"poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"},
"poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"},
- "postgrex": {:hex, :postgrex, "0.16.5", "fcc4035cc90e23933c5d69a9cd686e329469446ef7abba2cf70f08e2c4b69810", [:mix], [{:connection, "~> 1.1", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "edead639dc6e882618c01d8fc891214c481ab9a3788dfe38dd5e37fd1d5fb2e8"},
+ "postgrex": {:hex, :postgrex, "0.17.1", "01c29fd1205940ee55f7addb8f1dc25618ca63a8817e56fac4f6846fc2cddcbe", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "14b057b488e73be2beee508fb1955d8db90d6485c6466428fe9ccf1d6692a555"},
"pot": {:hex, :pot, "1.0.2", "13abb849139fdc04ab8154986abbcb63bdee5de6ed2ba7e1713527e33df923dd", [:rebar3], [], "hexpm", "78fe127f5a4f5f919d6ea5a2a671827bd53eb9d37e5b4128c0ad3df99856c2e0"},
"prom_ex": {:hex, :prom_ex, "1.7.1", "39331ee3fe6f9a8587d8208bf9274a253bb80281700e127dd18786cda5e08c37", [:mix], [{:absinthe, ">= 1.6.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.0.2", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.5.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.10.2", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.4.0", [hex: :oban, repo: "hexpm", optional: true]}, {:phoenix, ">= 1.5.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.14.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.12.1", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.5.1", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6.1", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.0.2", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.0.0", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "4c978872b88a929833925a0f4d0561824804c671fdd04581e765509ed0a6ed08"},
"prometheus": {:hex, :prometheus, "4.10.0", "792adbf0130ff61b5fa8826f013772af24b6e57b984445c8d602c8a0355704a1", [:mix, :rebar3], [{:quantile_estimator, "~> 0.2.1", [hex: :quantile_estimator, repo: "hexpm", optional: false]}], "hexpm", "2a99bb6dce85e238c7236fde6b0064f9834dc420ddbd962aac4ea2a3c3d59384"},
@@ -132,5 +132,7 @@
"unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
"unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"},
"web_push_encryption": {:hex, :web_push_encryption, "0.3.1", "76d0e7375142dfee67391e7690e89f92578889cbcf2879377900b5620ee4708d", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.11.1", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "4f82b2e57622fb9337559058e8797cb0df7e7c9790793bdc4e40bc895f70e2a2"},
+ "websock": {:hex, :websock, "0.5.1", "c496036ce95bc26d08ba086b2a827b212c67e7cabaa1c06473cd26b40ed8cf10", [:mix], [], "hexpm", "b9f785108b81cd457b06e5f5dabe5f65453d86a99118b2c0a515e1e296dc2d2c"},
+ "websock_adapter": {:hex, :websock_adapter, "0.5.1", "292e6c56724e3457e808e525af0e9bcfa088cc7b9c798218e78658c7f9b85066", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "8e2e1544bfde5f9d0442f9cec2f5235398b224f75c9e06b60557debf64248ec1"},
"websockex": {:hex, :websockex, "0.4.3", "92b7905769c79c6480c02daacaca2ddd49de936d912976a4d3c923723b647bf0", [:mix], [], "hexpm", "95f2e7072b85a3a4cc385602d42115b73ce0b74a9121d0d6dbbf557645ac53e4"},
}
From 6e1ea2eafc24cb31e796e93313dcf9c6b6ba9845 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Wed, 31 May 2023 13:36:07 -0400
Subject: [PATCH 093/823] Remove unnecessary compilers as of Phoenix 1.7
---
mix.exs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mix.exs b/mix.exs
index 3fcbfffaf..9eafa2474 100644
--- a/mix.exs
+++ b/mix.exs
@@ -7,7 +7,7 @@ defmodule Pleroma.Mixfile do
version: version("2.5.52"),
elixir: "~> 1.11",
elixirc_paths: elixirc_paths(Mix.env()),
- compilers: [:phoenix, :gettext] ++ Mix.compilers(),
+ compilers: Mix.compilers(),
elixirc_options: [warnings_as_errors: warnings_as_errors()],
xref: [exclude: [:eldap]],
start_permanent: Mix.env() == :prod,
From e3110cb34e350bc9353d082328778daae47ba9e8 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Wed, 31 May 2023 13:36:21 -0400
Subject: [PATCH 094/823] Fix deprecated calls to get_flash/2
---
lib/pleroma/web.ex | 2 +-
lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex | 8 ++++----
lib/pleroma/web/templates/o_auth/mfa/totp.html.eex | 8 ++++----
lib/pleroma/web/templates/o_auth/o_auth/register.html.eex | 8 ++++----
lib/pleroma/web/templates/o_auth/o_auth/show.html.eex | 8 ++++----
test/pleroma/web/o_auth/o_auth_controller_test.exs | 6 +++---
6 files changed, 20 insertions(+), 20 deletions(-)
diff --git a/lib/pleroma/web.ex b/lib/pleroma/web.ex
index aee41b0fe..7a8b176cd 100644
--- a/lib/pleroma/web.ex
+++ b/lib/pleroma/web.ex
@@ -136,7 +136,7 @@ defmodule Pleroma.Web do
namespace: Pleroma.Web
# Import convenience functions from controllers
- import Phoenix.Controller, only: [get_csrf_token: 0, get_flash: 2, view_module: 1]
+ import Phoenix.Controller, only: [get_csrf_token: 0, view_module: 1]
import Pleroma.Web.ErrorHelpers
import Pleroma.Web.Gettext
diff --git a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex
index e45d13bdf..e3639aae7 100644
--- a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex
+++ b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex
@@ -1,8 +1,8 @@
-<%= if get_flash(@conn, :info) do %>
-<%= get_flash(@conn, :info) %>
+<%= if Phoenix.Flash.get(@flash, :info) do %>
+<%= Phoenix.Flash.get(@flash, :info) %>
<% end %>
-<%= if get_flash(@conn, :error) do %>
-<%= get_flash(@conn, :error) %>
+<%= if Phoenix.Flash.get(@flash, :error) do %>
+<%= Phoenix.Flash.get(@flash, :error) %>
<% end %>
<%= Gettext.dpgettext("static_pages", "mfa recover page title", "Two-factor recovery") %>
diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex
index 50e6c04b6..f995b8805 100644
--- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex
+++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex
@@ -1,8 +1,8 @@
-<%= if get_flash(@conn, :info) do %>
-<%= get_flash(@conn, :info) %>
+<%= if Phoenix.Flash.get(@flash, :info) do %>
+<%= Phoenix.Flash.get(@flash, :info) %>
<% end %>
-<%= if get_flash(@conn, :error) do %>
-<%= get_flash(@conn, :error) %>
+<%= if Phoenix.Flash.get(@flash, :error) do %>
+<%= Phoenix.Flash.get(@flash, :error) %>
<% end %>
<%= Gettext.dpgettext("static_pages", "mfa auth page title", "Two-factor authentication") %>
diff --git a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex
index 1f661efb2..e7f65266f 100644
--- a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex
+++ b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex
@@ -1,8 +1,8 @@
-<%= if get_flash(@conn, :info) do %>
- <%= get_flash(@conn, :info) %>
+<%= if Phoenix.Flash.get(@flash, :info) do %>
+ <%= Phoenix.Flash.get(@flash, :info) %>
<% end %>
-<%= if get_flash(@conn, :error) do %>
- <%= get_flash(@conn, :error) %>
+<%= if Phoenix.Flash.get(@flash, :error) do %>
+ <%= Phoenix.Flash.get(@flash, :error) %>
<% end %>
<%= Gettext.dpgettext("static_pages", "oauth register page title", "Registration Details") %>
diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex
index b3654f3eb..5b38f7142 100644
--- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex
+++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex
@@ -1,8 +1,8 @@
-<%= if get_flash(@conn, :info) do %>
-<%= get_flash(@conn, :info) %>
+<%= if Phoenix.Flash.get(@flash, :info) do %>
+<%= Phoenix.Flash.get(@flash, :info) %>
<% end %>
-<%= if get_flash(@conn, :error) do %>
-<%= get_flash(@conn, :error) %>
+<%= if Phoenix.Flash.get(@flash, :error) do %>
+<%= Phoenix.Flash.get(@flash, :error) %>
<% end %>
<%= form_for @conn, Routes.o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %>
diff --git a/test/pleroma/web/o_auth/o_auth_controller_test.exs b/test/pleroma/web/o_auth/o_auth_controller_test.exs
index f41d6a322..83a08d9fc 100644
--- a/test/pleroma/web/o_auth/o_auth_controller_test.exs
+++ b/test/pleroma/web/o_auth/o_auth_controller_test.exs
@@ -186,7 +186,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do
assert html_response(conn, 302)
assert redirected_to(conn) == app.redirect_uris
- assert get_flash(conn, :error) == "Failed to authenticate: (error description)."
+ assert conn.assigns.flash["error"] == "Failed to authenticate: (error description)."
end
test "GET /oauth/registration_details renders registration details form", %{
@@ -307,7 +307,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do
|> post("/oauth/register", bad_params)
assert html_response(conn, 403) =~ ~r/name="op" type="submit" value="register"/
- assert get_flash(conn, :error) == "Error: #{bad_param} has already been taken."
+ assert conn.assigns.flash["error"] == "Error: #{bad_param} has already been taken."
end
end
@@ -398,7 +398,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do
|> post("/oauth/register", params)
assert html_response(conn, 401) =~ ~r/name="op" type="submit" value="connect"/
- assert get_flash(conn, :error) == "Invalid Username/Password"
+ assert conn.assigns.flash["error"] == "Invalid Username/Password"
end
end
From f622f82c0e1ca66f1dc3493d900f60a24ab96865 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Wed, 31 May 2023 13:38:46 -0400
Subject: [PATCH 095/823] No user facing changes
---
changelog.d/3900.skip | 0
1 file changed, 0 insertions(+), 0 deletions(-)
create mode 100644 changelog.d/3900.skip
diff --git a/changelog.d/3900.skip b/changelog.d/3900.skip
new file mode 100644
index 000000000..e69de29bb
From a7e7db4a29a09dc2193455de696d5cda561db5d4 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Wed, 31 May 2023 13:47:15 -0400
Subject: [PATCH 096/823] Phoenix.Endpoint.Cowboy2Handler ->
Plug.Cowboy.Handler
---
config/config.exs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/config/config.exs b/config/config.exs
index 178b6be99..6c9eb9bfc 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -133,7 +133,7 @@ config :pleroma, Pleroma.Web.Endpoint,
{"/websocket", Phoenix.Endpoint.CowboyWebSocket,
{Phoenix.Transports.WebSocket,
{Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}},
- {:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}}
+ {:_, Plug.Cowboy.Handler, {Pleroma.Web.Endpoint, []}}
]}
]
],
From 4dc2d4bf7b39ca427fdb9be90a9789b7c4d9824e Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Wed, 31 May 2023 15:30:31 -0400
Subject: [PATCH 097/823] Remove locked version of plug
Required for Phoenix 1.7 websocket changes
---
mix.exs | 3 ---
mix.lock | 2 +-
2 files changed, 1 insertion(+), 4 deletions(-)
diff --git a/mix.exs b/mix.exs
index 9eafa2474..e346834e5 100644
--- a/mix.exs
+++ b/mix.exs
@@ -197,9 +197,6 @@ defmodule Pleroma.Mixfile do
{:open_api_spex, "~> 3.16"},
{:ecto_psql_extras, "~> 0.6"},
- # indirect dependency version override
- {:plug, "~> 1.10.4", override: true},
-
## dev & test
{:ex_doc, "~> 0.22", only: :dev, runtime: false},
{:ex_machina, "~> 2.4", only: :test},
diff --git a/mix.lock b/mix.lock
index fd8e02f95..65863fc40 100644
--- a/mix.lock
+++ b/mix.lock
@@ -94,7 +94,7 @@
"phoenix_swoosh": {:hex, :phoenix_swoosh, "1.2.0", "a544d83fde4a767efb78f45404a74c9e37b2a9c5ea3339692e65a6966731f935", [:mix], [{:finch, "~> 0.8", [hex: :finch, repo: "hexpm", optional: true]}, {:hackney, "~> 1.10", [hex: :hackney, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_view, "~> 1.0 or ~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: false]}, {:swoosh, "~> 1.5", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm", "e88d117251e89a16b92222415a6d87b99a96747ddf674fc5c7631de734811dba"},
"phoenix_template": {:hex, :phoenix_template, "1.0.1", "85f79e3ad1b0180abb43f9725973e3b8c2c3354a87245f91431eec60553ed3ef", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "157dc078f6226334c91cb32c1865bf3911686f8bcd6bcff86736f6253e6993ee"},
"phoenix_view": {:hex, :phoenix_view, "2.0.2", "6bd4d2fd595ef80d33b439ede6a19326b78f0f1d8d62b9a318e3d9c1af351098", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "a929e7230ea5c7ee0e149ffcf44ce7cf7f4b6d2bfe1752dd7c084cdff152d36f"},
- "plug": {:hex, :plug, "1.10.4", "41eba7d1a2d671faaf531fa867645bd5a3dce0957d8e2a3f398ccff7d2ef017f", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ad1e233fe73d2eec56616568d260777b67f53148a999dc2d048f4eb9778fe4a0"},
+ "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"},
"plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"},
"plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"},
"plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"},
From ffee478ed0298f2cf29d4d51ed28105119552496 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Wed, 31 May 2023 15:30:58 -0400
Subject: [PATCH 098/823] Move websocket config for Shoutbox to the Endpoint
This is the modern way of configuring it
---
config/config.exs | 14 --------------
lib/pleroma/web/endpoint.ex | 15 ++++++++++++++-
2 files changed, 14 insertions(+), 15 deletions(-)
diff --git a/config/config.exs b/config/config.exs
index 6c9eb9bfc..fb1903db9 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -110,17 +110,6 @@ config :pleroma, :uri_schemes,
"xmpp"
]
-websocket_config = [
- path: "/websocket",
- serializer: [
- {Phoenix.Socket.V1.JSONSerializer, "~> 1.0.0"},
- {Phoenix.Socket.V2.JSONSerializer, "~> 2.0.0"}
- ],
- timeout: 60_000,
- transport_log: false,
- compress: false
-]
-
# Configures the endpoint
config :pleroma, Pleroma.Web.Endpoint,
url: [host: "localhost"],
@@ -130,9 +119,6 @@ config :pleroma, Pleroma.Web.Endpoint,
{:_,
[
{"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []},
- {"/websocket", Phoenix.Endpoint.CowboyWebSocket,
- {Phoenix.Transports.WebSocket,
- {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}},
{:_, Plug.Cowboy.Handler, {Pleroma.Web.Endpoint, []}}
]}
]
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index d8d40cceb..9e0340cd2 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -9,7 +9,20 @@ defmodule Pleroma.Web.Endpoint do
alias Pleroma.Config
- socket("/socket", Pleroma.Web.UserSocket)
+ socket("/socket", Pleroma.Web.UserSocket,
+ websocket: [
+ path: "/websocket",
+ serializer: [
+ {Phoenix.Socket.V1.JSONSerializer, "~> 1.0.0"},
+ {Phoenix.Socket.V2.JSONSerializer, "~> 2.0.0"}
+ ],
+ timeout: 60_000,
+ transport_log: false,
+ compress: false
+ ],
+ longpoll: false
+ )
+
socket("/live", Phoenix.LiveView.Socket)
plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint])
From 62322f71e2f2e607ee66a91a99e35eecbf89914d Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Wed, 31 May 2023 16:22:40 -0400
Subject: [PATCH 099/823] Clean up Plug.Parsers.MULTIPART deprecation warnings
There is no need to the length setting to :multipart. The length setting is global for all of the parsers.
---
lib/pleroma/web/endpoint.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index 9e0340cd2..201a92653 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -116,7 +116,7 @@ defmodule Pleroma.Web.Endpoint do
plug(Plug.Parsers,
parsers: [
:urlencoded,
- {:multipart, length: {Config, :get, [[:instance, :upload_limit]]}},
+ :multipart,
:json
],
pass: ["*/*"],
From ba988a9abc6cb92d47b14f9ddf8ac78946215972 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Wed, 31 May 2023 16:30:31 -0400
Subject: [PATCH 100/823] Fix test warnings
warning: the URI path used in plug tests must start with "/"
---
.../controllers/status_controller_test.exs | 22 +++++++++----------
.../controllers/timeline_controller_test.exs | 12 +++++-----
2 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
index 1e8979127..563034355 100644
--- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
@@ -37,7 +37,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
response =
conn
|> put_req_header("content-type", "application/json")
- |> post("api/v1/statuses", %{
+ |> post("/api/v1/statuses", %{
"content_type" => "text/plain",
"source" => "Pleroma FE",
"status" => "Hello world",
@@ -50,7 +50,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
response =
conn
- |> get("api/v1/statuses/#{response["id"]}", %{})
+ |> get("/api/v1/statuses/#{response["id"]}", %{})
|> json_response_and_validate_schema(200)
assert response["reblogs_count"] == 0
@@ -109,7 +109,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
conn_four =
conn
|> put_req_header("content-type", "application/json")
- |> post("api/v1/statuses", %{
+ |> post("/api/v1/statuses", %{
"status" => "oolong",
"expires_in" => expires_in
})
@@ -134,7 +134,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
assert %{"error" => "Expiry date is too soon"} =
conn
|> put_req_header("content-type", "application/json")
- |> post("api/v1/statuses", %{
+ |> post("/api/v1/statuses", %{
"status" => "oolong",
"expires_in" => expires_in
})
@@ -146,7 +146,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
assert %{"error" => "Expiry date is too soon"} =
conn
|> put_req_header("content-type", "application/json")
- |> post("api/v1/statuses", %{
+ |> post("/api/v1/statuses", %{
"status" => "oolong",
"expires_in" => expires_in
})
@@ -160,7 +160,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
assert %{"error" => "[KeywordPolicy] Matches with rejected keyword"} =
conn
|> put_req_header("content-type", "application/json")
- |> post("api/v1/statuses", %{"status" => "GNO/Linux"})
+ |> post("/api/v1/statuses", %{"status" => "GNO/Linux"})
|> json_response_and_validate_schema(422)
end
@@ -353,7 +353,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
conn =
conn
|> put_req_header("content-type", "application/json")
- |> post("api/v1/statuses", %{"status" => content, "visibility" => "direct"})
+ |> post("/api/v1/statuses", %{"status" => content, "visibility" => "direct"})
assert %{"id" => id} = response = json_response_and_validate_schema(conn, 200)
assert response["visibility"] == "direct"
@@ -390,7 +390,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
result =
conn
- |> get("api/v1/statuses/#{activity}")
+ |> get("/api/v1/statuses/#{activity}")
assert %{
"content" => "cofe is my copilot",
@@ -419,7 +419,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
result =
conn
- |> get("api/v1/statuses/#{activity}")
+ |> get("/api/v1/statuses/#{activity}")
assert %{
"content" => "club mate is my wingman",
@@ -1334,7 +1334,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
assert %{"id" => id} =
conn
|> put_req_header("content-type", "application/json")
- |> post("api/v1/statuses", %{
+ |> post("/api/v1/statuses", %{
"status" => "oolong",
"expires_in" => expires_in
})
@@ -1584,7 +1584,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
conn
|> assign(:user, user3)
|> assign(:token, insert(:oauth_token, user: user3, scopes: ["read:statuses"]))
- |> get("api/v1/timelines/home")
+ |> get("/api/v1/timelines/home")
[reblogged_activity] = json_response_and_validate_schema(conn3, 200)
diff --git a/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs
index b13a8033b..c120dd53c 100644
--- a/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs
@@ -527,7 +527,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do
|> assign(:token, insert(:oauth_token, user: user_two, scopes: ["read:statuses"]))
# Only direct should be visible here
- res_conn = get(conn_user_two, "api/v1/timelines/direct")
+ res_conn = get(conn_user_two, "/api/v1/timelines/direct")
assert [status] = json_response_and_validate_schema(res_conn, :ok)
@@ -539,14 +539,14 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do
build_conn()
|> assign(:user, user_one)
|> assign(:token, insert(:oauth_token, user: user_one, scopes: ["read:statuses"]))
- |> get("api/v1/timelines/direct")
+ |> get("/api/v1/timelines/direct")
[status] = json_response_and_validate_schema(res_conn, :ok)
assert %{"visibility" => "direct"} = status
# Both should be visible here
- res_conn = get(conn_user_two, "api/v1/timelines/home")
+ res_conn = get(conn_user_two, "/api/v1/timelines/home")
[_s1, _s2] = json_response_and_validate_schema(res_conn, :ok)
@@ -559,14 +559,14 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do
})
end)
- res_conn = get(conn_user_two, "api/v1/timelines/direct")
+ res_conn = get(conn_user_two, "/api/v1/timelines/direct")
statuses = json_response_and_validate_schema(res_conn, :ok)
assert length(statuses) == 20
max_id = List.last(statuses)["id"]
- res_conn = get(conn_user_two, "api/v1/timelines/direct?max_id=#{max_id}")
+ res_conn = get(conn_user_two, "/api/v1/timelines/direct?max_id=#{max_id}")
assert [status] = json_response_and_validate_schema(res_conn, :ok)
@@ -591,7 +591,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do
visibility: "direct"
})
- res_conn = get(conn, "api/v1/timelines/direct")
+ res_conn = get(conn, "/api/v1/timelines/direct")
[status] = json_response_and_validate_schema(res_conn, :ok)
assert status["id"] == direct.id
From d9f031c9da42a1bde12a37bf26298c3a3c731121 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Wed, 31 May 2023 21:11:13 +0000
Subject: [PATCH 101/823] Bump minimum Elixir to 1.12
---
.gitlab-ci.yml | 13 +++++++------
Dockerfile | 2 +-
ci/Dockerfile | 2 +-
mix.exs | 2 +-
4 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8daa9f434..2cb39a5c9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,7 @@
image: git.pleroma.social:5050/pleroma/pleroma/ci-base
variables: &global_variables
+ ELIXIR_VER: 1.12.3
POSTGRES_DB: pleroma_test
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
@@ -276,7 +277,7 @@ stop_review_app:
amd64:
stage: release
- image: elixir:1.11.4
+ image: elixir:$ELIXIR_VER
only: &release-only
- stable@pleroma/pleroma
- develop@pleroma/pleroma
@@ -316,7 +317,7 @@ amd64-musl:
stage: release
artifacts: *release-artifacts
only: *release-only
- image: elixir:1.11.4-alpine
+ image: elixir:$ELIXIR_VER-alpine
tags:
- amd64
cache: *release-cache
@@ -334,7 +335,7 @@ arm:
only: *release-only
tags:
- arm32-specified
- image: arm32v7/elixir:1.11.4
+ image: arm32v7/elixir:$ELIXIR_VER
cache: *release-cache
variables: *release-variables
before_script: *before-release
@@ -346,7 +347,7 @@ arm-musl:
only: *release-only
tags:
- arm32-specified
- image: arm32v7/elixir:1.11.4-alpine
+ image: arm32v7/elixir:$ELIXIR_VER-alpine
cache: *release-cache
variables: *release-variables
before_script: *before-release-musl
@@ -358,7 +359,7 @@ arm64:
only: *release-only
tags:
- arm
- image: arm64v8/elixir:1.11.4
+ image: arm64v8/elixir:$ELIXIR_VER
cache: *release-cache
variables: *release-variables
before_script: *before-release
@@ -370,7 +371,7 @@ arm64-musl:
only: *release-only
tags:
- arm
- image: arm64v8/elixir:1.11.4-alpine
+ image: arm64v8/elixir:$ELIXIR_VER-alpine
cache: *release-cache
variables: *release-variables
before_script: *before-release-musl
diff --git a/Dockerfile b/Dockerfile
index 310d18104..56c99da72 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
ARG ELIXIR_IMG=hexpm/elixir
-ARG ELIXIR_VER=1.11.4
+ARG ELIXIR_VER=1.12.3
ARG ERLANG_VER=24.2.1
ARG ALPINE_VER=3.17.0
diff --git a/ci/Dockerfile b/ci/Dockerfile
index ca28b7029..a2b566873 100644
--- a/ci/Dockerfile
+++ b/ci/Dockerfile
@@ -1,4 +1,4 @@
-FROM elixir:1.11.4
+FROM elixir:1.12.3
# Single RUN statement, otherwise intermediate images are created
# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run
diff --git a/mix.exs b/mix.exs
index e346834e5..6e3d69379 100644
--- a/mix.exs
+++ b/mix.exs
@@ -5,7 +5,7 @@ defmodule Pleroma.Mixfile do
[
app: :pleroma,
version: version("2.5.52"),
- elixir: "~> 1.11",
+ elixir: "~> 1.12",
elixirc_paths: elixirc_paths(Mix.env()),
compilers: Mix.compilers(),
elixirc_options: [warnings_as_errors: warnings_as_errors()],
From 2b8bbb288c26a95970e2d9e988f5eb303e052644 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Wed, 31 May 2023 22:11:17 +0000
Subject: [PATCH 102/823] Phoenix.Socket.Transport.force_ssl/4 no longer exists
---
lib/phoenix/transports/web_socket/raw.ex | 1 -
1 file changed, 1 deletion(-)
diff --git a/lib/phoenix/transports/web_socket/raw.ex b/lib/phoenix/transports/web_socket/raw.ex
index 8cf9c32a2..cf4fda79f 100644
--- a/lib/phoenix/transports/web_socket/raw.ex
+++ b/lib/phoenix/transports/web_socket/raw.ex
@@ -26,7 +26,6 @@ defmodule Phoenix.Transports.WebSocket.Raw do
conn
|> fetch_query_params
|> Transport.transport_log(opts[:transport_log])
- |> Transport.force_ssl(handler, endpoint, opts)
|> Transport.check_origin(handler, endpoint, opts)
case conn do
From f0e5f0e8375bb55e13b0f054fcf09871d2967d24 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Wed, 31 May 2023 22:14:55 +0000
Subject: [PATCH 103/823] Fix compile warning
warning: doing a prefix match with globs is deprecated, invalid segment "pleroma*path"
---
lib/pleroma/web/router.ex | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index c1a690e28..6b9e158a3 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -996,8 +996,8 @@ defmodule Pleroma.Web.Router do
scope "/", Pleroma.Web.Fallback do
get("/registration/:token", RedirectController, :registration_page)
get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta)
- match(:*, "/api/pleroma*path", LegacyPleromaApiRerouterPlug, [])
- get("/api*path", RedirectController, :api_not_implemented)
+ match(:*, "/api/pleroma/*path", LegacyPleromaApiRerouterPlug, [])
+ get("/api/*path", RedirectController, :api_not_implemented)
get("/*path", RedirectController, :redirector_with_preload)
options("/*path", RedirectController, :empty)
From 2e45be2653f662f60e87fce56aa3c256e20bd1fb Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Fri, 2 Jun 2023 11:34:04 -0400
Subject: [PATCH 104/823] Add :phoenix to extra_applications to suppress a
warning
Related to use of Phoenix.Flash.get/2
---
mix.exs | 1 +
1 file changed, 1 insertion(+)
diff --git a/mix.exs b/mix.exs
index 6e3d69379..771b491b8 100644
--- a/mix.exs
+++ b/mix.exs
@@ -78,6 +78,7 @@ defmodule Pleroma.Mixfile do
:comeonin,
:fast_sanitize,
:os_mon,
+ :phoenix,
:ssl
],
included_applications: [:ex_syslogger]
From bcd7ccac11f3f7217ffd8ec32d32457436315f58 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Sat, 3 Jun 2023 14:03:51 -0400
Subject: [PATCH 105/823] Support a type called "change"
---
tools/check-changelog | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/check-changelog b/tools/check-changelog
index 60692033f..64e59ae7f 100644
--- a/tools/check-changelog
+++ b/tools/check-changelog
@@ -6,7 +6,7 @@ git remote add upstream https://git.pleroma.social/pleroma/pleroma.git
git fetch upstream ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}:refs/remotes/upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME
git diff --raw --no-renames upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD -- changelog.d | \
- grep ' A\t' | grep '\.\(skip\|add\|remove\|fix\|security\)$'
+ grep ' A\t' | grep '\.\(skip\|add\|remove\|fix\|security|change\)$'
ret=$?
if [ $ret -eq 0 ]; then
From c665d532951c34c7d1185e66b59390202b54d0c9 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Sat, 3 Jun 2023 14:04:12 -0400
Subject: [PATCH 106/823] Update to Phoenix 1.7
---
changelog.d/3900.change | 1 +
changelog.d/3900.skip | 0
2 files changed, 1 insertion(+)
create mode 100644 changelog.d/3900.change
delete mode 100644 changelog.d/3900.skip
diff --git a/changelog.d/3900.change b/changelog.d/3900.change
new file mode 100644
index 000000000..fe0cc2fbf
--- /dev/null
+++ b/changelog.d/3900.change
@@ -0,0 +1 @@
+Update to Phoenix 1.7
diff --git a/changelog.d/3900.skip b/changelog.d/3900.skip
deleted file mode 100644
index e69de29bb..000000000
From 63ef1dcedca7fb315f611388e477d3847d1acaa1 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Sat, 3 Jun 2023 14:17:49 -0400
Subject: [PATCH 107/823] Phoenix.Router.routes/1 is the public function we are
meant to be using here
---
lib/pleroma/web/router.ex | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 6b9e158a3..a6e180c4c 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -1003,9 +1003,8 @@ defmodule Pleroma.Web.Router do
options("/*path", RedirectController, :empty)
end
- # TODO: Change to Phoenix.Router.routes/1 for Phoenix 1.6.0+
def get_api_routes do
- __MODULE__.__routes__()
+ Phoenix.Router.routes(__MODULE__)
|> Enum.reject(fn r -> r.plug == Pleroma.Web.Fallback.RedirectController end)
|> Enum.map(fn r ->
r.path
From ea4225a646b355150fb8e5e8c77d7fdc58b5e7ef Mon Sep 17 00:00:00 2001
From: tusooa
Date: Tue, 18 Jul 2023 18:39:59 -0400
Subject: [PATCH 108/823] Restrict attachments to only uploaded files only
---
changelog.d/attachment-type-check.fix | 1 +
lib/pleroma/constants.ex | 2 ++
lib/pleroma/web/common_api/utils.ex | 7 ++++++-
test/pleroma/web/common_api/utils_test.exs | 11 ++++++++---
4 files changed, 17 insertions(+), 4 deletions(-)
create mode 100644 changelog.d/attachment-type-check.fix
diff --git a/changelog.d/attachment-type-check.fix b/changelog.d/attachment-type-check.fix
new file mode 100644
index 000000000..9e14b75f1
--- /dev/null
+++ b/changelog.d/attachment-type-check.fix
@@ -0,0 +1 @@
+Restrict attachments to only uploaded files only
diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex
index 7b4fd03b6..6befc6897 100644
--- a/lib/pleroma/constants.ex
+++ b/lib/pleroma/constants.ex
@@ -81,4 +81,6 @@ defmodule Pleroma.Constants do
const(mime_regex,
do: ~r/^[^[:cntrl:] ()<>@,;:\\"\/\[\]?=]+\/[^[:cntrl:] ()<>@,;:\\"\/\[\]?=]+(; .*)?$/
)
+
+ const(upload_object_types, do: ["Document", "Image"])
end
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index a93c97e1e..b9fe0224c 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -59,7 +59,12 @@ defmodule Pleroma.Web.CommonAPI.Utils do
end
defp get_attachment(media_id) do
- Repo.get(Object, media_id)
+ with %Object{data: data} = object <- Repo.get(Object, media_id),
+ %{"type" => type} when type in Pleroma.Constants.upload_object_types() <- data do
+ object
+ else
+ _ -> nil
+ end
end
@spec get_to_and_cc(ActivityDraft.t()) :: {list(String.t()), list(String.t())}
diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs
index d309c6ded..ca5b92683 100644
--- a/test/pleroma/web/common_api/utils_test.exs
+++ b/test/pleroma/web/common_api/utils_test.exs
@@ -592,7 +592,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
end
test "returns list attachments with desc" do
- object = insert(:note)
+ object = insert(:attachment)
desc = Jason.encode!(%{object.id => "test-desc"})
assert Utils.attachments_from_ids_descs(["#{object.id}", "34"], desc) == [
@@ -603,7 +603,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
describe "attachments_from_ids/1" do
test "returns attachments with descs" do
- object = insert(:note)
+ object = insert(:attachment)
desc = Jason.encode!(%{object.id => "test-desc"})
assert Utils.attachments_from_ids(%{
@@ -615,13 +615,18 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
end
test "returns attachments without descs" do
- object = insert(:note)
+ object = insert(:attachment)
assert Utils.attachments_from_ids(%{media_ids: ["#{object.id}"]}) == [object.data]
end
test "returns [] when not pass media_ids" do
assert Utils.attachments_from_ids(%{}) == []
end
+
+ test "checks that the object is of upload type" do
+ object = insert(:note)
+ assert Utils.attachments_from_ids(%{media_ids: ["#{object.id}"]}) == []
+ end
end
describe "maybe_add_list_data/3" do
From dc4de79d43ba941d8aa16f7c14887faae9f65e9f Mon Sep 17 00:00:00 2001
From: faried nawaz
Date: Wed, 7 Dec 2022 22:37:50 +0500
Subject: [PATCH 109/823] status context: perform visibility check on
activities around a status
issue #2927
---
lib/pleroma/web/activity_pub/activity_pub.ex | 23 ++
.../controllers/status_controller_test.exs | 267 ++++++++++++++++++
2 files changed, 290 insertions(+)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index c93288b79..f1a12d22d 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -455,6 +455,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|> maybe_preload_objects(opts)
|> maybe_preload_bookmarks(opts)
|> maybe_set_thread_muted_field(opts)
+ |> restrict_unauthenticated(opts[:user])
|> restrict_blocked(opts)
|> restrict_blockers_visibility(opts)
|> restrict_recipients(recipients, opts[:user])
@@ -1215,6 +1216,28 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp restrict_filtered(query, _), do: query
+ defp restrict_unauthenticated(query, nil) do
+ local = Config.restrict_unauthenticated_access?(:activities, :local)
+ remote = Config.restrict_unauthenticated_access?(:activities, :remote)
+
+ cond do
+ local and remote ->
+ # is there a better way to handle this?
+ from(activity in query, where: 1 == 0)
+
+ local ->
+ from(activity in query, where: activity.local == false)
+
+ remote ->
+ from(activity in query, where: activity.local == true)
+
+ true ->
+ query
+ end
+ end
+
+ defp restrict_unauthenticated(query, _), do: query
+
defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query
defp exclude_poll_votes(query, _) do
diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
index 4f434cb69..76c289ee7 100644
--- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
@@ -771,6 +771,49 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
{:ok, local: local, remote: remote}
end
+ defp local_and_remote_context_activities do
+ local_user_1 = insert(:user)
+ local_user_2 = insert(:user)
+ remote_user = insert(:user, local: false)
+
+ {:ok, %{id: id1, data: %{"context" => context}}} =
+ CommonAPI.post(local_user_1, %{status: "post"})
+
+ {:ok, %{id: id2} = post} =
+ CommonAPI.post(local_user_2, %{status: "local reply", in_reply_to_status_id: id1})
+
+ params = %{
+ "@context" => "https://www.w3.org/ns/activitystreams",
+ "actor" => remote_user.ap_id,
+ "type" => "Create",
+ "context" => context,
+ "id" => "#{remote_user.ap_id}/activities/1",
+ "inReplyTo" => post.data["id"],
+ "object" => %{
+ "type" => "Note",
+ "content" => "remote reply",
+ "context" => context,
+ "id" => "#{remote_user.ap_id}/objects/1",
+ "attributedTo" => remote_user.ap_id,
+ "to" => [
+ local_user_1.ap_id,
+ local_user_2.ap_id,
+ "https://www.w3.org/ns/activitystreams#Public"
+ ]
+ },
+ "to" => [
+ local_user_1.ap_id,
+ local_user_2.ap_id,
+ "https://www.w3.org/ns/activitystreams#Public"
+ ]
+ }
+
+ {:ok, job} = Pleroma.Web.Federator.incoming_ap_doc(params)
+ {:ok, remote_activity} = ObanHelpers.perform(job)
+
+ %{locals: [id1, id2], remote: remote_activity.id, context: context}
+ end
+
describe "status with restrict unauthenticated activities for local and remote" do
setup do: local_and_remote_activities()
@@ -957,6 +1000,230 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
end
end
+ describe "getting status contexts restricted unauthenticated for local and remote" do
+ setup do: local_and_remote_context_activities()
+
+ setup do: clear_config([:restrict_unauthenticated, :activities, :local], true)
+
+ setup do: clear_config([:restrict_unauthenticated, :activities, :remote], true)
+
+ test "if user is unauthenticated", %{conn: conn, locals: [post_id, _]} do
+ res_conn = get(conn, "/api/v1/statuses/#{post_id}/context")
+
+ assert json_response_and_validate_schema(res_conn, 200) == %{
+ "ancestors" => [],
+ "descendants" => []
+ }
+ end
+
+ test "if user is unauthenticated reply", %{conn: conn, locals: [_, reply_id]} do
+ res_conn = get(conn, "/api/v1/statuses/#{reply_id}/context")
+
+ assert json_response_and_validate_schema(res_conn, 200) == %{
+ "ancestors" => [],
+ "descendants" => []
+ }
+ end
+
+ test "if user is authenticated", %{locals: [post_id, reply_id], remote: remote_reply_id} do
+ %{conn: conn} = oauth_access(["read"])
+ res_conn = get(conn, "/api/v1/statuses/#{post_id}/context")
+
+ %{"ancestors" => [], "descendants" => descendants} =
+ json_response_and_validate_schema(res_conn, 200)
+
+ descendant_ids =
+ descendants
+ |> Enum.map(& &1["id"])
+
+ assert reply_id in descendant_ids
+ assert remote_reply_id in descendant_ids
+ end
+
+ test "if user is authenticated reply", %{locals: [post_id, reply_id], remote: remote_reply_id} do
+ %{conn: conn} = oauth_access(["read"])
+ res_conn = get(conn, "/api/v1/statuses/#{reply_id}/context")
+
+ %{"ancestors" => ancestors, "descendants" => descendants} =
+ json_response_and_validate_schema(res_conn, 200)
+
+ ancestor_ids =
+ ancestors
+ |> Enum.map(& &1["id"])
+
+ descendant_ids =
+ descendants
+ |> Enum.map(& &1["id"])
+
+ assert post_id in ancestor_ids
+ assert remote_reply_id in descendant_ids
+ end
+ end
+
+ describe "getting status contexts restricted unauthenticated for local" do
+ setup do: local_and_remote_context_activities()
+
+ setup do: clear_config([:restrict_unauthenticated, :activities, :local], true)
+
+ setup do: clear_config([:restrict_unauthenticated, :activities, :remote], false)
+
+ test "if user is unauthenticated", %{
+ conn: conn,
+ locals: [post_id, reply_id],
+ remote: remote_reply_id
+ } do
+ res_conn = get(conn, "/api/v1/statuses/#{post_id}/context")
+
+ %{"ancestors" => [], "descendants" => descendants} =
+ json_response_and_validate_schema(res_conn, 200)
+
+ descendant_ids =
+ descendants
+ |> Enum.map(& &1["id"])
+
+ assert reply_id not in descendant_ids
+ assert remote_reply_id in descendant_ids
+ end
+
+ test "if user is unauthenticated reply", %{
+ conn: conn,
+ locals: [post_id, reply_id],
+ remote: remote_reply_id
+ } do
+ res_conn = get(conn, "/api/v1/statuses/#{reply_id}/context")
+
+ %{"ancestors" => ancestors, "descendants" => descendants} =
+ json_response_and_validate_schema(res_conn, 200)
+
+ ancestor_ids =
+ ancestors
+ |> Enum.map(& &1["id"])
+
+ descendant_ids =
+ descendants
+ |> Enum.map(& &1["id"])
+
+ assert post_id not in ancestor_ids
+ assert remote_reply_id in descendant_ids
+ end
+
+ test "if user is authenticated", %{locals: [post_id, reply_id], remote: remote_reply_id} do
+ %{conn: conn} = oauth_access(["read"])
+ res_conn = get(conn, "/api/v1/statuses/#{post_id}/context")
+
+ %{"ancestors" => [], "descendants" => descendants} =
+ json_response_and_validate_schema(res_conn, 200)
+
+ descendant_ids =
+ descendants
+ |> Enum.map(& &1["id"])
+
+ assert reply_id in descendant_ids
+ assert remote_reply_id in descendant_ids
+ end
+
+ test "if user is authenticated reply", %{locals: [post_id, reply_id], remote: remote_reply_id} do
+ %{conn: conn} = oauth_access(["read"])
+ res_conn = get(conn, "/api/v1/statuses/#{reply_id}/context")
+
+ %{"ancestors" => ancestors, "descendants" => descendants} =
+ json_response_and_validate_schema(res_conn, 200)
+
+ ancestor_ids =
+ ancestors
+ |> Enum.map(& &1["id"])
+
+ descendant_ids =
+ descendants
+ |> Enum.map(& &1["id"])
+
+ assert post_id in ancestor_ids
+ assert remote_reply_id in descendant_ids
+ end
+ end
+
+ describe "getting status contexts restricted unauthenticated for remote" do
+ setup do: local_and_remote_context_activities()
+
+ setup do: clear_config([:restrict_unauthenticated, :activities, :local], false)
+
+ setup do: clear_config([:restrict_unauthenticated, :activities, :remote], true)
+
+ test "if user is unauthenticated", %{
+ conn: conn,
+ locals: [post_id, reply_id],
+ remote: remote_reply_id
+ } do
+ res_conn = get(conn, "/api/v1/statuses/#{post_id}/context")
+
+ %{"ancestors" => [], "descendants" => descendants} =
+ json_response_and_validate_schema(res_conn, 200)
+
+ descendant_ids =
+ descendants
+ |> Enum.map(& &1["id"])
+
+ assert reply_id in descendant_ids
+ assert remote_reply_id not in descendant_ids
+ end
+
+ test "if user is unauthenticated reply", %{
+ conn: conn,
+ locals: [post_id, reply_id],
+ remote: remote_reply_id
+ } do
+ res_conn = get(conn, "/api/v1/statuses/#{reply_id}/context")
+
+ %{"ancestors" => ancestors, "descendants" => descendants} =
+ json_response_and_validate_schema(res_conn, 200)
+
+ ancestor_ids =
+ ancestors
+ |> Enum.map(& &1["id"])
+
+ descendant_ids =
+ descendants
+ |> Enum.map(& &1["id"])
+
+ assert post_id in ancestor_ids
+ assert remote_reply_id not in descendant_ids
+ end
+
+ test "if user is authenticated", %{locals: [post_id, reply_id], remote: remote_reply_id} do
+ %{conn: conn} = oauth_access(["read"])
+ res_conn = get(conn, "/api/v1/statuses/#{post_id}/context")
+
+ %{"ancestors" => [], "descendants" => descendants} =
+ json_response_and_validate_schema(res_conn, 200)
+
+ reply_ids =
+ descendants
+ |> Enum.map(& &1["id"])
+
+ assert reply_id in reply_ids
+ assert remote_reply_id in reply_ids
+ end
+
+ test "if user is authenticated reply", %{locals: [post_id, reply_id], remote: remote_reply_id} do
+ %{conn: conn} = oauth_access(["read"])
+ res_conn = get(conn, "/api/v1/statuses/#{reply_id}/context")
+
+ %{"ancestors" => ancestors, "descendants" => descendants} =
+ json_response_and_validate_schema(res_conn, 200)
+
+ ancestor_ids =
+ ancestors
+ |> Enum.map(& &1["id"])
+
+ descendant_ids =
+ descendants
+ |> Enum.map(& &1["id"])
+
+ assert post_id in ancestor_ids
+ assert remote_reply_id in descendant_ids
+ end
+ end
+
describe "deleting a status" do
test "when you created it" do
%{user: author, conn: conn} = oauth_access(["write:statuses"])
From e5e76ec44559af62eb56043add0489c61e6c1ee5 Mon Sep 17 00:00:00 2001
From: Faried Nawaz
Date: Fri, 10 Feb 2023 01:32:32 +0500
Subject: [PATCH 110/823] cleaner ecto query to handle restrict_unauthenticated
for activities
This fix is for this case:
config :pleroma, :restrict_unauthenticated,
activities: %{local: true, remote: true}
---
lib/pleroma/web/activity_pub/activity_pub.ex | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index f1a12d22d..3979d418e 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1222,8 +1222,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
cond do
local and remote ->
- # is there a better way to handle this?
- from(activity in query, where: 1 == 0)
+ from(activity in query, where: false)
local ->
from(activity in query, where: activity.local == false)
From 11ce81d4af6b428fabb9d4c6f0098d786a21487b Mon Sep 17 00:00:00 2001
From: faried nawaz
Date: Fri, 28 Jul 2023 18:49:05 +0500
Subject: [PATCH 111/823] add changelog entry
---
changelog.d/3801.fix | 1 +
1 file changed, 1 insertion(+)
create mode 100644 changelog.d/3801.fix
diff --git a/changelog.d/3801.fix b/changelog.d/3801.fix
new file mode 100644
index 000000000..8c2ec0199
--- /dev/null
+++ b/changelog.d/3801.fix
@@ -0,0 +1 @@
+Filter context activities using Visibility.visible_for_user?
From 18a0c923d0da4c8fb6e33b383dabd1d06bb22968 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Thu, 3 Aug 2023 13:08:37 -0400
Subject: [PATCH 112/823] Resolve information disclosure vulnerability through
emoji pack archive download endpoint
The pack name has been sanitized so an attacker cannot upload a media
file called pack.json with their own handcrafted list of emoji files as
arbitrary files on the filesystem and then call the emoji pack archive
download endpoint with a pack name crafted to the location of the media
file they uploaded which tricks Pleroma into generating a zip file of
the target files the attacker wants to download.
The attack only works if the Pleroma instance does not have the
AnonymizeFilename upload filter enabled, which is currently the default.
Reported by: graf@poast.org
---
changelog.d/emoji-pack-sanitization.security | 1 +
lib/pleroma/emoji/pack.ex | 1 +
test/pleroma/emoji/pack_test.exs | 4 ++++
3 files changed, 6 insertions(+)
create mode 100644 changelog.d/emoji-pack-sanitization.security
diff --git a/changelog.d/emoji-pack-sanitization.security b/changelog.d/emoji-pack-sanitization.security
new file mode 100644
index 000000000..f3218abd4
--- /dev/null
+++ b/changelog.d/emoji-pack-sanitization.security
@@ -0,0 +1 @@
+Emoji pack loader sanitizes pack names
diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex
index a361ea200..6e58f8898 100644
--- a/lib/pleroma/emoji/pack.ex
+++ b/lib/pleroma/emoji/pack.ex
@@ -285,6 +285,7 @@ defmodule Pleroma.Emoji.Pack do
@spec load_pack(String.t()) :: {:ok, t()} | {:error, :file.posix()}
def load_pack(name) do
+ name = Path.basename(name)
pack_file = Path.join([emoji_path(), name, "pack.json"])
with {:ok, _} <- File.stat(pack_file),
diff --git a/test/pleroma/emoji/pack_test.exs b/test/pleroma/emoji/pack_test.exs
index 18b99da75..00001abfc 100644
--- a/test/pleroma/emoji/pack_test.exs
+++ b/test/pleroma/emoji/pack_test.exs
@@ -90,4 +90,8 @@ defmodule Pleroma.Emoji.PackTest do
assert updated_pack.files_count == 1
end
+
+ test "load_pack/1 ignores path traversal in a forged pack name", %{pack: pack} do
+ assert {:ok, ^pack} = Pack.load_pack("../../../../../dump_pack")
+ end
end
From 2c795094535537a8607cc0d3b7f076a609636f40 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Thu, 3 Aug 2023 13:08:37 -0400
Subject: [PATCH 113/823] Resolve information disclosure vulnerability through
emoji pack archive download endpoint
The pack name has been sanitized so an attacker cannot upload a media
file called pack.json with their own handcrafted list of emoji files as
arbitrary files on the filesystem and then call the emoji pack archive
download endpoint with a pack name crafted to the location of the media
file they uploaded which tricks Pleroma into generating a zip file of
the target files the attacker wants to download.
The attack only works if the Pleroma instance does not have the
AnonymizeFilename upload filter enabled, which is currently the default.
Reported by: graf@poast.org
---
changelog.d/emoji-pack-sanitization.security | 1 +
lib/pleroma/emoji/pack.ex | 1 +
test/pleroma/emoji/pack_test.exs | 4 ++++
3 files changed, 6 insertions(+)
create mode 100644 changelog.d/emoji-pack-sanitization.security
diff --git a/changelog.d/emoji-pack-sanitization.security b/changelog.d/emoji-pack-sanitization.security
new file mode 100644
index 000000000..f3218abd4
--- /dev/null
+++ b/changelog.d/emoji-pack-sanitization.security
@@ -0,0 +1 @@
+Emoji pack loader sanitizes pack names
diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex
index a361ea200..6e58f8898 100644
--- a/lib/pleroma/emoji/pack.ex
+++ b/lib/pleroma/emoji/pack.ex
@@ -285,6 +285,7 @@ defmodule Pleroma.Emoji.Pack do
@spec load_pack(String.t()) :: {:ok, t()} | {:error, :file.posix()}
def load_pack(name) do
+ name = Path.basename(name)
pack_file = Path.join([emoji_path(), name, "pack.json"])
with {:ok, _} <- File.stat(pack_file),
diff --git a/test/pleroma/emoji/pack_test.exs b/test/pleroma/emoji/pack_test.exs
index 18b99da75..00001abfc 100644
--- a/test/pleroma/emoji/pack_test.exs
+++ b/test/pleroma/emoji/pack_test.exs
@@ -90,4 +90,8 @@ defmodule Pleroma.Emoji.PackTest do
assert updated_pack.files_count == 1
end
+
+ test "load_pack/1 ignores path traversal in a forged pack name", %{pack: pack} do
+ assert {:ok, ^pack} = Pack.load_pack("../../../../../dump_pack")
+ end
end
From 4befb3b1d02f32eb2c56f12e4684a7bb3167b0ee Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Thu, 22 Jun 2023 00:46:52 +0200
Subject: [PATCH 114/823] Config: Restrict permissions of OTP config file
---
lib/pleroma/config/release_runtime_provider.ex | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/lib/pleroma/config/release_runtime_provider.ex b/lib/pleroma/config/release_runtime_provider.ex
index 91e5f1a54..9ec0f975e 100644
--- a/lib/pleroma/config/release_runtime_provider.ex
+++ b/lib/pleroma/config/release_runtime_provider.ex
@@ -20,6 +20,20 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do
with_runtime_config =
if File.exists?(config_path) do
+ #
+ %File.Stat{mode: mode} = File.lstat!(config_path)
+
+ if Bitwise.band(mode, 0o007) > 0 do
+ raise "Configuration at #{config_path} has world-permissions, execute the following: chmod o= #{config_path}"
+ end
+
+ if Bitwise.band(mode, 0o020) > 0 do
+ raise "Configuration at #{config_path} has group-wise write permissions, execute the following: chmod g-w #{config_path}"
+ end
+
+ # Note: Elixir doesn't provides a getuid(2)
+ # so cannot forbid group-read only when config is owned by us
+
runtime_config = Config.Reader.read!(config_path)
with_defaults
From bd7381f2f4139c26d9dbb8aad77ce77be7777532 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Thu, 22 Jun 2023 00:58:05 +0200
Subject: [PATCH 115/823] instance gen: Reduce permissions of pleroma
directories and config files
---
lib/mix/tasks/pleroma/instance.ex | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index 5c93f19ff..5d8b254a2 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -266,12 +266,20 @@ defmodule Mix.Tasks.Pleroma.Instance do
config_dir = Path.dirname(config_path)
psql_dir = Path.dirname(psql_path)
+ # Note: Distros requiring group read (0o750) on those directories should
+ # pre-create the directories.
[config_dir, psql_dir, static_dir, uploads_dir]
|> Enum.reject(&File.exists?/1)
- |> Enum.map(&File.mkdir_p!/1)
+ |> Enum.each(fn dir ->
+ File.mkdir_p!(dir)
+ File.chmod!(dir, 0o700)
+ end)
shell_info("Writing config to #{config_path}.")
+ # Sadly no fchmod(2) equivalent in Elixir…
+ File.touch!(config_path)
+ File.chmod!(config_path, 0o640)
File.write(config_path, result_config)
shell_info("Writing the postgres script to #{psql_path}.")
File.write(psql_path, result_psql)
@@ -290,8 +298,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
else
shell_error(
"The task would have overwritten the following files:\n" <>
- (Enum.map(will_overwrite, &"- #{&1}\n") |> Enum.join("")) <>
- "Rerun with `--force` to overwrite them."
+ Enum.map_join(will_overwrite, &"- #{&1}\n") <> "Rerun with `--force` to overwrite them."
)
end
end
From 22df32b3f5cfe9fe0a4a97ff799df72c091b676e Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Thu, 22 Jun 2023 01:00:25 +0200
Subject: [PATCH 116/823] changelog: Entry for config permissions restrictions
Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/3135
---
changelog.d/otp_perms.security | 1 +
1 file changed, 1 insertion(+)
create mode 100644 changelog.d/otp_perms.security
diff --git a/changelog.d/otp_perms.security b/changelog.d/otp_perms.security
new file mode 100644
index 000000000..a3da1c677
--- /dev/null
+++ b/changelog.d/otp_perms.security
@@ -0,0 +1 @@
+- Reduced permissions of config files and directories, distros requiring greater permissions like group-read need to pre-create the directories
\ No newline at end of file
From 76e408e42d1da123a955b85490f05f6d810172f9 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Fri, 4 Aug 2023 07:16:50 +0200
Subject: [PATCH 117/823] release_runtime_provider_test: chmod config for
hardened permissions
Git doesn't manages file permissions precisely enough for us.
---
test/pleroma/config/release_runtime_provider_test.exs | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/test/pleroma/config/release_runtime_provider_test.exs b/test/pleroma/config/release_runtime_provider_test.exs
index 4e0d4c838..8ff578e63 100644
--- a/test/pleroma/config/release_runtime_provider_test.exs
+++ b/test/pleroma/config/release_runtime_provider_test.exs
@@ -17,6 +17,8 @@ defmodule Pleroma.Config.ReleaseRuntimeProviderTest do
end
test "merged runtime config" do
+ assert :ok == File.chmod!("test/fixtures/config/temp.secret.exs", 0o640)
+
merged =
ReleaseRuntimeProvider.load([], config_path: "test/fixtures/config/temp.secret.exs")
@@ -25,6 +27,8 @@ defmodule Pleroma.Config.ReleaseRuntimeProviderTest do
end
test "merged exported config" do
+ assert :ok == File.chmod!("test/fixtures/config/temp.exported_from_db.secret.exs", 0o640)
+
ExUnit.CaptureIO.capture_io(fn ->
merged =
ReleaseRuntimeProvider.load([],
@@ -37,6 +41,9 @@ defmodule Pleroma.Config.ReleaseRuntimeProviderTest do
end
test "runtime config is merged with exported config" do
+ assert :ok == File.chmod!("test/fixtures/config/temp.secret.exs", 0o640)
+ assert :ok == File.chmod!("test/fixtures/config/temp.exported_from_db.secret.exs", 0o640)
+
merged =
ReleaseRuntimeProvider.load([],
config_path: "test/fixtures/config/temp.secret.exs",
From c37561214a803f9011d5ec6af8b8c07e547c19ff Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Fri, 4 Aug 2023 06:49:19 +0200
Subject: [PATCH 118/823] Force the use of amd64 runners for jobs using ci-base
---
.gitlab-ci.yml | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8b0381d11..91e568a32 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,7 +32,13 @@ before_script:
after_script:
- rm -rf _build/*/lib/pleroma
+.using-ci-base:
+ tags:
+ - amd64
+
build:
+ extends:
+ - .using-ci-base
stage: build
only:
changes: &build_changes_policy
@@ -44,6 +50,8 @@ build:
- mix compile --force
spec-build:
+ extends:
+ - .using-ci-base
stage: test
only:
changes:
@@ -57,6 +65,8 @@ spec-build:
- mix pleroma.openapi_spec spec.json
benchmark:
+ extends:
+ - .using-ci-base
stage: benchmark
when: manual
variables:
@@ -71,6 +81,8 @@ benchmark:
- mix pleroma.load_testing
unit-testing:
+ extends:
+ - .using-ci-base
stage: test
only:
changes: *build_changes_policy
@@ -94,6 +106,8 @@ unit-testing:
path: coverage.xml
unit-testing-erratic:
+ extends:
+ - .using-ci-base
stage: test
retry: 2
allow_failure: true
@@ -129,6 +143,8 @@ unit-testing-erratic:
# - mix test --trace --only federated
unit-testing-rum:
+ extends:
+ - .using-ci-base
stage: test
only:
changes: *build_changes_policy
@@ -162,6 +178,8 @@ lint:
- mix format --check-formatted
analysis:
+ extends:
+ - .using-ci-base
stage: test
only:
changes: *build_changes_policy
From 5ac2b7417d052a493b38ee05b393ae7c78e89484 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Fri, 4 Aug 2023 09:16:08 +0200
Subject: [PATCH 119/823] test: Fix warnings
---
.../activity_pub/transmogrifier/emoji_react_handling_test.exs | 2 +-
test/pleroma/web/mastodon_api/update_credentials_test.exs | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/test/pleroma/web/activity_pub/transmogrifier/emoji_react_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/emoji_react_handling_test.exs
index 9d99df27c..83bf59c6f 100644
--- a/test/pleroma/web/activity_pub/transmogrifier/emoji_react_handling_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier/emoji_react_handling_test.exs
@@ -65,7 +65,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.EmojiReactHandlingTest do
object = Object.get_by_ap_id(data["object"])
assert object.data["reaction_count"] == 1
- assert match?([[emoji, _]], object.data["reactions"])
+ assert match?([[^emoji, _]], object.data["reactions"])
end
test "it reject invalid emoji reactions" do
diff --git a/test/pleroma/web/mastodon_api/update_credentials_test.exs b/test/pleroma/web/mastodon_api/update_credentials_test.exs
index 57fa0f047..40f79d103 100644
--- a/test/pleroma/web/mastodon_api/update_credentials_test.exs
+++ b/test/pleroma/web/mastodon_api/update_credentials_test.exs
@@ -375,7 +375,7 @@ defmodule Pleroma.Web.MastodonAPI.UpdateCredentialsTest do
"pleroma_background_image" => new_background_oversized
})
- assert user_response = json_response_and_validate_schema(res, 413)
+ assert _user_response = json_response_and_validate_schema(res, 413)
assert user.background == %{}
clear_config([:instance, :upload_limit], upload_limit)
From 57f74537486cf7f721679f125741de9008478b00 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Fri, 4 Aug 2023 05:13:28 +0200
Subject: [PATCH 120/823] Release 2.5.3
---
CHANGELOG.md | 6 ++++++
mix.exs | 2 +-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f6fc6aaee..468ec1012 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Removed
+## 2.5.3
+
+### Security
+- Emoji pack loader sanitizes pack names
+- Reduced permissions of config files and directories, distros requiring greater permissions like group-read need to pre-create the directories
+
## 2.5.2
### Security
diff --git a/mix.exs b/mix.exs
index 79fd9c9ef..d1cdb151d 100644
--- a/mix.exs
+++ b/mix.exs
@@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do
def project do
[
app: :pleroma,
- version: version("2.5.2"),
+ version: version("2.5.3"),
elixir: "~> 1.11",
elixirc_paths: elixirc_paths(Mix.env()),
compilers: [:phoenix, :gettext] ++ Mix.compilers(),
From 8cc8100120abdbf26cfe4cdac2c0a012d7919e05 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Thu, 22 Jun 2023 00:46:52 +0200
Subject: [PATCH 121/823] Config: Restrict permissions of OTP config file
---
lib/pleroma/config/release_runtime_provider.ex | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/lib/pleroma/config/release_runtime_provider.ex b/lib/pleroma/config/release_runtime_provider.ex
index 91e5f1a54..9ec0f975e 100644
--- a/lib/pleroma/config/release_runtime_provider.ex
+++ b/lib/pleroma/config/release_runtime_provider.ex
@@ -20,6 +20,20 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do
with_runtime_config =
if File.exists?(config_path) do
+ #
+ %File.Stat{mode: mode} = File.lstat!(config_path)
+
+ if Bitwise.band(mode, 0o007) > 0 do
+ raise "Configuration at #{config_path} has world-permissions, execute the following: chmod o= #{config_path}"
+ end
+
+ if Bitwise.band(mode, 0o020) > 0 do
+ raise "Configuration at #{config_path} has group-wise write permissions, execute the following: chmod g-w #{config_path}"
+ end
+
+ # Note: Elixir doesn't provides a getuid(2)
+ # so cannot forbid group-read only when config is owned by us
+
runtime_config = Config.Reader.read!(config_path)
with_defaults
From 69caedc591bd61029f897f37ef7ecddd470cf935 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Thu, 22 Jun 2023 00:58:05 +0200
Subject: [PATCH 122/823] instance gen: Reduce permissions of pleroma
directories and config files
---
lib/mix/tasks/pleroma/instance.ex | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index 5c93f19ff..5d8b254a2 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -266,12 +266,20 @@ defmodule Mix.Tasks.Pleroma.Instance do
config_dir = Path.dirname(config_path)
psql_dir = Path.dirname(psql_path)
+ # Note: Distros requiring group read (0o750) on those directories should
+ # pre-create the directories.
[config_dir, psql_dir, static_dir, uploads_dir]
|> Enum.reject(&File.exists?/1)
- |> Enum.map(&File.mkdir_p!/1)
+ |> Enum.each(fn dir ->
+ File.mkdir_p!(dir)
+ File.chmod!(dir, 0o700)
+ end)
shell_info("Writing config to #{config_path}.")
+ # Sadly no fchmod(2) equivalent in Elixir…
+ File.touch!(config_path)
+ File.chmod!(config_path, 0o640)
File.write(config_path, result_config)
shell_info("Writing the postgres script to #{psql_path}.")
File.write(psql_path, result_psql)
@@ -290,8 +298,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
else
shell_error(
"The task would have overwritten the following files:\n" <>
- (Enum.map(will_overwrite, &"- #{&1}\n") |> Enum.join("")) <>
- "Rerun with `--force` to overwrite them."
+ Enum.map_join(will_overwrite, &"- #{&1}\n") <> "Rerun with `--force` to overwrite them."
)
end
end
From 9f0ad901ed4f8f0ad3e1d896fd41d25b93a97d76 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Thu, 22 Jun 2023 01:00:25 +0200
Subject: [PATCH 123/823] changelog: Entry for config permissions restrictions
Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/3135
---
changelog.d/otp_perms.security | 1 +
1 file changed, 1 insertion(+)
create mode 100644 changelog.d/otp_perms.security
diff --git a/changelog.d/otp_perms.security b/changelog.d/otp_perms.security
new file mode 100644
index 000000000..a3da1c677
--- /dev/null
+++ b/changelog.d/otp_perms.security
@@ -0,0 +1 @@
+- Reduced permissions of config files and directories, distros requiring greater permissions like group-read need to pre-create the directories
\ No newline at end of file
From 65ef8f19c5e21baadfa7daabdbd37fb9a1fbf863 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Fri, 4 Aug 2023 07:16:50 +0200
Subject: [PATCH 124/823] release_runtime_provider_test: chmod config for
hardened permissions
Git doesn't manages file permissions precisely enough for us.
---
test/pleroma/config/release_runtime_provider_test.exs | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/test/pleroma/config/release_runtime_provider_test.exs b/test/pleroma/config/release_runtime_provider_test.exs
index e6d10b13e..8d2a93d6c 100644
--- a/test/pleroma/config/release_runtime_provider_test.exs
+++ b/test/pleroma/config/release_runtime_provider_test.exs
@@ -17,6 +17,8 @@ defmodule Pleroma.Config.ReleaseRuntimeProviderTest do
end
test "merged runtime config" do
+ assert :ok == File.chmod!("test/fixtures/config/temp.secret.exs", 0o640)
+
merged =
ReleaseRuntimeProvider.load([], config_path: "test/fixtures/config/temp.secret.exs")
@@ -25,6 +27,8 @@ defmodule Pleroma.Config.ReleaseRuntimeProviderTest do
end
test "merged exported config" do
+ assert :ok == File.chmod!("test/fixtures/config/temp.exported_from_db.secret.exs", 0o640)
+
ExUnit.CaptureIO.capture_io(fn ->
merged =
ReleaseRuntimeProvider.load([],
@@ -37,6 +41,9 @@ defmodule Pleroma.Config.ReleaseRuntimeProviderTest do
end
test "runtime config is merged with exported config" do
+ assert :ok == File.chmod!("test/fixtures/config/temp.secret.exs", 0o640)
+ assert :ok == File.chmod!("test/fixtures/config/temp.exported_from_db.secret.exs", 0o640)
+
merged =
ReleaseRuntimeProvider.load([],
config_path: "test/fixtures/config/temp.secret.exs",
From 6a0fd77c48946b4b2100585b4f32d125680f5f82 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Fri, 4 Aug 2023 05:13:28 +0200
Subject: [PATCH 125/823] Release 2.5.53
---
CHANGELOG.md | 6 ++++++
mix.exs | 2 +-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 42a1bbb8f..87e9c5298 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Removed
- BREAKING: Support for passwords generated with `crypt(3)` (Gnu Social migration artifact)
+## 2.5.3
+
+### Security
+- Emoji pack loader sanitizes pack names
+- Reduced permissions of config files and directories, distros requiring greater permissions like group-read need to pre-create the directories
+
## 2.5.2
### Security
diff --git a/mix.exs b/mix.exs
index fc4e87ee3..115e7113f 100644
--- a/mix.exs
+++ b/mix.exs
@@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do
def project do
[
app: :pleroma,
- version: version("2.5.52"),
+ version: version("2.5.53"),
elixir: "~> 1.11",
elixirc_paths: elixirc_paths(Mix.env()),
compilers: [:phoenix] ++ Mix.compilers(),
From 0e321698d21766772aa2b54b518dcd76a6abce8c Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Fri, 4 Aug 2023 17:09:50 +0200
Subject: [PATCH 126/823] gentoo_otp_en.md: Indicate which install method it
covers
---
changelog.d/gentoo_otp_intro.skip | 0
docs/installation/gentoo_otp_en.md | 2 +-
2 files changed, 1 insertion(+), 1 deletion(-)
create mode 100644 changelog.d/gentoo_otp_intro.skip
diff --git a/changelog.d/gentoo_otp_intro.skip b/changelog.d/gentoo_otp_intro.skip
new file mode 100644
index 000000000..e69de29bb
diff --git a/docs/installation/gentoo_otp_en.md b/docs/installation/gentoo_otp_en.md
index 76968eb38..4fafc0c17 100644
--- a/docs/installation/gentoo_otp_en.md
+++ b/docs/installation/gentoo_otp_en.md
@@ -2,7 +2,7 @@
{! backend/installation/otp_vs_from_source.include !}
-A [manual installation guide for gentoo](./gentoo_en.md) is also available.
+This guide covers installation via Gentoo provided packaging. A [manual installation guide for gentoo](./gentoo_en.md) is also available.
## Installation
From ca0859b90f0f3cb9bb369d38d29868de59796c2c Mon Sep 17 00:00:00 2001
From: Mae
Date: Fri, 4 Aug 2023 22:24:17 +0100
Subject: [PATCH 127/823] Prevent XML parser from loading external entities
---
lib/pleroma/web/xml.ex | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/lib/pleroma/web/xml.ex b/lib/pleroma/web/xml.ex
index b699446b0..380a80ab8 100644
--- a/lib/pleroma/web/xml.ex
+++ b/lib/pleroma/web/xml.ex
@@ -29,7 +29,10 @@ defmodule Pleroma.Web.XML do
{doc, _rest} =
text
|> :binary.bin_to_list()
- |> :xmerl_scan.string(quiet: true)
+ |> :xmerl_scan.string(
+ quiet: true,
+ fetch_fun: fn _, _ -> raise "Resolving external entities not supported" end
+ )
{:ok, doc}
rescue
From 307692cee8cdd0dbe3e6cf40c1192fcf43910610 Mon Sep 17 00:00:00 2001
From: FloatingGhost
Date: Fri, 4 Aug 2023 22:24:32 +0100
Subject: [PATCH 128/823] Add unit test for external entity loading
---
test/fixtures/xml_external_entities.xml | 3 +++
test/pleroma/web/web_finger_test.exs | 23 +++++++++++++++++++++++
test/pleroma/web/xml_test.exs | 10 ++++++++++
3 files changed, 36 insertions(+)
create mode 100644 test/fixtures/xml_external_entities.xml
create mode 100644 test/pleroma/web/xml_test.exs
diff --git a/test/fixtures/xml_external_entities.xml b/test/fixtures/xml_external_entities.xml
new file mode 100644
index 000000000..d5ff87134
--- /dev/null
+++ b/test/fixtures/xml_external_entities.xml
@@ -0,0 +1,3 @@
+
+ ]>
+&xxe;
diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs
index fafef54fe..be5e08776 100644
--- a/test/pleroma/web/web_finger_test.exs
+++ b/test/pleroma/web/web_finger_test.exs
@@ -180,5 +180,28 @@ defmodule Pleroma.Web.WebFingerTest do
{:ok, _data} = WebFinger.finger("pekorino@pawoo.net")
end
+
+ test "refuses to process XML remote entities" do
+ Tesla.Mock.mock(fn
+ %{
+ url: "https://pawoo.net/.well-known/webfinger?resource=acct:pekorino@pawoo.net"
+ } ->
+ {:ok,
+ %Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/xml_external_entities.xml"),
+ headers: [{"content-type", "application/xrd+xml"}]
+ }}
+
+ %{url: "https://pawoo.net/.well-known/host-meta"} ->
+ {:ok,
+ %Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/tesla_mock/pawoo.net_host_meta")
+ }}
+ end)
+
+ assert :error = WebFinger.finger("pekorino@pawoo.net")
+ end
end
end
diff --git a/test/pleroma/web/xml_test.exs b/test/pleroma/web/xml_test.exs
new file mode 100644
index 000000000..89d4709b6
--- /dev/null
+++ b/test/pleroma/web/xml_test.exs
@@ -0,0 +1,10 @@
+defmodule Pleroma.Web.XMLTest do
+ use Pleroma.DataCase, async: true
+
+ alias Pleroma.Web.XML
+
+ test "refuses to load external entities from XML" do
+ data = File.read!("test/fixtures/xml_external_entities.xml")
+ assert(:error == XML.parse_document(data))
+ end
+end
From 6d48b0f1a93a5a44b95497063e885342240fbc27 Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Fri, 4 Aug 2023 22:44:09 -0400
Subject: [PATCH 129/823] Document and test that XXE processing is disabled
https://vuln.be/post/xxe-in-erlang-and-elixir/
---
changelog.d/akkoma-xml-remote-entities.security | 1 +
1 file changed, 1 insertion(+)
create mode 100644 changelog.d/akkoma-xml-remote-entities.security
diff --git a/changelog.d/akkoma-xml-remote-entities.security b/changelog.d/akkoma-xml-remote-entities.security
new file mode 100644
index 000000000..b3c86bee1
--- /dev/null
+++ b/changelog.d/akkoma-xml-remote-entities.security
@@ -0,0 +1 @@
+Restrict XML parser from processing external entitites (XXE)
From fc10e07ffbc9d81c7a2ac38a3f9175f2edf2bd1f Mon Sep 17 00:00:00 2001
From: Mae
Date: Fri, 4 Aug 2023 22:24:17 +0100
Subject: [PATCH 130/823] Prevent XML parser from loading external entities
---
lib/pleroma/web/xml.ex | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/lib/pleroma/web/xml.ex b/lib/pleroma/web/xml.ex
index b699446b0..380a80ab8 100644
--- a/lib/pleroma/web/xml.ex
+++ b/lib/pleroma/web/xml.ex
@@ -29,7 +29,10 @@ defmodule Pleroma.Web.XML do
{doc, _rest} =
text
|> :binary.bin_to_list()
- |> :xmerl_scan.string(quiet: true)
+ |> :xmerl_scan.string(
+ quiet: true,
+ fetch_fun: fn _, _ -> raise "Resolving external entities not supported" end
+ )
{:ok, doc}
rescue
From 77d57c974ad83fcea77e424d53dc16a27e5d88b6 Mon Sep 17 00:00:00 2001
From: FloatingGhost
Date: Fri, 4 Aug 2023 22:24:32 +0100
Subject: [PATCH 131/823] Add unit test for external entity loading
---
test/fixtures/xml_external_entities.xml | 3 +++
test/pleroma/web/web_finger_test.exs | 23 +++++++++++++++++++++++
test/pleroma/web/xml_test.exs | 10 ++++++++++
3 files changed, 36 insertions(+)
create mode 100644 test/fixtures/xml_external_entities.xml
create mode 100644 test/pleroma/web/xml_test.exs
diff --git a/test/fixtures/xml_external_entities.xml b/test/fixtures/xml_external_entities.xml
new file mode 100644
index 000000000..d5ff87134
--- /dev/null
+++ b/test/fixtures/xml_external_entities.xml
@@ -0,0 +1,3 @@
+
+ ]>
+&xxe;
diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs
index fafef54fe..be5e08776 100644
--- a/test/pleroma/web/web_finger_test.exs
+++ b/test/pleroma/web/web_finger_test.exs
@@ -180,5 +180,28 @@ defmodule Pleroma.Web.WebFingerTest do
{:ok, _data} = WebFinger.finger("pekorino@pawoo.net")
end
+
+ test "refuses to process XML remote entities" do
+ Tesla.Mock.mock(fn
+ %{
+ url: "https://pawoo.net/.well-known/webfinger?resource=acct:pekorino@pawoo.net"
+ } ->
+ {:ok,
+ %Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/xml_external_entities.xml"),
+ headers: [{"content-type", "application/xrd+xml"}]
+ }}
+
+ %{url: "https://pawoo.net/.well-known/host-meta"} ->
+ {:ok,
+ %Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/tesla_mock/pawoo.net_host_meta")
+ }}
+ end)
+
+ assert :error = WebFinger.finger("pekorino@pawoo.net")
+ end
end
end
diff --git a/test/pleroma/web/xml_test.exs b/test/pleroma/web/xml_test.exs
new file mode 100644
index 000000000..89d4709b6
--- /dev/null
+++ b/test/pleroma/web/xml_test.exs
@@ -0,0 +1,10 @@
+defmodule Pleroma.Web.XMLTest do
+ use Pleroma.DataCase, async: true
+
+ alias Pleroma.Web.XML
+
+ test "refuses to load external entities from XML" do
+ data = File.read!("test/fixtures/xml_external_entities.xml")
+ assert(:error == XML.parse_document(data))
+ end
+end
From cc848b78dca51fcd7e785eb92a7a3a4d5d1c419e Mon Sep 17 00:00:00 2001
From: Mark Felder
Date: Fri, 4 Aug 2023 22:44:09 -0400
Subject: [PATCH 132/823] Document and test that XXE processing is disabled
https://vuln.be/post/xxe-in-erlang-and-elixir/
---
changelog.d/akkoma-xml-remote-entities.security | 1 +
1 file changed, 1 insertion(+)
create mode 100644 changelog.d/akkoma-xml-remote-entities.security
diff --git a/changelog.d/akkoma-xml-remote-entities.security b/changelog.d/akkoma-xml-remote-entities.security
new file mode 100644
index 000000000..b3c86bee1
--- /dev/null
+++ b/changelog.d/akkoma-xml-remote-entities.security
@@ -0,0 +1 @@
+Restrict XML parser from processing external entitites (XXE)
From b631180b38ac63029f08bef137b13231bcf57b59 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Sat, 5 Aug 2023 08:27:42 +0200
Subject: [PATCH 133/823] Release 2.5.4
---
CHANGELOG.md | 5 +++++
changelog.d/akkoma-xml-remote-entities.security | 2 +-
mix.exs | 2 +-
3 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 468ec1012..9d9aadc6e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Removed
+## 2.5.54
+
+## Security
+- Fix XML External Entity (XXE) loading vulnerability allowing to fetch arbitary files from the server's filesystem
+
## 2.5.3
### Security
diff --git a/changelog.d/akkoma-xml-remote-entities.security b/changelog.d/akkoma-xml-remote-entities.security
index b3c86bee1..5e6725e5b 100644
--- a/changelog.d/akkoma-xml-remote-entities.security
+++ b/changelog.d/akkoma-xml-remote-entities.security
@@ -1 +1 @@
-Restrict XML parser from processing external entitites (XXE)
+Fix XML External Entity (XXE) loading vulnerability allowing to fetch arbitary files from the server's filesystem
diff --git a/mix.exs b/mix.exs
index d1cdb151d..12f721364 100644
--- a/mix.exs
+++ b/mix.exs
@@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do
def project do
[
app: :pleroma,
- version: version("2.5.3"),
+ version: version("2.5.4"),
elixir: "~> 1.11",
elixirc_paths: elixirc_paths(Mix.env()),
compilers: [:phoenix, :gettext] ++ Mix.compilers(),
From 4099ddb3dc5840fa10cff743d87464acf7898a80 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Sat, 5 Aug 2023 08:27:42 +0200
Subject: [PATCH 134/823] Mergeback release 2.5.4
---
CHANGELOG.md | 5 +++++
changelog.d/akkoma-xml-remote-entities.security | 2 +-
mix.exs | 2 +-
3 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 87e9c5298..65acfad3e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Removed
- BREAKING: Support for passwords generated with `crypt(3)` (Gnu Social migration artifact)
+## 2.5.4
+
+## Security
+- Fix XML External Entity (XXE) loading vulnerability allowing to fetch arbitary files from the server's filesystem
+
## 2.5.3
### Security
diff --git a/changelog.d/akkoma-xml-remote-entities.security b/changelog.d/akkoma-xml-remote-entities.security
index b3c86bee1..5e6725e5b 100644
--- a/changelog.d/akkoma-xml-remote-entities.security
+++ b/changelog.d/akkoma-xml-remote-entities.security
@@ -1 +1 @@
-Restrict XML parser from processing external entitites (XXE)
+Fix XML External Entity (XXE) loading vulnerability allowing to fetch arbitary files from the server's filesystem
diff --git a/mix.exs b/mix.exs
index 115e7113f..b071e7c7b 100644
--- a/mix.exs
+++ b/mix.exs
@@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do
def project do
[
app: :pleroma,
- version: version("2.5.53"),
+ version: version("2.5.54"),
elixir: "~> 1.11",
elixirc_paths: elixirc_paths(Mix.env()),
compilers: [:phoenix] ++ Mix.compilers(),
From 48b1e9bdc7382ec6ef33e95f2bd8674ae92f17b2 Mon Sep 17 00:00:00 2001
From: mae
Date: Sat, 5 Aug 2023 14:13:49 +0200
Subject: [PATCH 135/823] Completely disable xml entity resolution
---
.../disable-xml-entity-resolution.security | 1 +
lib/pleroma/web/xml.ex | 2 +-
test/fixtures/xml_billion_laughs.xml | 15 +++++++++++++++
test/pleroma/web/xml_test.exs | 5 +++++
4 files changed, 22 insertions(+), 1 deletion(-)
create mode 100644 changelog.d/disable-xml-entity-resolution.security
create mode 100644 test/fixtures/xml_billion_laughs.xml
diff --git a/changelog.d/disable-xml-entity-resolution.security b/changelog.d/disable-xml-entity-resolution.security
new file mode 100644
index 000000000..db8e12f67
--- /dev/null
+++ b/changelog.d/disable-xml-entity-resolution.security
@@ -0,0 +1 @@
+Disable XML entity resolution completely to fix a dos vulnerability
diff --git a/lib/pleroma/web/xml.ex b/lib/pleroma/web/xml.ex
index 380a80ab8..64329e4ba 100644
--- a/lib/pleroma/web/xml.ex
+++ b/lib/pleroma/web/xml.ex
@@ -31,7 +31,7 @@ defmodule Pleroma.Web.XML do
|> :binary.bin_to_list()
|> :xmerl_scan.string(
quiet: true,
- fetch_fun: fn _, _ -> raise "Resolving external entities not supported" end
+ allow_entities: false
)
{:ok, doc}
diff --git a/test/fixtures/xml_billion_laughs.xml b/test/fixtures/xml_billion_laughs.xml
new file mode 100644
index 000000000..75fb24cae
--- /dev/null
+++ b/test/fixtures/xml_billion_laughs.xml
@@ -0,0 +1,15 @@
+
+
+
+
+
+
+
+
+
+
+
+
+]>
+&lol9;
diff --git a/test/pleroma/web/xml_test.exs b/test/pleroma/web/xml_test.exs
index 89d4709b6..49306430b 100644
--- a/test/pleroma/web/xml_test.exs
+++ b/test/pleroma/web/xml_test.exs
@@ -3,6 +3,11 @@ defmodule Pleroma.Web.XMLTest do
alias Pleroma.Web.XML
+ test "refuses to parse any entities from XML" do
+ data = File.read!("test/fixtures/xml_billion_laughs.xml")
+ assert(:error == XML.parse_document(data))
+ end
+
test "refuses to load external entities from XML" do
data = File.read!("test/fixtures/xml_external_entities.xml")
assert(:error == XML.parse_document(data))
From c298e0165c29b30380466910bb328964a7264c4c Mon Sep 17 00:00:00 2001
From: Cat pony Black
Date: Sat, 5 Aug 2023 13:10:35 +0200
Subject: [PATCH 136/823] Fix config ownership in dockerfile to pass
restriction test
---
Dockerfile | 2 +-
changelog.d/dockerfile-config-perms.fix | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
create mode 100644 changelog.d/dockerfile-config-perms.fix
diff --git a/Dockerfile b/Dockerfile
index 310d18104..72c227b76 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -49,7 +49,7 @@ USER pleroma
COPY --from=build --chown=pleroma:0 /release ${HOME}
-COPY ./config/docker.exs /etc/pleroma/config.exs
+COPY --chown=pleroma --chmod=640 ./config/docker.exs /etc/pleroma/config.exs
COPY ./docker-entrypoint.sh ${HOME}
EXPOSE 4000
diff --git a/changelog.d/dockerfile-config-perms.fix b/changelog.d/dockerfile-config-perms.fix
new file mode 100644
index 000000000..49ea5becb
--- /dev/null
+++ b/changelog.d/dockerfile-config-perms.fix
@@ -0,0 +1 @@
+- Fix config ownership in dockerfile to pass restriction test
From 9effa24f308917f70276c41f91fb204b7684d942 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Thu, 10 Aug 2023 22:52:38 +0200
Subject: [PATCH 137/823] Implement api/v2/instance route
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
changelog.d/instance-v2.add | 1 +
.../api_spec/operations/instance_operation.ex | 172 ++++++++++++++++++
.../controllers/instance_controller.ex | 5 +
.../web/mastodon_api/views/instance_view.ex | 120 +++++++++---
lib/pleroma/web/router.ex | 3 +
lib/pleroma/web/web_finger.ex | 2 +-
.../controllers/instance_controller_test.exs | 7 +
7 files changed, 287 insertions(+), 23 deletions(-)
create mode 100644 changelog.d/instance-v2.add
diff --git a/changelog.d/instance-v2.add b/changelog.d/instance-v2.add
new file mode 100644
index 000000000..4dd7ce8c0
--- /dev/null
+++ b/changelog.d/instance-v2.add
@@ -0,0 +1 @@
+Implement /api/v2/instance route
\ No newline at end of file
diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex
index a07be7e40..8e395bde8 100644
--- a/lib/pleroma/web/api_spec/operations/instance_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex
@@ -23,6 +23,18 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
}
end
+ def show2_operation do
+ %Operation{
+ tags: ["Instance misc"],
+ summary: "Retrieve instance information",
+ description: "Information about the server",
+ operationId: "InstanceController.show2",
+ responses: %{
+ 200 => Operation.response("Instance", "application/json", instance2())
+ }
+ }
+ end
+
def peers_operation do
%Operation{
tags: ["Instance misc"],
@@ -165,6 +177,166 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
}
end
+ defp instance2 do
+ %Schema{
+ type: :object,
+ properties: %{
+ domain: %Schema{type: :string, description: "The domain name of the instance"},
+ title: %Schema{type: :string, description: "The title of the website"},
+ version: %Schema{
+ type: :string,
+ description: "The version of Pleroma installed on the instance"
+ },
+ source_url: %Schema{
+ type: :string,
+ description: "The version of Pleroma installed on the instance"
+ },
+ description: %Schema{
+ type: :string,
+ description: "Admin-defined description of the Pleroma site"
+ },
+ usage: %Schema{
+ type: :object,
+ description: "Instance usage statistics",
+ properties: %{
+ users: %Schema{
+ type: :object,
+ description: "User count statistics",
+ properties: %{
+ active_month: %Schema{
+ type: :integer,
+ description: "Monthly active users"
+ }
+ }
+ }
+ }
+ },
+ email: %Schema{
+ type: :string,
+ description: "An email that may be contacted for any inquiries",
+ format: :email
+ },
+ urls: %Schema{
+ type: :object,
+ description: "URLs of interest for clients apps",
+ properties: %{}
+ },
+ stats: %Schema{
+ type: :object,
+ description: "Statistics about how much information the instance contains",
+ properties: %{
+ user_count: %Schema{
+ type: :integer,
+ description: "Users registered on this instance"
+ },
+ status_count: %Schema{
+ type: :integer,
+ description: "Statuses authored by users on instance"
+ },
+ domain_count: %Schema{
+ type: :integer,
+ description: "Domains federated with this instance"
+ }
+ }
+ },
+ thumbnail: %Schema{
+ type: :object,
+ properties: %{
+ url: %Schema{
+ type: :string,
+ description: "Banner image for the website",
+ nullable: true
+ }
+ }
+ },
+ languages: %Schema{
+ type: :array,
+ items: %Schema{type: :string},
+ description: "Primary langauges of the website and its staff"
+ },
+ registrations: %Schema{
+ type: :object,
+ description: "Registrations-related configuration",
+ properties: %{
+ enabled: %Schema{
+ type: :boolean,
+ description: "Whether registrations are enabled"
+ },
+ approval_required: %Schema{
+ type: :boolean,
+ description: "Whether users need to be manually approved by admin"
+ }
+ }
+ },
+ configuration: %Schema{
+ type: :object,
+ description: "Instance configuration",
+ properties: %{
+ urls: %Schema{
+ type: :object,
+ properties: %{
+ streaming: %Schema{
+ type: :string,
+ description: "Websockets address for push streaming"
+ }
+ }
+ },
+ statuses: %Schema{
+ type: :object,
+ description: "A map with poll limits for local statuses",
+ properties: %{
+ max_characters: %Schema{
+ type: :integer,
+ description: "Posts character limit (CW/Subject included in the counter)"
+ },
+ max_media_attachments: %Schema{
+ type: :integer,
+ description: "Media attachment limit"
+ }
+ }
+ },
+ media_attachments: %Schema{
+ type: :object,
+ description: "A map with poll limits for media attachments",
+ properties: %{
+ image_size_limit: %Schema{
+ type: :integer,
+ description: "File size limit of uploaded images"
+ },
+ video_size_limit: %Schema{
+ type: :integer,
+ description: "File size limit of uploaded videos"
+ }
+ }
+ },
+ polls: %Schema{
+ type: :object,
+ description: "A map with poll limits for local polls",
+ properties: %{
+ max_options: %Schema{
+ type: :integer,
+ description: "Maximum number of options."
+ },
+ max_characters_per_option: %Schema{
+ type: :integer,
+ description: "Maximum number of characters per option."
+ },
+ min_expiration: %Schema{
+ type: :integer,
+ description: "Minimum expiration time (in seconds)."
+ },
+ max_expiration: %Schema{
+ type: :integer,
+ description: "Maximum expiration time (in seconds)."
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ end
+
defp array_of_domains do
%Schema{
type: :array,
diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex
index 6410e872c..3757a850a 100644
--- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex
@@ -16,6 +16,11 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do
render(conn, "show.json")
end
+ @doc "GET /api/v2/instance"
+ def show2(conn, _params) do
+ render(conn, "show2.json")
+ end
+
@doc "GET /api/v1/instance/peers"
def peers(conn, _params) do
json(conn, Pleroma.Stats.get_peers())
diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index efd2a0af6..c987cafd6 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -14,7 +14,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
instance = Config.get(:instance)
%{
- uri: Pleroma.Web.Endpoint.url(),
+ uri: Pleroma.Web.WebFinger.domain(),
title: Keyword.get(instance, :name),
description: Keyword.get(instance, :description),
short_description: Keyword.get(instance, :short_description),
@@ -30,6 +30,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
languages: Keyword.get(instance, :languages, ["en"]),
registrations: Keyword.get(instance, :registrations_open),
approval_required: Keyword.get(instance, :account_approval_required),
+ configuration: configuration(),
# Extra (not present in Mastodon):
max_toot_chars: Keyword.get(instance, :limit),
max_media_attachments: Keyword.get(instance, :max_media_attachments),
@@ -41,19 +42,38 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image),
shout_limit: Config.get([:shout, :limit]),
description_limit: Keyword.get(instance, :description_limit),
- pleroma: %{
- metadata: %{
- account_activation_required: Keyword.get(instance, :account_activation_required),
- features: features(),
- federation: federation(),
- fields_limits: fields_limits(),
- post_formats: Config.get([:instance, :allowed_post_formats]),
- birthday_required: Config.get([:instance, :birthday_required]),
- birthday_min_age: Config.get([:instance, :birthday_min_age])
- },
- stats: %{mau: Pleroma.User.active_user_count()},
- vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key)
- }
+ pleroma: pleroma_configuration(instance)
+ }
+ end
+
+ def render("show2.json", _) do
+ instance = Config.get(:instance)
+
+ %{
+ domain: Pleroma.Web.WebFinger.domain(),
+ title: Keyword.get(instance, :name),
+ version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})",
+ source_url: Pleroma.Application.repository(),
+ description: Keyword.get(instance, :short_description),
+ usage: %{users: %{active_month: Pleroma.User.active_user_count()}},
+ thumbnail: %{
+ url:
+ URI.merge(Pleroma.Web.Endpoint.url(), Keyword.get(instance, :instance_thumbnail))
+ |> to_string
+ },
+ languages: Keyword.get(instance, :languages, ["en"]),
+ configuration: configuration2(),
+ registrations: %{
+ enabled: Keyword.get(instance, :registrations_open),
+ approval_required: Keyword.get(instance, :account_approval_required),
+ message: nil
+ },
+ contact: %{
+ email: Keyword.get(instance, :email),
+ account: nil
+ },
+ # Extra (not present in Mastodon):
+ pleroma: pleroma_configuration2(instance)
}
end
@@ -68,6 +88,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
"shareable_emoji_packs",
"multifetch",
"pleroma:api/v1/notifications:include_types_filter",
+ "quote_posting",
"editing",
if Config.get([:activitypub, :blockers_visible]) do
"blockers_visible"
@@ -78,13 +99,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
if Config.get([:gopher, :enabled]) do
"gopher"
end,
- # backwards compat
- if Config.get([:shout, :enabled]) do
- "chat"
- end,
- if Config.get([:shout, :enabled]) do
- "shout"
- end,
if Config.get([:instance, :allow_relay]) do
"relay"
end,
@@ -94,6 +108,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
"pleroma_emoji_reactions",
"pleroma_custom_emoji_reactions",
"pleroma_chat_messages",
+ "email_list",
if Config.get([:instance, :show_reactions]) do
"exposable_reactions"
end,
@@ -132,7 +147,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
|> Map.put(:enabled, Config.get([:instance, :federating]))
end
- def fields_limits do
+ defp fields_limits do
%{
max_fields: Config.get([:instance, :max_account_fields]),
max_remote_fields: Config.get([:instance, :max_remote_account_fields]),
@@ -140,4 +155,65 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
value_length: Config.get([:instance, :account_field_value_length])
}
end
+
+ defp configuration do
+ %{
+ statuses: %{
+ max_characters: Config.get([:instance, :limit]),
+ max_media_attachments: Config.get([:instance, :max_media_attachments])
+ },
+ media_attachments: %{
+ image_size_limit: Config.get([:instance, :upload_limit]),
+ video_size_limit: Config.get([:instance, :upload_limit])
+ },
+ polls: %{
+ max_options: Config.get([:instance, :poll_limits, :max_options]),
+ max_characters_per_option: Config.get([:instance, :poll_limits, :max_option_chars]),
+ min_expiration: Config.get([:instance, :poll_limits, :min_expiration]),
+ max_expiration: Config.get([:instance, :poll_limits, :max_expiration])
+ }
+ }
+ end
+
+ defp configuration2 do
+ configuration()
+ |> Map.merge(%{
+ urls: %{streaming: Pleroma.Web.Endpoint.websocket_url()}
+ })
+ end
+
+ defp pleroma_configuration(instance) do
+ %{
+ metadata: %{
+ account_activation_required: Keyword.get(instance, :account_activation_required),
+ features: features(),
+ federation: federation(),
+ fields_limits: fields_limits(),
+ post_formats: Config.get([:instance, :allowed_post_formats]),
+ birthday_required: Config.get([:instance, :birthday_required]),
+ birthday_min_age: Config.get([:instance, :birthday_min_age])
+ },
+ stats: %{mau: Pleroma.User.active_user_count()},
+ vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key)
+ }
+ end
+
+ defp pleroma_configuration2(instance) do
+ configuration = pleroma_configuration(instance)
+
+ configuration
+ |> Map.merge(%{
+ metadata:
+ configuration.metadata
+ |> Map.merge(%{
+ avatar_upload_limit: Keyword.get(instance, :avatar_upload_limit),
+ background_upload_limit: Keyword.get(instance, :background_upload_limit),
+ banner_upload_limit: Keyword.get(instance, :banner_upload_limit),
+ background_image:
+ Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image),
+ description_limit: Keyword.get(instance, :description_limit),
+ shout_limit: Config.get([:shout, :limit])
+ })
+ })
+ end
end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 6b9e158a3..5362100cc 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -774,11 +774,14 @@ defmodule Pleroma.Web.Router do
scope "/api/v2", Pleroma.Web.MastodonAPI do
pipe_through(:api)
+
get("/search", SearchController, :search2)
post("/media", MediaController, :create2)
get("/suggestions", SuggestionController, :index2)
+
+ get("/instance", InstanceController, :show2)
end
scope "/api", Pleroma.Web do
diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex
index f95dc2458..b28fad8d1 100644
--- a/lib/pleroma/web/web_finger.ex
+++ b/lib/pleroma/web/web_finger.ex
@@ -96,7 +96,7 @@ defmodule Pleroma.Web.WebFinger do
|> XmlBuilder.to_doc()
end
- defp domain do
+ def domain do
Pleroma.Config.get([__MODULE__, :domain]) || Pleroma.Web.Endpoint.host()
end
diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs
index a556ef6a8..2243b0d4a 100644
--- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs
@@ -106,4 +106,11 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do
|> get("/api/v1/instance")
|> json_response_and_validate_schema(200)
end
+
+ test "get instance information v2", %{conn: conn} do
+ clear_config([:auth, :oauth_consumer_strategies], [])
+
+ assert get(conn, "/api/v2/instance")
+ |> json_response_and_validate_schema(200)
+ end
end
From 79e46ce73f782a83654986adc9fd0b256be6a2e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Fri, 11 Aug 2023 13:57:22 +0200
Subject: [PATCH 138/823] InstanceView: Add common_information function
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
.../web/mastodon_api/views/instance_view.ex | 22 +++++++++++--------
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index c987cafd6..e7a2feb7f 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -13,12 +13,11 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
def render("show.json", _) do
instance = Config.get(:instance)
- %{
+ common_information(instance)
+ |> Map.merge(%{
uri: Pleroma.Web.WebFinger.domain(),
- title: Keyword.get(instance, :name),
description: Keyword.get(instance, :description),
short_description: Keyword.get(instance, :short_description),
- version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})",
email: Keyword.get(instance, :email),
urls: %{
streaming_api: Pleroma.Web.Endpoint.websocket_url()
@@ -27,7 +26,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
thumbnail:
URI.merge(Pleroma.Web.Endpoint.url(), Keyword.get(instance, :instance_thumbnail))
|> to_string,
- languages: Keyword.get(instance, :languages, ["en"]),
registrations: Keyword.get(instance, :registrations_open),
approval_required: Keyword.get(instance, :account_approval_required),
configuration: configuration(),
@@ -43,16 +41,15 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
shout_limit: Config.get([:shout, :limit]),
description_limit: Keyword.get(instance, :description_limit),
pleroma: pleroma_configuration(instance)
- }
+ })
end
def render("show2.json", _) do
instance = Config.get(:instance)
- %{
+ common_information(instance)
+ |> Map.merge(%{
domain: Pleroma.Web.WebFinger.domain(),
- title: Keyword.get(instance, :name),
- version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})",
source_url: Pleroma.Application.repository(),
description: Keyword.get(instance, :short_description),
usage: %{users: %{active_month: Pleroma.User.active_user_count()}},
@@ -61,7 +58,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
URI.merge(Pleroma.Web.Endpoint.url(), Keyword.get(instance, :instance_thumbnail))
|> to_string
},
- languages: Keyword.get(instance, :languages, ["en"]),
configuration: configuration2(),
registrations: %{
enabled: Keyword.get(instance, :registrations_open),
@@ -74,6 +70,14 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
},
# Extra (not present in Mastodon):
pleroma: pleroma_configuration2(instance)
+ })
+ end
+
+ defp common_information(instance) do
+ %{
+ title: Keyword.get(instance, :name),
+ version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})",
+ languages: Keyword.get(instance, :languages, ["en"])
}
end
From d838d1990bf23d452c1cc830629e42e51dbd7047 Mon Sep 17 00:00:00 2001
From: Haelwenn
Date: Wed, 16 Aug 2023 13:34:32 +0000
Subject: [PATCH 139/823] Apply lanodan's suggestion(s) to 1 file(s)
---
lib/pleroma/web/plugs/http_security_plug.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex
index b3dc8a3a6..a3166bc96 100644
--- a/lib/pleroma/web/plugs/http_security_plug.ex
+++ b/lib/pleroma/web/plugs/http_security_plug.ex
@@ -93,7 +93,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
img_src = "img-src 'self' data: blob:"
media_src = "media-src 'self'"
- connect_src = ["connect-src 'self' blob: ", ?\s, websocket_url]
+ connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url]
# Strict multimedia CSP enforcement only when MediaProxy is enabled
{img_src, media_src, connect_src} =
From 3d09bc320efcc68beb9b57fba23b6b9f3dc17905 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Wed, 30 Aug 2023 20:34:16 -0400
Subject: [PATCH 140/823] Make lint happy
---
lib/pleroma/web/plugs/http_security_plug.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex
index a3166bc96..5093414c4 100644
--- a/lib/pleroma/web/plugs/http_security_plug.ex
+++ b/lib/pleroma/web/plugs/http_security_plug.ex
@@ -100,6 +100,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
if Config.get([:media_proxy, :enabled]) &&
!Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
sources = build_csp_multimedia_source_list()
+
{
[img_src, sources],
[media_src, sources],
@@ -113,7 +114,6 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
}
end
-
connect_src =
if Config.get(:env) == :dev do
[connect_src, " http://localhost:3035/"]
From 3c5ecca37718a1eba05be1f379b8f47362079c65 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Wed, 30 Aug 2023 20:37:45 -0400
Subject: [PATCH 141/823] Skip changelog
---
changelog.d/lint.skip | 0
1 file changed, 0 insertions(+), 0 deletions(-)
create mode 100644 changelog.d/lint.skip
diff --git a/changelog.d/lint.skip b/changelog.d/lint.skip
new file mode 100644
index 000000000..e69de29bb
From 1afde067b12ad0062c1820091ea9b0a680819281 Mon Sep 17 00:00:00 2001
From: Mint
Date: Sat, 2 Sep 2023 01:43:25 +0300
Subject: [PATCH 142/823] CommonAPI: Prevent users from accessing media of
other users
---
.../check-attachment-attribution.security | 1 +
lib/pleroma/scheduled_activity.ex | 6 ++-
lib/pleroma/web/common_api.ex | 12 ++++++
lib/pleroma/web/common_api/activity_draft.ex | 2 +-
lib/pleroma/web/common_api/utils.ex | 27 ++++++------
test/pleroma/web/common_api/utils_test.exs | 43 +++++++++++++------
test/pleroma/web/common_api_test.exs | 18 ++++++++
.../views/scheduled_activity_view_test.exs | 2 +-
.../chat_message_reference_view_test.exs | 2 +-
9 files changed, 82 insertions(+), 31 deletions(-)
create mode 100644 changelog.d/check-attachment-attribution.security
diff --git a/changelog.d/check-attachment-attribution.security b/changelog.d/check-attachment-attribution.security
new file mode 100644
index 000000000..e0e46525b
--- /dev/null
+++ b/changelog.d/check-attachment-attribution.security
@@ -0,0 +1 @@
+CommonAPI: Prevent users from accessing media of other users by creating a status with reused attachment ID
diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex
index a7be58512..0ed51ad07 100644
--- a/lib/pleroma/scheduled_activity.ex
+++ b/lib/pleroma/scheduled_activity.ex
@@ -40,7 +40,11 @@ defmodule Pleroma.ScheduledActivity do
%{changes: %{params: %{"media_ids" => media_ids} = params}} = changeset
)
when is_list(media_ids) do
- media_attachments = Utils.attachments_from_ids(%{media_ids: media_ids})
+ media_attachments =
+ Utils.attachments_from_ids(
+ %{media_ids: media_ids},
+ User.get_cached_by_id(changeset.data.user_id)
+ )
params =
params
diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex
index 77b3fa5d2..82c7f70d2 100644
--- a/lib/pleroma/web/common_api.ex
+++ b/lib/pleroma/web/common_api.ex
@@ -33,6 +33,7 @@ defmodule Pleroma.Web.CommonAPI do
def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ []) do
with maybe_attachment <- opts[:media_id] && Object.get_by_id(opts[:media_id]),
+ :ok <- validate_chat_attachment_attribution(maybe_attachment, user),
:ok <- validate_chat_content_length(content, !!maybe_attachment),
{_, {:ok, chat_message_data, _meta}} <-
{:build_object,
@@ -71,6 +72,17 @@ defmodule Pleroma.Web.CommonAPI do
text
end
+ defp validate_chat_attachment_attribution(nil, _), do: :ok
+
+ defp validate_chat_attachment_attribution(attachment, user) do
+ with :ok <- Object.authorize_access(attachment, user) do
+ :ok
+ else
+ e ->
+ e
+ end
+ end
+
defp validate_chat_content_length(_, true), do: :ok
defp validate_chat_content_length(nil, false), do: {:error, :no_content}
diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex
index 9af635da8..63ed48a27 100644
--- a/lib/pleroma/web/common_api/activity_draft.ex
+++ b/lib/pleroma/web/common_api/activity_draft.ex
@@ -111,7 +111,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
end
defp attachments(%{params: params} = draft) do
- attachments = Utils.attachments_from_ids(params)
+ attachments = Utils.attachments_from_ids(params, draft.user)
draft = %__MODULE__{draft | attachments: attachments}
case Utils.validate_attachments_count(attachments) do
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index b9fe0224c..0f394e951 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -23,21 +23,21 @@ defmodule Pleroma.Web.CommonAPI.Utils do
require Logger
require Pleroma.Constants
- def attachments_from_ids(%{media_ids: ids, descriptions: desc}) do
- attachments_from_ids_descs(ids, desc)
+ def attachments_from_ids(%{media_ids: ids, descriptions: desc}, user) do
+ attachments_from_ids_descs(ids, desc, user)
end
- def attachments_from_ids(%{media_ids: ids}) do
- attachments_from_ids_no_descs(ids)
+ def attachments_from_ids(%{media_ids: ids}, user) do
+ attachments_from_ids_no_descs(ids, user)
end
- def attachments_from_ids(_), do: []
+ def attachments_from_ids(_, _), do: []
- def attachments_from_ids_no_descs([]), do: []
+ def attachments_from_ids_no_descs([], _), do: []
- def attachments_from_ids_no_descs(ids) do
+ def attachments_from_ids_no_descs(ids, user) do
Enum.map(ids, fn media_id ->
- case get_attachment(media_id) do
+ case get_attachment(media_id, user) do
%Object{data: data} -> data
_ -> nil
end
@@ -45,22 +45,23 @@ defmodule Pleroma.Web.CommonAPI.Utils do
|> Enum.reject(&is_nil/1)
end
- def attachments_from_ids_descs([], _), do: []
+ def attachments_from_ids_descs([], _, _), do: []
- def attachments_from_ids_descs(ids, descs_str) do
+ def attachments_from_ids_descs(ids, descs_str, user) do
{_, descs} = Jason.decode(descs_str)
Enum.map(ids, fn media_id ->
- with %Object{data: data} <- get_attachment(media_id) do
+ with %Object{data: data} <- get_attachment(media_id, user) do
Map.put(data, "name", descs[media_id])
end
end)
|> Enum.reject(&is_nil/1)
end
- defp get_attachment(media_id) do
+ defp get_attachment(media_id, user) do
with %Object{data: data} = object <- Repo.get(Object, media_id),
- %{"type" => type} when type in Pleroma.Constants.upload_object_types() <- data do
+ %{"type" => type} when type in Pleroma.Constants.upload_object_types() <- data,
+ :ok <- Object.authorize_access(object, user) do
object
else
_ -> nil
diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs
index ca5b92683..4ce039d64 100644
--- a/test/pleroma/web/common_api/utils_test.exs
+++ b/test/pleroma/web/common_api/utils_test.exs
@@ -586,46 +586,61 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
end
end
- describe "attachments_from_ids_descs/2" do
+ describe "attachments_from_ids_descs/3" do
test "returns [] when attachment ids is empty" do
- assert Utils.attachments_from_ids_descs([], "{}") == []
+ assert Utils.attachments_from_ids_descs([], "{}", nil) == []
end
test "returns list attachments with desc" do
- object = insert(:attachment)
+ user = insert(:user)
+ object = insert(:attachment, %{user: user})
desc = Jason.encode!(%{object.id => "test-desc"})
- assert Utils.attachments_from_ids_descs(["#{object.id}", "34"], desc) == [
+ assert Utils.attachments_from_ids_descs(["#{object.id}", "34"], desc, user) == [
Map.merge(object.data, %{"name" => "test-desc"})
]
end
end
- describe "attachments_from_ids/1" do
+ describe "attachments_from_ids/2" do
test "returns attachments with descs" do
- object = insert(:attachment)
+ user = insert(:user)
+ object = insert(:attachment, %{user: user})
desc = Jason.encode!(%{object.id => "test-desc"})
- assert Utils.attachments_from_ids(%{
- media_ids: ["#{object.id}"],
- descriptions: desc
- }) == [
+ assert Utils.attachments_from_ids(
+ %{
+ media_ids: ["#{object.id}"],
+ descriptions: desc
+ },
+ user
+ ) == [
Map.merge(object.data, %{"name" => "test-desc"})
]
end
test "returns attachments without descs" do
- object = insert(:attachment)
- assert Utils.attachments_from_ids(%{media_ids: ["#{object.id}"]}) == [object.data]
+ user = insert(:user)
+ object = insert(:attachment, %{user: user})
+ assert Utils.attachments_from_ids(%{media_ids: ["#{object.id}"]}, user) == [object.data]
end
test "returns [] when not pass media_ids" do
- assert Utils.attachments_from_ids(%{}) == []
+ assert Utils.attachments_from_ids(%{}, nil) == []
+ end
+
+ test "returns [] when media_ids not belong to current user" do
+ user = insert(:user)
+ user2 = insert(:user)
+
+ object = insert(:attachment, %{user: user})
+
+ assert Utils.attachments_from_ids(%{media_ids: ["#{object.id}"]}, user2) == []
end
test "checks that the object is of upload type" do
object = insert(:note)
- assert Utils.attachments_from_ids(%{media_ids: ["#{object.id}"]}) == []
+ assert Utils.attachments_from_ids(%{media_ids: ["#{object.id}"]}, nil) == []
end
end
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index 968e11a14..0d76d6581 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -279,6 +279,24 @@ defmodule Pleroma.Web.CommonAPITest do
assert {:reject, "[KeywordPolicy] Matches with rejected keyword"} ==
CommonAPI.post_chat_message(author, recipient, "GNO/Linux")
end
+
+ test "it reject messages with attachments not belonging to user" do
+ author = insert(:user)
+ not_author = insert(:user)
+ recipient = author
+
+ attachment = insert(:attachment, %{user: not_author})
+
+ {:error, message} =
+ CommonAPI.post_chat_message(
+ author,
+ recipient,
+ "123",
+ media_id: attachment.id
+ )
+
+ assert message == :forbidden
+ end
end
describe "unblocking" do
diff --git a/test/pleroma/web/mastodon_api/views/scheduled_activity_view_test.exs b/test/pleroma/web/mastodon_api/views/scheduled_activity_view_test.exs
index e5e510d33..07a65a3bc 100644
--- a/test/pleroma/web/mastodon_api/views/scheduled_activity_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/scheduled_activity_view_test.exs
@@ -48,7 +48,7 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityViewTest do
id: to_string(scheduled_activity.id),
media_attachments:
%{media_ids: [upload.id]}
- |> Utils.attachments_from_ids()
+ |> Utils.attachments_from_ids(user)
|> Enum.map(&StatusView.render("attachment.json", %{attachment: &1})),
params: %{
in_reply_to_id: to_string(activity.id),
diff --git a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
index 017c9c5c0..7ab3f5acd 100644
--- a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
+++ b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
@@ -24,7 +24,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceViewTest do
filename: "an_image.jpg"
}
- {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id)
+ {:ok, upload} = ActivityPub.upload(file, actor: recipient.ap_id)
{:ok, activity} =
CommonAPI.post_chat_message(user, recipient, "kippis :firefox:", idempotency_key: "123")
From 535a5ecad04c9c49105a77e7025fe9f4b4d23ba6 Mon Sep 17 00:00:00 2001
From: Mint
Date: Sat, 2 Sep 2023 01:43:25 +0300
Subject: [PATCH 143/823] CommonAPI: Prevent users from accessing media of
other users
commit 1afde067b12ad0062c1820091ea9b0a680819281 upstream.
---
.../check-attachment-attribution.security | 1 +
lib/pleroma/scheduled_activity.ex | 6 ++-
lib/pleroma/web/common_api.ex | 12 ++++++
lib/pleroma/web/common_api/activity_draft.ex | 2 +-
lib/pleroma/web/common_api/utils.ex | 31 ++++++++------
test/pleroma/web/common_api/utils_test.exs | 41 +++++++++++++------
test/pleroma/web/common_api_test.exs | 18 ++++++++
.../views/scheduled_activity_view_test.exs | 2 +-
.../chat_message_reference_view_test.exs | 2 +-
9 files changed, 85 insertions(+), 30 deletions(-)
create mode 100644 changelog.d/check-attachment-attribution.security
diff --git a/changelog.d/check-attachment-attribution.security b/changelog.d/check-attachment-attribution.security
new file mode 100644
index 000000000..e0e46525b
--- /dev/null
+++ b/changelog.d/check-attachment-attribution.security
@@ -0,0 +1 @@
+CommonAPI: Prevent users from accessing media of other users by creating a status with reused attachment ID
diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex
index a7be58512..0ed51ad07 100644
--- a/lib/pleroma/scheduled_activity.ex
+++ b/lib/pleroma/scheduled_activity.ex
@@ -40,7 +40,11 @@ defmodule Pleroma.ScheduledActivity do
%{changes: %{params: %{"media_ids" => media_ids} = params}} = changeset
)
when is_list(media_ids) do
- media_attachments = Utils.attachments_from_ids(%{media_ids: media_ids})
+ media_attachments =
+ Utils.attachments_from_ids(
+ %{media_ids: media_ids},
+ User.get_cached_by_id(changeset.data.user_id)
+ )
params =
params
diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex
index 89cc0d6fe..44eb00075 100644
--- a/lib/pleroma/web/common_api.ex
+++ b/lib/pleroma/web/common_api.ex
@@ -33,6 +33,7 @@ defmodule Pleroma.Web.CommonAPI do
def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ []) do
with maybe_attachment <- opts[:media_id] && Object.get_by_id(opts[:media_id]),
+ :ok <- validate_chat_attachment_attribution(maybe_attachment, user),
:ok <- validate_chat_content_length(content, !!maybe_attachment),
{_, {:ok, chat_message_data, _meta}} <-
{:build_object,
@@ -71,6 +72,17 @@ defmodule Pleroma.Web.CommonAPI do
text
end
+ defp validate_chat_attachment_attribution(nil, _), do: :ok
+
+ defp validate_chat_attachment_attribution(attachment, user) do
+ with :ok <- Object.authorize_access(attachment, user) do
+ :ok
+ else
+ e ->
+ e
+ end
+ end
+
defp validate_chat_content_length(_, true), do: :ok
defp validate_chat_content_length(nil, false), do: {:error, :no_content}
diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex
index 9af635da8..63ed48a27 100644
--- a/lib/pleroma/web/common_api/activity_draft.ex
+++ b/lib/pleroma/web/common_api/activity_draft.ex
@@ -111,7 +111,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
end
defp attachments(%{params: params} = draft) do
- attachments = Utils.attachments_from_ids(params)
+ attachments = Utils.attachments_from_ids(params, draft.user)
draft = %__MODULE__{draft | attachments: attachments}
case Utils.validate_attachments_count(attachments) do
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index ff0814329..6410815ea 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -23,21 +23,21 @@ defmodule Pleroma.Web.CommonAPI.Utils do
require Logger
require Pleroma.Constants
- def attachments_from_ids(%{media_ids: ids, descriptions: desc}) do
- attachments_from_ids_descs(ids, desc)
+ def attachments_from_ids(%{media_ids: ids, descriptions: desc}, user) do
+ attachments_from_ids_descs(ids, desc, user)
end
- def attachments_from_ids(%{media_ids: ids}) do
- attachments_from_ids_no_descs(ids)
+ def attachments_from_ids(%{media_ids: ids}, user) do
+ attachments_from_ids_no_descs(ids, user)
end
- def attachments_from_ids(_), do: []
+ def attachments_from_ids(_, _), do: []
- def attachments_from_ids_no_descs([]), do: []
+ def attachments_from_ids_no_descs([], _), do: []
- def attachments_from_ids_no_descs(ids) do
+ def attachments_from_ids_no_descs(ids, user) do
Enum.map(ids, fn media_id ->
- case get_attachment(media_id) do
+ case get_attachment(media_id, user) do
%Object{data: data} -> data
_ -> nil
end
@@ -45,21 +45,26 @@ defmodule Pleroma.Web.CommonAPI.Utils do
|> Enum.reject(&is_nil/1)
end
- def attachments_from_ids_descs([], _), do: []
+ def attachments_from_ids_descs([], _, _), do: []
- def attachments_from_ids_descs(ids, descs_str) do
+ def attachments_from_ids_descs(ids, descs_str, user) do
{_, descs} = Jason.decode(descs_str)
Enum.map(ids, fn media_id ->
- with %Object{data: data} <- get_attachment(media_id) do
+ with %Object{data: data} <- get_attachment(media_id, user) do
Map.put(data, "name", descs[media_id])
end
end)
|> Enum.reject(&is_nil/1)
end
- defp get_attachment(media_id) do
- Repo.get(Object, media_id)
+ defp get_attachment(media_id, user) do
+ with %Object{data: _data} = object <- Repo.get(Object, media_id),
+ :ok <- Object.authorize_access(object, user) do
+ object
+ else
+ _ -> nil
+ end
end
@spec get_to_and_cc(ActivityDraft.t()) :: {list(String.t()), list(String.t())}
diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs
index d309c6ded..c52d3e9c5 100644
--- a/test/pleroma/web/common_api/utils_test.exs
+++ b/test/pleroma/web/common_api/utils_test.exs
@@ -586,41 +586,56 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
end
end
- describe "attachments_from_ids_descs/2" do
+ describe "attachments_from_ids_descs/3" do
test "returns [] when attachment ids is empty" do
- assert Utils.attachments_from_ids_descs([], "{}") == []
+ assert Utils.attachments_from_ids_descs([], "{}", nil) == []
end
test "returns list attachments with desc" do
- object = insert(:note)
+ user = insert(:user)
+ object = insert(:note, %{user: user})
desc = Jason.encode!(%{object.id => "test-desc"})
- assert Utils.attachments_from_ids_descs(["#{object.id}", "34"], desc) == [
+ assert Utils.attachments_from_ids_descs(["#{object.id}", "34"], desc, user) == [
Map.merge(object.data, %{"name" => "test-desc"})
]
end
end
- describe "attachments_from_ids/1" do
+ describe "attachments_from_ids/2" do
test "returns attachments with descs" do
- object = insert(:note)
+ user = insert(:user)
+ object = insert(:note, %{user: user})
desc = Jason.encode!(%{object.id => "test-desc"})
- assert Utils.attachments_from_ids(%{
- media_ids: ["#{object.id}"],
- descriptions: desc
- }) == [
+ assert Utils.attachments_from_ids(
+ %{
+ media_ids: ["#{object.id}"],
+ descriptions: desc
+ },
+ user
+ ) == [
Map.merge(object.data, %{"name" => "test-desc"})
]
end
test "returns attachments without descs" do
- object = insert(:note)
- assert Utils.attachments_from_ids(%{media_ids: ["#{object.id}"]}) == [object.data]
+ user = insert(:user)
+ object = insert(:note, %{user: user})
+ assert Utils.attachments_from_ids(%{media_ids: ["#{object.id}"]}, user) == [object.data]
end
test "returns [] when not pass media_ids" do
- assert Utils.attachments_from_ids(%{}) == []
+ assert Utils.attachments_from_ids(%{}, nil) == []
+ end
+
+ test "returns [] when media_ids not belong to current user" do
+ user = insert(:user)
+ user2 = insert(:user)
+
+ object = insert(:attachment, %{user: user})
+
+ assert Utils.attachments_from_ids(%{media_ids: ["#{object.id}"]}, user2) == []
end
end
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index 5c9103e9f..e60691995 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -279,6 +279,24 @@ defmodule Pleroma.Web.CommonAPITest do
assert {:reject, "[KeywordPolicy] Matches with rejected keyword"} ==
CommonAPI.post_chat_message(author, recipient, "GNO/Linux")
end
+
+ test "it reject messages with attachments not belonging to user" do
+ author = insert(:user)
+ not_author = insert(:user)
+ recipient = author
+
+ attachment = insert(:attachment, %{user: not_author})
+
+ {:error, message} =
+ CommonAPI.post_chat_message(
+ author,
+ recipient,
+ "123",
+ media_id: attachment.id
+ )
+
+ assert message == :forbidden
+ end
end
describe "unblocking" do
diff --git a/test/pleroma/web/mastodon_api/views/scheduled_activity_view_test.exs b/test/pleroma/web/mastodon_api/views/scheduled_activity_view_test.exs
index e5e510d33..07a65a3bc 100644
--- a/test/pleroma/web/mastodon_api/views/scheduled_activity_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/scheduled_activity_view_test.exs
@@ -48,7 +48,7 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityViewTest do
id: to_string(scheduled_activity.id),
media_attachments:
%{media_ids: [upload.id]}
- |> Utils.attachments_from_ids()
+ |> Utils.attachments_from_ids(user)
|> Enum.map(&StatusView.render("attachment.json", %{attachment: &1})),
params: %{
in_reply_to_id: to_string(activity.id),
diff --git a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
index 017c9c5c0..7ab3f5acd 100644
--- a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
+++ b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
@@ -24,7 +24,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceViewTest do
filename: "an_image.jpg"
}
- {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id)
+ {:ok, upload} = ActivityPub.upload(file, actor: recipient.ap_id)
{:ok, activity} =
CommonAPI.post_chat_message(user, recipient, "kippis :firefox:", idempotency_key: "123")
From 385492577d11e9667064d7f7e0dacdc00457064a Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Fri, 23 Dec 2022 18:46:14 +0100
Subject: [PATCH 144/823] mix: version 2.5.5
---
CHANGELOG.md | 7 ++++++-
mix.exs | 2 +-
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9d9aadc6e..32ec440de 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,7 +14,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Removed
-## 2.5.54
+## 2.5.5
+
+## Security
+- Prevent users from accessing media of other users by creating a status with reused attachment ID
+
+## 2.5.4
## Security
- Fix XML External Entity (XXE) loading vulnerability allowing to fetch arbitary files from the server's filesystem
diff --git a/mix.exs b/mix.exs
index 12f721364..e2aac0fc5 100644
--- a/mix.exs
+++ b/mix.exs
@@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do
def project do
[
app: :pleroma,
- version: version("2.5.4"),
+ version: version("2.5.5"),
elixir: "~> 1.11",
elixirc_paths: elixirc_paths(Mix.env()),
compilers: [:phoenix, :gettext] ++ Mix.compilers(),
From 28ef5ebd3c7c2fc083424dbf4434392fcd2a7aef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Thu, 7 Sep 2023 15:00:24 +0200
Subject: [PATCH 145/823] Update InstanceView.features
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
lib/pleroma/web/mastodon_api/views/instance_view.ex | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index e7a2feb7f..c1cecf6e6 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -92,7 +92,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
"shareable_emoji_packs",
"multifetch",
"pleroma:api/v1/notifications:include_types_filter",
- "quote_posting",
"editing",
if Config.get([:activitypub, :blockers_visible]) do
"blockers_visible"
@@ -103,6 +102,13 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
if Config.get([:gopher, :enabled]) do
"gopher"
end,
+ # backwards compat
+ if Config.get([:shout, :enabled]) do
+ "chat"
+ end,
+ if Config.get([:shout, :enabled]) do
+ "shout"
+ end,
if Config.get([:instance, :allow_relay]) do
"relay"
end,
@@ -112,7 +118,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
"pleroma_emoji_reactions",
"pleroma_custom_emoji_reactions",
"pleroma_chat_messages",
- "email_list",
if Config.get([:instance, :show_reactions]) do
"exposable_reactions"
end,
From 31eb3dc24587ad3715da9fe00886867a6c0bf0c4 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 16:41:30 -0600
Subject: [PATCH 146/823] ObjectValidators: accept "quoteUrl" field
---
.../web/activity_pub/object_validators/common_fields.ex | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex
index d580208df..835ed97b7 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex
@@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do
end
end
- # All objects except Answer and CHatMessage
+ # All objects except Answer and ChatMessage
defmacro object_fields do
quote bind_quoted: binding() do
field(:content, :string)
@@ -58,6 +58,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do
field(:like_count, :integer, default: 0)
field(:announcement_count, :integer, default: 0)
field(:inReplyTo, ObjectValidators.ObjectID)
+ field(:quoteUrl, ObjectValidators.ObjectID)
field(:url, ObjectValidators.BareUri)
field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
From 7deda1fa18cb95a588ba66950ac45262c467a7f4 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 17:30:49 -0600
Subject: [PATCH 147/823] Quote post: add fixtures
---
.../quote_post/fedibird_quote_post.json | 52 +++++++++++++++++++
.../quote_post/misskey_quote_post.json | 46 ++++++++++++++++
2 files changed, 98 insertions(+)
create mode 100644 test/fixtures/quote_post/fedibird_quote_post.json
create mode 100644 test/fixtures/quote_post/misskey_quote_post.json
diff --git a/test/fixtures/quote_post/fedibird_quote_post.json b/test/fixtures/quote_post/fedibird_quote_post.json
new file mode 100644
index 000000000..ebf383356
--- /dev/null
+++ b/test/fixtures/quote_post/fedibird_quote_post.json
@@ -0,0 +1,52 @@
+{
+ "@context": [
+ "https://www.w3.org/ns/activitystreams",
+ {
+ "ostatus": "http://ostatus.org#",
+ "atomUri": "ostatus:atomUri",
+ "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+ "conversation": "ostatus:conversation",
+ "sensitive": "as:sensitive",
+ "toot": "http://joinmastodon.org/ns#",
+ "votersCount": "toot:votersCount",
+ "expiry": "toot:expiry"
+ }
+ ],
+ "id": "https://fedibird.com/users/noellabo/statuses/107663670404015196",
+ "type": "Note",
+ "summary": null,
+ "inReplyTo": null,
+ "published": "2022-01-22T02:07:16Z",
+ "url": "https://fedibird.com/@noellabo/107663670404015196",
+ "attributedTo": "https://fedibird.com/users/noellabo",
+ "to": [
+ "https://www.w3.org/ns/activitystreams#Public"
+ ],
+ "cc": [
+ "https://fedibird.com/users/noellabo/followers"
+ ],
+ "sensitive": false,
+ "atomUri": "https://fedibird.com/users/noellabo/statuses/107663670404015196",
+ "inReplyToAtomUri": null,
+ "conversation": "tag:fedibird.com,2022-01-22:objectId=107663670404038002:objectType=Conversation",
+ "context": "https://fedibird.com/contexts/107663670404038002",
+ "quoteURL": "https://misskey.io/notes/8vsn2izjwh",
+ "_misskey_quote": "https://misskey.io/notes/8vsn2izjwh",
+ "_misskey_content": "いつの生まれだシトリン",
+ "content": "いつの生まれだシトリン QT: https:// misskey.io/notes/8vsn2izjwh
",
+ "contentMap": {
+ "ja": "いつの生まれだシトリン QT: https:// misskey.io/notes/8vsn2izjwh
"
+ },
+ "attachment": [],
+ "tag": [],
+ "replies": {
+ "id": "https://fedibird.com/users/noellabo/statuses/107663670404015196/replies",
+ "type": "Collection",
+ "first": {
+ "type": "CollectionPage",
+ "next": "https://fedibird.com/users/noellabo/statuses/107663670404015196/replies?only_other_accounts=true&page=true",
+ "partOf": "https://fedibird.com/users/noellabo/statuses/107663670404015196/replies",
+ "items": []
+ }
+ }
+}
diff --git a/test/fixtures/quote_post/misskey_quote_post.json b/test/fixtures/quote_post/misskey_quote_post.json
new file mode 100644
index 000000000..59f677ca9
--- /dev/null
+++ b/test/fixtures/quote_post/misskey_quote_post.json
@@ -0,0 +1,46 @@
+{
+ "@context": [
+ "https://www.w3.org/ns/activitystreams",
+ "https://w3id.org/security/v1",
+ {
+ "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+ "sensitive": "as:sensitive",
+ "Hashtag": "as:Hashtag",
+ "quoteUrl": "as:quoteUrl",
+ "toot": "http://joinmastodon.org/ns#",
+ "Emoji": "toot:Emoji",
+ "featured": "toot:featured",
+ "discoverable": "toot:discoverable",
+ "schema": "http://schema.org#",
+ "PropertyValue": "schema:PropertyValue",
+ "value": "schema:value",
+ "misskey": "https://misskey.io/ns#",
+ "_misskey_content": "misskey:_misskey_content",
+ "_misskey_quote": "misskey:_misskey_quote",
+ "_misskey_reaction": "misskey:_misskey_reaction",
+ "_misskey_votes": "misskey:_misskey_votes",
+ "_misskey_talk": "misskey:_misskey_talk",
+ "isCat": "misskey:isCat",
+ "vcard": "http://www.w3.org/2006/vcard/ns#"
+ }
+ ],
+ "id": "https://misskey.io/notes/8vs6ylpfez",
+ "type": "Note",
+ "attributedTo": "https://misskey.io/users/7rkrarq81i",
+ "summary": null,
+ "content": "投稿者の設定によるね Fanboxについても投稿者によっては過去の投稿は高額なプランに移動してることがある RE: https://misskey.io/notes/8vs6wxufd0
",
+ "_misskey_content": "投稿者の設定によるね\nFanboxについても投稿者によっては過去の投稿は高額なプランに移動してることがある",
+ "_misskey_quote": "https://misskey.io/notes/8vs6wxufd0",
+ "quoteUrl": "https://misskey.io/notes/8vs6wxufd0",
+ "published": "2022-01-21T16:38:30.243Z",
+ "to": [
+ "https://www.w3.org/ns/activitystreams#Public"
+ ],
+ "cc": [
+ "https://misskey.io/users/7rkrarq81i/followers"
+ ],
+ "inReplyTo": null,
+ "attachment": [],
+ "sensitive": false,
+ "tag": []
+}
From 795736af16dca77929725e7dd55f5de04a796fdb Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 18:03:22 -0600
Subject: [PATCH 148/823] ObjectValidators: improve quoteUrl compatibility
---
.../article_note_page_validator.ex | 16 +++++++++++++++
.../article_note_page_validator_test.exs | 20 +++++++++++++++++++
2 files changed, 36 insertions(+)
diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
index 2670e3f17..40bb67934 100644
--- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
@@ -76,6 +76,21 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
def fix_attachments(data), do: data
+ defp fix_quote_url(%{"quoteUrl" => _quote_url} = data), do: data
+
+ # Fix for Fedibird
+ # https://github.com/fedibird/mastodon/issues/9
+ defp fix_quote_url(%{"quoteURL" => quote_url} = data) do
+ Map.put(data, "quoteUrl", quote_url)
+ end
+
+ # Misskey fallback
+ defp fix_quote_url(%{"_misskey_quote" => quote_url} = data) do
+ Map.put(data, "quoteUrl", quote_url)
+ end
+
+ defp fix_quote_url(data), do: data
+
defp fix(data) do
data
|> CommonFixes.fix_actor()
@@ -84,6 +99,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
|> fix_tag()
|> fix_replies()
|> fix_attachments()
+ |> fix_quote_url()
|> Transmogrifier.fix_emoji()
|> Transmogrifier.fix_content_map()
end
diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
index c7a62be18..c3cde00b5 100644
--- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
+++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
@@ -116,4 +116,24 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest
%{valid?: true} = ArticleNotePageValidator.cast_and_validate(note)
end
+
+ test "Fedibird quote post" do
+ insert(:user, ap_id: "https://fedibird.com/users/noellabo")
+
+ data = File.read!("test/fixtures/quote_post/fedibird_quote_post.json") |> Jason.decode!()
+ chg = ArticleNotePageValidator.cast_and_validate(data)
+
+ assert chg.valid?
+ assert chg.changes.quoteUrl == "https://misskey.io/notes/8vsn2izjwh"
+ end
+
+ test "Misskey quote post" do
+ insert(:user, ap_id: "https://misskey.io/users/7rkrarq81i")
+
+ data = File.read!("test/fixtures/quote_post/misskey_quote_post.json") |> Jason.decode!()
+ chg = ArticleNotePageValidator.cast_and_validate(data)
+
+ assert chg.valid?
+ assert chg.changes.quoteUrl == "https://misskey.io/notes/8vs6wxufd0"
+ end
end
From b022d6635dad4b2769fbf1fd4b97f77a4cc646b4 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 18:46:58 -0600
Subject: [PATCH 149/823] Transmogrifier: fetch quoted post
---
.../web/activity_pub/transmogrifier.ex | 17 +++++
test/fixtures/tesla_mock/aimu@misskey.io.json | 64 +++++++++++++++++++
.../tesla_mock/misskey.io_8vs6wxufd0.json | 44 +++++++++++++
.../web/activity_pub/transmogrifier_test.exs | 22 +++++++
test/support/http_request_mock.ex | 18 ++++++
5 files changed, 165 insertions(+)
create mode 100644 test/fixtures/tesla_mock/aimu@misskey.io.json
create mode 100644 test/fixtures/tesla_mock/misskey.io_8vs6wxufd0.json
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 0e6c429f9..c466271ca 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -166,6 +166,22 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def fix_in_reply_to(object, _options), do: object
+ def fix_quote(object, options \\ [])
+
+ def fix_quote(%{"quoteUrl" => quote_url} = object, options)
+ when not is_nil(quote_url) do
+ with {:ok, quoted_object} <- get_obj_helper(quote_url, options),
+ %Activity{} <- Activity.get_create_by_object_ap_id(quoted_object.data["id"]) do
+ Map.put(object, "quoteUrl", quoted_object.data["id"])
+ else
+ e ->
+ Logger.warn("Couldn't fetch #{inspect(quote_url)}, error: #{inspect(e)}")
+ object
+ end
+ end
+
+ def fix_quote(object, _options), do: object
+
defp prepare_in_reply_to(in_reply_to) do
cond do
is_bitstring(in_reply_to) ->
@@ -454,6 +470,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
|> strip_internal_fields()
|> fix_type(fetch_options)
|> fix_in_reply_to(fetch_options)
+ |> fix_quote(fetch_options)
data = Map.put(data, "object", object)
options = Keyword.put(options, :local, false)
diff --git a/test/fixtures/tesla_mock/aimu@misskey.io.json b/test/fixtures/tesla_mock/aimu@misskey.io.json
new file mode 100644
index 000000000..9ff4cb6d0
--- /dev/null
+++ b/test/fixtures/tesla_mock/aimu@misskey.io.json
@@ -0,0 +1,64 @@
+{
+ "@context": [
+ "https://www.w3.org/ns/activitystreams",
+ "https://w3id.org/security/v1",
+ {
+ "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+ "sensitive": "as:sensitive",
+ "Hashtag": "as:Hashtag",
+ "quoteUrl": "as:quoteUrl",
+ "toot": "http://joinmastodon.org/ns#",
+ "Emoji": "toot:Emoji",
+ "featured": "toot:featured",
+ "discoverable": "toot:discoverable",
+ "schema": "http://schema.org#",
+ "PropertyValue": "schema:PropertyValue",
+ "value": "schema:value",
+ "misskey": "https://misskey.io/ns#",
+ "_misskey_content": "misskey:_misskey_content",
+ "_misskey_quote": "misskey:_misskey_quote",
+ "_misskey_reaction": "misskey:_misskey_reaction",
+ "_misskey_votes": "misskey:_misskey_votes",
+ "_misskey_talk": "misskey:_misskey_talk",
+ "isCat": "misskey:isCat",
+ "vcard": "http://www.w3.org/2006/vcard/ns#"
+ }
+ ],
+ "type": "Person",
+ "id": "https://misskey.io/users/83ssedkv53",
+ "inbox": "https://misskey.io/users/83ssedkv53/inbox",
+ "outbox": "https://misskey.io/users/83ssedkv53/outbox",
+ "followers": "https://misskey.io/users/83ssedkv53/followers",
+ "following": "https://misskey.io/users/83ssedkv53/following",
+ "sharedInbox": "https://misskey.io/inbox",
+ "endpoints": {
+ "sharedInbox": "https://misskey.io/inbox"
+ },
+ "url": "https://misskey.io/@aimu",
+ "preferredUsername": "aimu",
+ "name": "あいむ",
+ "summary": "わずかな作曲要素 巣穴で独り言 Twitter https://twitter.com/aimu_53 Soundcloud https://soundcloud.com/aimu-53
",
+ "icon": {
+ "type": "Image",
+ "url": "https://s3.arkjp.net/misskey/webpublic-3f7e93c0-34f5-443c-acc0-f415cb2342b4.jpg",
+ "sensitive": false,
+ "name": null
+ },
+ "image": {
+ "type": "Image",
+ "url": "https://s3.arkjp.net/misskey/webpublic-2db63d1d-490b-488b-ab62-c93c285f26b6.png",
+ "sensitive": false,
+ "name": null
+ },
+ "tag": [],
+ "manuallyApprovesFollowers": false,
+ "discoverable": true,
+ "publicKey": {
+ "id": "https://misskey.io/users/83ssedkv53#main-key",
+ "type": "Key",
+ "owner": "https://misskey.io/users/83ssedkv53",
+ "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1ylhePJ6qGHmwHSBP17b\nIosxGaiFKvgDBgZdm8vzvKeRSqJV9uLHfZL3pO/Zt02EwaZd2GohZAtBZEF8DbMA\n3s93WAesvyGF9mjGrYYKlhp/glwyrrrbf+RdD0DLtyDwRRlrxp3pS2lLmv5Tp1Zl\npH+UKpOnNrpQqjHI5P+lEc9bnflzbRrX+UiyLNsVAP80v4wt7SZfT/telrU6mDru\n998UdfhUo7bDKeDsHG1PfLpyhhtfdoZub4kBpkyacHiwAd+CdCjR54Eu7FDwVK3p\nY3JcrT2q5stgMqN1m4QgSL4XAADIotWwDYttTJejM1n9dr+6VWv5bs0F2Q/6gxOp\nu5DQZLk4Q+64U4LWNox6jCMOq3fYe0g7QalJIHnanYQQo+XjoH6S1Aw64gQ3Ip2Y\nZBmZREAOR7GMFVDPFnVnsbCHnIAv16TdgtLgQBAihkWEUuPqITLi8PMu6kMr3uyq\nYkObEfH0TNTcqaiVpoXv791GZLEUV5ROl0FSUANLNkHZZv29xZ5JDOBOR1rNBLyH\ngVtW8rpszYqOXwzX23hh4WsVXfB7YgNvIijwjiaWbzsecleaENGEnLNMiVKVumTj\nmtyTeFJpH0+OaSrUYpemRRJizmqIjklKsNwUEwUb2WcUUg92o56T2obrBkooabZe\nwgSXSKTOcjsR/ju7+AuIyvkCAwEAAQ==\n-----END PUBLIC KEY-----\n"
+ },
+ "isCat": true,
+ "vcard:bday": "5353-05-03"
+}
diff --git a/test/fixtures/tesla_mock/misskey.io_8vs6wxufd0.json b/test/fixtures/tesla_mock/misskey.io_8vs6wxufd0.json
new file mode 100644
index 000000000..323ca10ed
--- /dev/null
+++ b/test/fixtures/tesla_mock/misskey.io_8vs6wxufd0.json
@@ -0,0 +1,44 @@
+{
+ "@context": [
+ "https://www.w3.org/ns/activitystreams",
+ "https://w3id.org/security/v1",
+ {
+ "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+ "sensitive": "as:sensitive",
+ "Hashtag": "as:Hashtag",
+ "quoteUrl": "as:quoteUrl",
+ "toot": "http://joinmastodon.org/ns#",
+ "Emoji": "toot:Emoji",
+ "featured": "toot:featured",
+ "discoverable": "toot:discoverable",
+ "schema": "http://schema.org#",
+ "PropertyValue": "schema:PropertyValue",
+ "value": "schema:value",
+ "misskey": "https://misskey.io/ns#",
+ "_misskey_content": "misskey:_misskey_content",
+ "_misskey_quote": "misskey:_misskey_quote",
+ "_misskey_reaction": "misskey:_misskey_reaction",
+ "_misskey_votes": "misskey:_misskey_votes",
+ "_misskey_talk": "misskey:_misskey_talk",
+ "isCat": "misskey:isCat",
+ "vcard": "http://www.w3.org/2006/vcard/ns#"
+ }
+ ],
+ "id": "https://misskey.io/notes/8vs6wxufd0",
+ "type": "Note",
+ "attributedTo": "https://misskey.io/users/83ssedkv53",
+ "summary": null,
+ "content": "Fantiaこれできないように過去のやつは従量課金だった気がする
",
+ "_misskey_content": "Fantiaこれできないように過去のやつは従量課金だった気がする",
+ "published": "2022-01-21T16:37:12.663Z",
+ "to": [
+ "https://www.w3.org/ns/activitystreams#Public"
+ ],
+ "cc": [
+ "https://misskey.io/users/83ssedkv53/followers"
+ ],
+ "inReplyTo": null,
+ "attachment": [],
+ "sensitive": false,
+ "tag": []
+}
diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs
index 3e0c8dc65..2c8e5ba21 100644
--- a/test/pleroma/web/activity_pub/transmogrifier_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs
@@ -136,6 +136,28 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
tag = object.data["tag"] |> List.first()
assert tag["type"] == "Mention"
end
+
+ test "it accepts quote posts" do
+ insert(:user, ap_id: "https://misskey.io/users/7rkrarq81i")
+
+ object = File.read!("test/fixtures/quote_post/misskey_quote_post.json") |> Jason.decode!()
+
+ message = %{
+ "@context" => "https://www.w3.org/ns/activitystreams",
+ "type" => "Create",
+ "actor" => "https://misskey.io/users/7rkrarq81i",
+ "object" => object
+ }
+
+ assert {:ok, activity} = Transmogrifier.handle_incoming(message)
+
+ # Object was created in the database
+ object = Object.normalize(activity)
+ assert object.data["quoteUrl"] == "https://misskey.io/notes/8vs6wxufd0"
+
+ # It fetched the quoted post
+ assert Object.normalize("https://misskey.io/notes/8vs6wxufd0")
+ end
end
describe "prepare outgoing" do
diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex
index b0cf613ac..78a367024 100644
--- a/test/support/http_request_mock.ex
+++ b/test/support/http_request_mock.ex
@@ -1380,6 +1380,15 @@ defmodule HttpRequestMock do
}}
end
+ def get("https://misskey.io/users/83ssedkv53", _, _, _) do
+ {:ok,
+ %Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/tesla_mock/aimu@misskey.io.json"),
+ headers: activitypub_object_headers()
+ }}
+ end
+
def get("https://gleasonator.com/users/macgirvin", _, _, _) do
{:ok,
%Tesla.Env{
@@ -1446,6 +1455,15 @@ defmodule HttpRequestMock do
}}
end
+ def get("https://misskey.io/notes/8vs6wxufd0", _, _, _) do
+ {:ok,
+ %Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/tesla_mock/misskey.io_8vs6wxufd0.json"),
+ headers: activitypub_object_headers()
+ }}
+ end
+
def get(url, query, body, headers) do
{:error,
"Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"}
From cc4badaf60462fdb8bb57225437e3dd360ee0dfb Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 19:14:39 -0600
Subject: [PATCH 150/823] Transmogrifier: fix quoteUrl here too
---
.../web/activity_pub/transmogrifier.ex | 23 +++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index c466271ca..f5771e75e 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -166,9 +166,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def fix_in_reply_to(object, _options), do: object
- def fix_quote(object, options \\ [])
+ def fix_quote_url(object, options \\ [])
- def fix_quote(%{"quoteUrl" => quote_url} = object, options)
+ def fix_quote_url(%{"quoteUrl" => quote_url} = object, options)
when not is_nil(quote_url) do
with {:ok, quoted_object} <- get_obj_helper(quote_url, options),
%Activity{} <- Activity.get_create_by_object_ap_id(quoted_object.data["id"]) do
@@ -180,7 +180,22 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
end
end
- def fix_quote(object, _options), do: object
+ # Fix for Fedibird
+ # https://github.com/fedibird/mastodon/issues/9
+ def fix_quote_url(%{"quoteURL" => quote_url} = object, options) do
+ object
+ |> Map.put("quoteUrl", quote_url)
+ |> fix_quote_url(options)
+ end
+
+ # Misskey fallback
+ def fix_quote_url(%{"_misskey_quote" => quote_url} = object, options) do
+ object
+ |> Map.put("quoteUrl", quote_url)
+ |> fix_quote_url(options)
+ end
+
+ def fix_quote_url(object, _options), do: object
defp prepare_in_reply_to(in_reply_to) do
cond do
@@ -470,7 +485,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
|> strip_internal_fields()
|> fix_type(fetch_options)
|> fix_in_reply_to(fetch_options)
- |> fix_quote(fetch_options)
+ |> fix_quote_url(fetch_options)
data = Map.put(data, "object", object)
options = Keyword.put(options, :local, false)
From ce5eb3172321f0ef2ae85d7819b44cc8241a5581 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 19:47:08 -0600
Subject: [PATCH 151/823] StatusView: show quoted posts through the API,
probably
---
.../web/mastodon_api/views/status_view.ex | 42 ++++++++++++++++++-
1 file changed, 41 insertions(+), 1 deletion(-)
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index dea22f9c2..b966a84d0 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -57,6 +57,27 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
end)
end
+ defp get_quoted_activities([]), do: %{}
+
+ defp get_quoted_activities(activities) do
+ activities
+ |> Enum.map(fn
+ %{data: %{"type" => "Create"}} = activity ->
+ object = Object.normalize(activity, fetch: false)
+ object && object.data["quoteUrl"] != "" && object.data["quoteUrl"]
+
+ _ ->
+ nil
+ end)
+ |> Enum.filter(& &1)
+ |> Activity.create_by_object_ap_id_with_object()
+ |> Repo.all()
+ |> Enum.reduce(%{}, fn activity, acc ->
+ object = Object.normalize(activity, fetch: false)
+ if object, do: Map.put(acc, object.data["id"], activity), else: acc
+ end)
+ end
+
# DEPRECATED This field seems to be a left-over from the StatusNet era.
# If your application uses `pleroma.conversation_id`: this field is deprecated.
# It is currently stubbed instead by doing a CRC32 of the context, and
@@ -97,6 +118,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
# length(activities_with_links) * timeout
fetch_rich_media_for_activities(activities)
replied_to_activities = get_replied_to_activities(activities)
+ quoted_activities = get_quoted_activities(activities)
parent_activities =
activities
@@ -129,6 +151,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
opts =
opts
|> Map.put(:replied_to_activities, replied_to_activities)
+ |> Map.put(:quoted_activities, quoted_activities)
|> Map.put(:parent_activities, parent_activities)
|> Map.put(:relationships, relationships_opt)
@@ -277,7 +300,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
end
reply_to = get_reply_to(activity, opts)
-
reply_to_user = reply_to && CommonAPI.get_user(reply_to.data["actor"])
history_len =
@@ -290,6 +312,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
# Here the implicit index of the current content is 0
chrono_order = history_len - 1
+ quote_activity = get_quote(activity, opts)
+
content =
object
|> render_content()
@@ -398,6 +422,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
conversation_id: get_context_id(activity),
context: object.data["context"],
in_reply_to_account_acct: reply_to_user && reply_to_user.nickname,
+ quote_id: quote_activity && to_string(quote_activity.id),
content: %{"text/plain" => content_plaintext},
spoiler_text: %{"text/plain" => summary},
expires_at: expires_at,
@@ -633,6 +658,21 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
end
end
+ def get_quote(activity, %{quoted_activities: quoted_activities}) do
+ object = Object.normalize(activity, fetch: false)
+ quoted_activities[object.data["quoteUrl"]]
+ end
+
+ def get_quote(%{data: %{"object" => _object}} = activity, _) do
+ object = Object.normalize(activity, fetch: false)
+
+ if object.data["quoteUrl"] && object.data["quoteUrl"] != "" do
+ Activity.get_create_by_object_ap_id(object.data["quoteUrl"])
+ else
+ nil
+ end
+ end
+
def render_content(%{data: %{"name" => name}} = object) when not is_nil(name) and name != "" do
url = object.data["url"] || object.data["id"]
From 0d9c443e51c85d9ded3e20954c9620f7a9d2361e Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 20:05:58 -0600
Subject: [PATCH 152/823] StatusView: render the whole quoted status
---
lib/pleroma/web/api_spec/schemas/status.ex | 5 +++++
lib/pleroma/web/mastodon_api/views/status_view.ex | 10 +++++++++-
.../web/mastodon_api/views/status_view_test.exs | 1 +
3 files changed, 15 insertions(+), 1 deletion(-)
diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex
index bc29cf4a6..39241aa39 100644
--- a/lib/pleroma/web/api_spec/schemas/status.ex
+++ b/lib/pleroma/web/api_spec/schemas/status.ex
@@ -193,6 +193,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
nullable: true,
description: "The `acct` property of User entity for replied user (if any)"
},
+ quote: %Schema{
+ allOf: [%OpenApiSpex.Reference{"$ref": "#/components/schemas/Status"}],
+ nullable: true,
+ description: "Quoted status (if any)"
+ },
local: %Schema{
type: :boolean,
description: "`true` if the post was made on the local instance"
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index b966a84d0..5bde1ce04 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -314,6 +314,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
quote_activity = get_quote(activity, opts)
+ quote_post =
+ if quote_activity do
+ quote_rendering_opts = Map.put(opts, :activity, quote_activity)
+ render("show.json", quote_rendering_opts)
+ else
+ nil
+ end
+
content =
object
|> render_content()
@@ -422,7 +430,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
conversation_id: get_context_id(activity),
context: object.data["context"],
in_reply_to_account_acct: reply_to_user && reply_to_user.nickname,
- quote_id: quote_activity && to_string(quote_activity.id),
+ quote: quote_post,
content: %{"text/plain" => content_plaintext},
spoiler_text: %{"text/plain" => summary},
expires_at: expires_at,
diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs
index b93335190..b10b0f0b9 100644
--- a/test/pleroma/web/mastodon_api/views/status_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs
@@ -326,6 +326,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
conversation_id: convo_id,
context: object_data["context"],
in_reply_to_account_acct: nil,
+ quote: nil,
content: %{"text/plain" => HTML.strip_tags(object_data["content"])},
spoiler_text: %{"text/plain" => HTML.strip_tags(object_data["summary"])},
expires_at: nil,
From 6ac19c3999c543e5a26bbf04932a6a7aaa447b99 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 21:27:05 -0600
Subject: [PATCH 153/823] ActivityDraft: create quote posts
---
lib/pleroma/web/common_api/activity_draft.ex | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex
index 63ed48a27..375aabc91 100644
--- a/lib/pleroma/web/common_api/activity_draft.ex
+++ b/lib/pleroma/web/common_api/activity_draft.ex
@@ -22,6 +22,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
attachments: [],
in_reply_to: nil,
in_reply_to_conversation: nil,
+ quote_post: nil,
visibility: nil,
expires_at: nil,
extra: nil,
@@ -53,6 +54,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
|> poll()
|> with_valid(&in_reply_to/1)
|> with_valid(&in_reply_to_conversation/1)
+ |> with_valid("e_post/1)
|> with_valid(&visibility/1)
|> content()
|> with_valid(&to_and_cc/1)
@@ -132,6 +134,18 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
defp in_reply_to(draft), do: draft
+ defp quote_post(%{params: %{quote_id: ""}} = draft), do: draft
+
+ defp quote_post(%{params: %{quote_id: id}} = draft) when is_binary(id) do
+ %__MODULE__{draft | quote_post: Activity.get_by_id(id)}
+ end
+
+ defp quote_post(%{params: %{quote_id: %Activity{} = quote_post}} = draft) do
+ %__MODULE__{draft | quote_post: quote_post}
+ end
+
+ defp quote_post(draft), do: draft
+
defp in_reply_to_conversation(draft) do
in_reply_to_conversation = Participation.get(draft.params[:in_reply_to_conversation_id])
%__MODULE__{draft | in_reply_to_conversation: in_reply_to_conversation}
From d4fea8b5595e9e6cd37bdb1cee21285f905693f1 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 22:15:54 -0600
Subject: [PATCH 154/823] ActivityDraft: allow quoting
---
lib/pleroma/web/activity_pub/builder.ex | 11 +++++++++++
.../web/api_spec/operations/status_operation.ex | 7 ++++++-
test/pleroma/web/common_api_test.exs | 12 ++++++++++++
3 files changed, 29 insertions(+), 1 deletion(-)
diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex
index 8eab3a241..eb0bb0e33 100644
--- a/lib/pleroma/web/activity_pub/builder.ex
+++ b/lib/pleroma/web/activity_pub/builder.ex
@@ -217,6 +217,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do
"tag" => Keyword.values(draft.tags) |> Enum.uniq()
}
|> add_in_reply_to(draft.in_reply_to)
+ |> add_quote(draft.quote_post)
|> Map.merge(draft.extra)
{:ok, data, []}
@@ -232,6 +233,16 @@ defmodule Pleroma.Web.ActivityPub.Builder do
end
end
+ defp add_quote(object, nil), do: object
+
+ defp add_quote(object, quote_post) do
+ with %Object{} = quote_object <- Object.normalize(quote_post, fetch: false) do
+ Map.put(object, "quoteUrl", quote_object.data["id"])
+ else
+ _ -> object
+ end
+ end
+
def chat_message(actor, recipient, content, opts \\ []) do
basic = %{
"id" => Utils.generate_object_id(),
diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex
index 5d6e82f3c..8fa3b0890 100644
--- a/lib/pleroma/web/api_spec/operations/status_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/status_operation.ex
@@ -581,7 +581,12 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do
type: :string,
description:
"Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`."
- }
+ },
+ quote_id: %Schema{
+ nullable: true,
+ allOf: [FlakeID],
+ description: "ID of the status being quoted, if any"
+ },
},
example: %{
"status" => "What time is it?",
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index 0d76d6581..09df27acb 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -796,6 +796,18 @@ defmodule Pleroma.Web.CommonAPITest do
scheduled_at: expires_at
)
end
+
+ test "it allows allows quote posting" do
+ user = insert(:user)
+
+ {:ok, quoted} = CommonAPI.post(user, %{status: "Hello world"})
+ {:ok, quote_post} = CommonAPI.post(user, %{status: "nice post", quote_id: quoted.id})
+
+ quoted = Object.normalize(quoted)
+ quote_post = Object.normalize(quote_post)
+
+ assert quote_post.data["quoteUrl"] == quoted.data["id"]
+ end
end
describe "reactions" do
From c20e90e898affc9d00d8b7d6b71f11157f5c7837 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 22:29:13 -0600
Subject: [PATCH 155/823] BuilderTest: build quote post
---
.../pleroma/web/activity_pub/builder_test.exs | 29 +++++++++++++++++++
1 file changed, 29 insertions(+)
diff --git a/test/pleroma/web/activity_pub/builder_test.exs b/test/pleroma/web/activity_pub/builder_test.exs
index eb175a1be..52058a0a3 100644
--- a/test/pleroma/web/activity_pub/builder_test.exs
+++ b/test/pleroma/web/activity_pub/builder_test.exs
@@ -44,5 +44,34 @@ defmodule Pleroma.Web.ActivityPub.BuilderTest do
assert {:ok, ^expected, []} = Builder.note(draft)
end
+
+ test "quote post" do
+ user = insert(:user)
+ note = insert(:note)
+
+ draft = %ActivityDraft{
+ user: user,
+ context: "2hu",
+ content_html: "This is :moominmamma: note ",
+ quote_post: note,
+ extra: %{}
+ }
+
+ expected = %{
+ "actor" => user.ap_id,
+ "attachment" => [],
+ "content" => "This is :moominmamma: note ",
+ "context" => "2hu",
+ "sensitive" => false,
+ "type" => "Note",
+ "quoteUrl" => note.data["id"],
+ "cc" => [],
+ "summary" => nil,
+ "tag" => [],
+ "to" => []
+ }
+
+ assert {:ok, ^expected, []} = Builder.note(draft)
+ end
end
end
From 3a8b5d90df6de502debaee4d670211bcf64ad1db Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 22:35:08 -0600
Subject: [PATCH 156/823] StatusControllerTest: test creating a quote post
---
.../controllers/status_controller_test.exs | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
index 76c289ee7..ba49256b5 100644
--- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
@@ -125,6 +125,25 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
)
end
+ test "posting a quote post", %{conn: conn} do
+ user = insert(:user)
+
+ {:ok, %{id: activity_id}} = CommonAPI.post(user, %{status: "yolo"})
+
+ conn =
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/v1/statuses", %{
+ "status" => "indeed",
+ "quote_id" => activity_id
+ })
+
+ assert %{"id" => id, "pleroma" => %{"quote" => %{"id" => ^activity_id}}} =
+ json_response_and_validate_schema(conn, 200)
+
+ assert Activity.get_by_id(id)
+ end
+
test "it fails to create a status if `expires_in` is less or equal than an hour", %{
conn: conn
} do
From cbd1760efac872c00edad15f352ffe4d2e0e1e12 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 22:41:57 -0600
Subject: [PATCH 157/823] TransmogrifierTest: prepare an outgoing quote post
---
.../pleroma/web/activity_pub/transmogrifier_test.exs | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs
index 2c8e5ba21..824398e38 100644
--- a/test/pleroma/web/activity_pub/transmogrifier_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs
@@ -372,6 +372,18 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
}
} = prepared["object"]
end
+
+ test "it prepares a quote post" do
+ user = insert(:user)
+
+ {:ok, quoted_post} = CommonAPI.post(user, %{status: "hey"})
+ {:ok, quote_post} = CommonAPI.post(user, %{status: "hey", quote_id: quoted_post.id})
+
+ {:ok, modified} = Transmogrifier.prepare_outgoing(quote_post.data)
+
+ quoted_post = Object.normalize(quoted_post)
+ assert modified["object"]["quoteUrl"] == quoted_post.data["id"]
+ end
end
describe "actor rewriting" do
From 96009739173e5e48a636bb964855eb7aea11c828 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 22:57:42 -0600
Subject: [PATCH 158/823] mix format
---
lib/pleroma/web/api_spec/operations/status_operation.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex
index 8fa3b0890..c133a3aac 100644
--- a/lib/pleroma/web/api_spec/operations/status_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/status_operation.ex
@@ -586,7 +586,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do
nullable: true,
allOf: [FlakeID],
description: "ID of the status being quoted, if any"
- },
+ }
},
example: %{
"status" => "What time is it?",
From f4ccdfd5033e7b1136ae0fe4e41dba78d83e80cf Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 23:02:44 -0600
Subject: [PATCH 159/823] Fix typos
---
.../article_note_page_validator_test.exs | 12 ++++++------
test/pleroma/web/common_api_test.exs | 2 +-
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
index c3cde00b5..dec2e28c9 100644
--- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
+++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
@@ -121,19 +121,19 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest
insert(:user, ap_id: "https://fedibird.com/users/noellabo")
data = File.read!("test/fixtures/quote_post/fedibird_quote_post.json") |> Jason.decode!()
- chg = ArticleNotePageValidator.cast_and_validate(data)
+ cng = ArticleNotePageValidator.cast_and_validate(data)
- assert chg.valid?
- assert chg.changes.quoteUrl == "https://misskey.io/notes/8vsn2izjwh"
+ assert cng.valid?
+ assert cng.changes.quoteUrl == "https://misskey.io/notes/8vsn2izjwh"
end
test "Misskey quote post" do
insert(:user, ap_id: "https://misskey.io/users/7rkrarq81i")
data = File.read!("test/fixtures/quote_post/misskey_quote_post.json") |> Jason.decode!()
- chg = ArticleNotePageValidator.cast_and_validate(data)
+ cng = ArticleNotePageValidator.cast_and_validate(data)
- assert chg.valid?
- assert chg.changes.quoteUrl == "https://misskey.io/notes/8vs6wxufd0"
+ assert cng.valid?
+ assert cng.changes.quoteUrl == "https://misskey.io/notes/8vs6wxufd0"
end
end
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index 09df27acb..734e6dd82 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -797,7 +797,7 @@ defmodule Pleroma.Web.CommonAPITest do
)
end
- test "it allows allows quote posting" do
+ test "it allows quote posting" do
user = insert(:user)
{:ok, quoted} = CommonAPI.post(user, %{status: "Hello world"})
From 5716f88a1d8424cf7c62a0491b3bf9607dc9aa3f Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 23:09:33 -0600
Subject: [PATCH 160/823] InstanceView: add "quote_posting" feature
---
lib/pleroma/web/mastodon_api/views/instance_view.ex | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index efd2a0af6..1b01d7371 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -69,6 +69,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
"multifetch",
"pleroma:api/v1/notifications:include_types_filter",
"editing",
+ "quote_posting",
if Config.get([:activitypub, :blockers_visible]) do
"blockers_visible"
end,
From db46abce477b83b252e0ad87f74ac9266e9cb118 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sat, 22 Jan 2022 23:29:55 -0600
Subject: [PATCH 161/823] @context: add quoteUrl
---
priv/static/schemas/litepub-0.1.jsonld | 1 +
1 file changed, 1 insertion(+)
diff --git a/priv/static/schemas/litepub-0.1.jsonld b/priv/static/schemas/litepub-0.1.jsonld
index 650118475..5d8244a11 100644
--- a/priv/static/schemas/litepub-0.1.jsonld
+++ b/priv/static/schemas/litepub-0.1.jsonld
@@ -26,6 +26,7 @@
"@id": "litepub:listMessage",
"@type": "@id"
},
+ "quoteUrl": "as:quoteUrl",
"oauthRegistrationEndpoint": {
"@id": "litepub:oauthRegistrationEndpoint",
"@type": "@id"
From 80ab2572a4d5590d738cc763a87156b3f79362fb Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sun, 23 Jan 2022 13:55:25 -0600
Subject: [PATCH 162/823] Return quote_url through the API, don't render quotes
more than 1 level deep
---
lib/pleroma/web/api_spec/schemas/status.ex | 6 +++++
.../web/mastodon_api/views/status_view.ex | 7 +++++-
.../controllers/status_controller_test.exs | 9 +++++---
.../mastodon_api/views/status_view_test.exs | 23 +++++++++++++++++++
4 files changed, 41 insertions(+), 4 deletions(-)
diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex
index 39241aa39..f4ee9b38c 100644
--- a/lib/pleroma/web/api_spec/schemas/status.ex
+++ b/lib/pleroma/web/api_spec/schemas/status.ex
@@ -198,6 +198,12 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
nullable: true,
description: "Quoted status (if any)"
},
+ quote_url: %Schema{
+ type: :string,
+ format: :uri,
+ nullable: true,
+ description: "URL of the quoted status"
+ },
local: %Schema{
type: :boolean,
description: "`true` if the post was made on the local instance"
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 5bde1ce04..06adfb221 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -316,7 +316,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
quote_post =
if quote_activity do
- quote_rendering_opts = Map.put(opts, :activity, quote_activity)
+ quote_rendering_opts = Map.merge(opts, %{activity: quote_activity, show_quote: false})
render("show.json", quote_rendering_opts)
else
nil
@@ -431,6 +431,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
context: object.data["context"],
in_reply_to_account_acct: reply_to_user && reply_to_user.nickname,
quote: quote_post,
+ quote_url: object.data["quoteUrl"],
content: %{"text/plain" => content_plaintext},
spoiler_text: %{"text/plain" => summary},
expires_at: expires_at,
@@ -666,6 +667,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
end
end
+ def get_quote(_activity, %{show_quote: false}) do
+ nil
+ end
+
def get_quote(activity, %{quoted_activities: quoted_activities}) do
object = Object.normalize(activity, fetch: false)
quoted_activities[object.data["quoteUrl"]]
diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
index ba49256b5..de3b52e26 100644
--- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
@@ -128,7 +128,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
test "posting a quote post", %{conn: conn} do
user = insert(:user)
- {:ok, %{id: activity_id}} = CommonAPI.post(user, %{status: "yolo"})
+ {:ok, %{id: activity_id} = activity} = CommonAPI.post(user, %{status: "yolo"})
+ %{data: %{"id" => quote_url}} = Object.normalize(activity)
conn =
conn
@@ -138,8 +139,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
"quote_id" => activity_id
})
- assert %{"id" => id, "pleroma" => %{"quote" => %{"id" => ^activity_id}}} =
- json_response_and_validate_schema(conn, 200)
+ assert %{
+ "id" => id,
+ "pleroma" => %{"quote" => %{"id" => ^activity_id}, "quote_url" => ^quote_url}
+ } = json_response_and_validate_schema(conn, 200)
assert Activity.get_by_id(id)
end
diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs
index b10b0f0b9..f50b02799 100644
--- a/test/pleroma/web/mastodon_api/views/status_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs
@@ -327,6 +327,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
context: object_data["context"],
in_reply_to_account_acct: nil,
quote: nil,
+ quote_url: nil,
content: %{"text/plain" => HTML.strip_tags(object_data["content"])},
spoiler_text: %{"text/plain" => HTML.strip_tags(object_data["summary"])},
expires_at: nil,
@@ -423,6 +424,28 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
assert status.in_reply_to_id == to_string(note.id)
end
+ test "a quote post" do
+ post = insert(:note_activity)
+ user = insert(:user)
+
+ {:ok, quote_post} = CommonAPI.post(user, %{status: "he", quote_id: post.id})
+ {:ok, quoted_quote_post} = CommonAPI.post(user, %{status: "yo", quote_id: quote_post.id})
+
+ status = StatusView.render("show.json", %{activity: quoted_quote_post})
+
+ assert status.pleroma.quote.id == to_string(quote_post.id)
+ assert status.pleroma.quote_url == Object.normalize(quote_post).data["id"]
+
+ # Quotes don't go more than one level deep
+ refute status.pleroma.quote.pleroma.quote
+ assert status.pleroma.quote.pleroma.quote_url == Object.normalize(post).data["id"]
+
+ # In an index
+ [status] = StatusView.render("index.json", %{activities: [quoted_quote_post], as: :activity})
+
+ assert status.pleroma.quote.id == to_string(quote_post.id)
+ end
+
test "contains mentions" do
user = insert(:user)
mentioned = insert(:user)
From 54a989793878c63900d2c6de7b4ffc8fbd8fe8c6 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sun, 23 Jan 2022 15:46:44 -0600
Subject: [PATCH 163/823] ActivityDraft: mention the OP of a quoted post
---
lib/pleroma/web/common_api/activity_draft.ex | 19 +++++++++++--------
test/pleroma/web/common_api_test.exs | 12 ++++++++++++
2 files changed, 23 insertions(+), 8 deletions(-)
diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex
index 375aabc91..588aba55e 100644
--- a/lib/pleroma/web/common_api/activity_draft.ex
+++ b/lib/pleroma/web/common_api/activity_draft.ex
@@ -137,11 +137,11 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
defp quote_post(%{params: %{quote_id: ""}} = draft), do: draft
defp quote_post(%{params: %{quote_id: id}} = draft) when is_binary(id) do
- %__MODULE__{draft | quote_post: Activity.get_by_id(id)}
- end
-
- defp quote_post(%{params: %{quote_id: %Activity{} = quote_post}} = draft) do
- %__MODULE__{draft | quote_post: quote_post}
+ with %Activity{actor: actor_ap_id} = activity <- Activity.get_by_id(id) do
+ %__MODULE__{draft | quote_post: activity, mentions: [actor_ap_id]}
+ else
+ _ -> draft
+ end
end
defp quote_post(draft), do: draft
@@ -178,12 +178,15 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
end
end
- defp content(draft) do
+ defp content(%{mentions: mentions} = draft) do
{content_html, mentioned_users, tags} = Utils.make_content_html(draft)
+ mentioned_ap_ids =
+ Enum.map(mentioned_users, fn {_, mentioned_user} -> mentioned_user.ap_id end)
+
mentions =
- mentioned_users
- |> Enum.map(fn {_, mentioned_user} -> mentioned_user.ap_id end)
+ mentions
+ |> Kernel.++(mentioned_ap_ids)
|> Utils.get_addressed_users(draft.params[:to])
%__MODULE__{draft | content_html: content_html, mentions: mentions, tags: tags}
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index 734e6dd82..051e770d7 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -807,6 +807,18 @@ defmodule Pleroma.Web.CommonAPITest do
quote_post = Object.normalize(quote_post)
assert quote_post.data["quoteUrl"] == quoted.data["id"]
+
+ # The OP is mentioned
+ assert quoted.data["actor"] in quote_post.data["to"]
+ end
+
+ test "quote posting with explicit addressing doesn't mention the OP" do
+ user = insert(:user)
+
+ {:ok, quoted} = CommonAPI.post(user, %{status: "Hello world"})
+ {:ok, quote_post} = CommonAPI.post(user, %{status: "nice post", quote_id: quoted.id, to: []})
+
+ assert Object.normalize(quote_post).data["to"] == [Pleroma.Constants.as_public()]
end
end
From 1f19dd76f66ca657ddfe79a51e59b8997a4c6321 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sun, 23 Jan 2022 16:03:46 -0600
Subject: [PATCH 164/823] ActivityDraft: mix format, defensive actor ID
---
lib/pleroma/web/common_api/activity_draft.ex | 16 ++++++++++------
test/pleroma/web/common_api_test.exs | 4 +++-
2 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex
index 588aba55e..95534f3cb 100644
--- a/lib/pleroma/web/common_api/activity_draft.ex
+++ b/lib/pleroma/web/common_api/activity_draft.ex
@@ -11,6 +11,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
alias Pleroma.Web.CommonAPI.Utils
import Pleroma.Web.Gettext
+ import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
defstruct valid?: true,
errors: [],
@@ -134,13 +135,16 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
defp in_reply_to(draft), do: draft
- defp quote_post(%{params: %{quote_id: ""}} = draft), do: draft
+ defp quote_post(%{params: %{quote_id: id}} = draft) when not_empty_string(id) do
+ case Activity.get_by_id(id) do
+ %Activity{actor: actor_ap_id} = activity when not_empty_string(actor_ap_id) ->
+ %__MODULE__{draft | quote_post: activity, mentions: [actor_ap_id]}
- defp quote_post(%{params: %{quote_id: id}} = draft) when is_binary(id) do
- with %Activity{actor: actor_ap_id} = activity <- Activity.get_by_id(id) do
- %__MODULE__{draft | quote_post: activity, mentions: [actor_ap_id]}
- else
- _ -> draft
+ %Activity{} = activity ->
+ %__MODULE__{draft | quote_post: activity}
+
+ _ ->
+ draft
end
end
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index 051e770d7..960d0cf16 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -816,7 +816,9 @@ defmodule Pleroma.Web.CommonAPITest do
user = insert(:user)
{:ok, quoted} = CommonAPI.post(user, %{status: "Hello world"})
- {:ok, quote_post} = CommonAPI.post(user, %{status: "nice post", quote_id: quoted.id, to: []})
+
+ {:ok, quote_post} =
+ CommonAPI.post(user, %{status: "nice post", quote_id: quoted.id, to: []})
assert Object.normalize(quote_post).data["to"] == [Pleroma.Constants.as_public()]
end
From 36a5578d2b16ba9f771fff55daafa85ec606a6be Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Mon, 24 Jan 2022 15:34:23 -0600
Subject: [PATCH 165/823] Scrubber.Default: allow span.quote-inline for quote
post compatibility
---
priv/scrubbers/default.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex
index d1215d2e0..56324a9fa 100644
--- a/priv/scrubbers/default.ex
+++ b/priv/scrubbers/default.ex
@@ -60,7 +60,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
Meta.allow_tag_with_these_attributes(:u, ["lang"])
Meta.allow_tag_with_these_attributes(:ul, ["lang"])
- Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card", "recipients-inline"])
+ Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card", "recipients-inline", "quote-inline"])
Meta.allow_tag_with_these_attributes(:span, ["lang"])
Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"])
From 57ef1d121101d785c043ef6aaf2d33bb9be3ec3b Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Mon, 24 Jan 2022 16:44:35 -0600
Subject: [PATCH 166/823] Add InlineQuotePolicy to force quote URLs inline
---
config/config.exs | 2 +
docs/configuration/cheatsheet.md | 4 ++
.../activity_pub/mrf/inline_quote_policy.ex | 53 ++++++++++++++++++
.../mrf/inline_quote_policy_test.exs | 56 +++++++++++++++++++
4 files changed, 115 insertions(+)
create mode 100644 lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
create mode 100644 test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
diff --git a/config/config.exs b/config/config.exs
index ebcbf8b49..56cc34db5 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -434,6 +434,8 @@ config :pleroma, :mrf_object_age,
config :pleroma, :mrf_follow_bot, follower_nickname: nil
+config :pleroma, :mrf_inline_quote, prefix: "RT"
+
config :pleroma, :rich_media,
enabled: true,
ignore_hosts: [],
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index f43cde114..32cc5811a 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -160,6 +160,7 @@ To add configuration to your config file, you can copy it from the base config.
* `Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy`: Drops follow requests from followbots. Users can still allow bots to follow them by first following the bot.
* `Pleroma.Web.ActivityPub.MRF.KeywordPolicy`: Rejects or removes from the federated timeline or replaces keywords. (See [`:mrf_keyword`](#mrf_keyword)).
* `Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent`: Forces every mentioned user to be reflected in the post content.
+ * `Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy`: Forces quote post URLs to be reflected in the message content inline.
* `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
* `transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.
@@ -267,6 +268,9 @@ Notes:
* `federated_timeline_removal_url`: A list of patterns which result in message with emojis whose URLs match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses. Each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html).
* `federated_timeline_removal_shortcode`: A list of patterns which result in message with emojis whose shortcodes match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses. Each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html).
+#### :mrf_inline_quote
+* `prefix`: Prefix before the link (default: `RT`)
+
### :activitypub
* `unfollow_blocked`: Whether blocks result in people getting unfollowed
* `outgoing_blocks`: Whether to federate blocks to other instances
diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
new file mode 100644
index 000000000..0f1dc9f42
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
@@ -0,0 +1,53 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do
+ @moduledoc "Force a quote line into the message content."
+ @behaviour Pleroma.Web.ActivityPub.MRF.Policy
+
+ defp build_inline_quote(prefix, url) do
+ " #{prefix}: #{url} "
+ end
+
+ defp filter_object(%{"quoteUrl" => quote_url} = object) do
+ content = object["content"] || ""
+
+ if content =~ quote_url do
+ object
+ else
+ prefix = Pleroma.Config.get([:mrf_inline_quote, :prefix])
+ content = content <> build_inline_quote(prefix, quote_url)
+ Map.put(object, "content", content)
+ end
+ end
+
+ @impl true
+ def filter(%{"object" => %{"quoteUrl" => _} = object} = activity) do
+ {:ok, Map.put(activity, "object", filter_object(object))}
+ end
+
+ @impl true
+ def filter(object), do: {:ok, object}
+
+ @impl true
+ def describe, do: {:ok, %{}}
+
+ @impl true
+ def config_description do
+ %{
+ key: :mrf_inline_quote,
+ related_policy: "Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy",
+ label: "MRF Inline Quote",
+ description: "Force quote post URLs inline",
+ children: [
+ %{
+ key: :prefix,
+ type: :string,
+ description: "Prefix before the link",
+ suggestions: ["RT", "QT", "RE", "RN"]
+ }
+ ]
+ }
+ end
+end
diff --git a/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs b/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
new file mode 100644
index 000000000..81dc06dda
--- /dev/null
+++ b/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
@@ -0,0 +1,56 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicyTest do
+ alias Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy
+ use Pleroma.DataCase
+
+ test "adds quote URL to post content" do
+ quote_url = "https://gleasonator.com/objects/1234"
+
+ activity = %{
+ "type" => "Create",
+ "actor" => "https://gleasonator.com/users/alex",
+ "object" => %{
+ "type" => "Note",
+ "content" => "Nice post
",
+ "quoteUrl" => quote_url
+ }
+ }
+
+ {:ok, %{"object" => %{"content" => filtered}}} = InlineQuotePolicy.filter(activity)
+
+ assert filtered ==
+ "Nice post
RT: https://gleasonator.com/objects/1234 "
+ end
+
+ test "ignores Misskey quote posts" do
+ object = File.read!("test/fixtures/quote_post/misskey_quote_post.json") |> Jason.decode!()
+
+ activity = %{
+ "type" => "Create",
+ "actor" => "https://misskey.io/users/7rkrarq81i",
+ "object" => object
+ }
+
+ {:ok, filtered} = InlineQuotePolicy.filter(activity)
+ assert filtered == activity
+ end
+
+ test "ignores Fedibird quote posts" do
+ object = File.read!("test/fixtures/quote_post/fedibird_quote_post.json") |> Jason.decode!()
+
+ # Normally the ObjectValidator will fix this before it reaches MRF
+ object = Map.put(object, "quoteUrl", object["quoteURL"])
+
+ activity = %{
+ "type" => "Create",
+ "actor" => "https://fedibird.com/users/noellabo",
+ "object" => object
+ }
+
+ {:ok, filtered} = InlineQuotePolicy.filter(activity)
+ assert filtered == activity
+ end
+end
From 59326247aa754991add9170e204257a8bf94c40f Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Wed, 26 Jan 2022 11:21:49 -0600
Subject: [PATCH 167/823] CommonAPI: disallow quoting private posts through the
API
---
lib/pleroma/web/common_api/activity_draft.ex | 15 ++++++++++-
.../web/common_api/activity_draft_test.exs | 26 +++++++++++++++++++
test/pleroma/web/common_api_test.exs | 14 ++++++++++
3 files changed, 54 insertions(+), 1 deletion(-)
create mode 100644 test/pleroma/web/common_api/activity_draft_test.exs
diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex
index 95534f3cb..d4875765c 100644
--- a/lib/pleroma/web/common_api/activity_draft.ex
+++ b/lib/pleroma/web/common_api/activity_draft.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
alias Pleroma.Conversation.Participation
alias Pleroma.Object
alias Pleroma.Web.ActivityPub.Builder
+ alias Pleroma.Web.ActivityPub.Visibility
alias Pleroma.Web.CommonAPI
alias Pleroma.Web.CommonAPI.Utils
@@ -57,6 +58,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
|> with_valid(&in_reply_to_conversation/1)
|> with_valid("e_post/1)
|> with_valid(&visibility/1)
+ |> with_valid("ing_visibility/1)
|> content()
|> with_valid(&to_and_cc/1)
|> with_valid(&context/1)
@@ -136,7 +138,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
defp in_reply_to(draft), do: draft
defp quote_post(%{params: %{quote_id: id}} = draft) when not_empty_string(id) do
- case Activity.get_by_id(id) do
+ case Activity.get_by_id_with_object(id) do
%Activity{actor: actor_ap_id} = activity when not_empty_string(actor_ap_id) ->
%__MODULE__{draft | quote_post: activity, mentions: [actor_ap_id]}
@@ -165,6 +167,17 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
end
end
+ defp quoting_visibility(%{quote_post: %Activity{}} = draft) do
+ with %Object{} = object <- Object.normalize(draft.quote_post, fetch: false),
+ visibility when visibility in ~w(public unlisted) <- Visibility.get_visibility(object) do
+ draft
+ else
+ _ -> add_error(draft, dgettext("errors", "Cannot quote private message"))
+ end
+ end
+
+ defp quoting_visibility(draft), do: draft
+
defp expires_at(draft) do
case CommonAPI.check_expiry_date(draft.params[:expires_in]) do
{:ok, expires_at} -> %__MODULE__{draft | expires_at: expires_at}
diff --git a/test/pleroma/web/common_api/activity_draft_test.exs b/test/pleroma/web/common_api/activity_draft_test.exs
new file mode 100644
index 000000000..8a09fc710
--- /dev/null
+++ b/test/pleroma/web/common_api/activity_draft_test.exs
@@ -0,0 +1,26 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.CommonAPI.ActivityDraftTest do
+ use Pleroma.DataCase
+
+ alias Pleroma.Web.CommonAPI
+ alias Pleroma.Web.CommonAPI.ActivityDraft
+
+ import Pleroma.Factory
+
+ test "create/2 with a quote post" do
+ user = insert(:user)
+
+ {:ok, direct} = CommonAPI.post(user, %{status: ".", visibility: "direct"})
+ {:ok, private} = CommonAPI.post(user, %{status: ".", visibility: "private"})
+ {:ok, unlisted} = CommonAPI.post(user, %{status: ".", visibility: "unlisted"})
+ {:ok, public} = CommonAPI.post(user, %{status: ".", visibility: "public"})
+
+ {:error, _} = ActivityDraft.create(user, %{status: "nice", quote_id: direct.id})
+ {:error, _} = ActivityDraft.create(user, %{status: "nice", quote_id: private.id})
+ {:ok, _} = ActivityDraft.create(user, %{status: "nice", quote_id: unlisted.id})
+ {:ok, _} = ActivityDraft.create(user, %{status: "nice", quote_id: public.id})
+ end
+end
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index 960d0cf16..c4eba8b9c 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -822,6 +822,20 @@ defmodule Pleroma.Web.CommonAPITest do
assert Object.normalize(quote_post).data["to"] == [Pleroma.Constants.as_public()]
end
+
+ test "quote posting visibility" do
+ user = insert(:user)
+
+ {:ok, direct} = CommonAPI.post(user, %{status: ".", visibility: "direct"})
+ {:ok, private} = CommonAPI.post(user, %{status: ".", visibility: "private"})
+ {:ok, unlisted} = CommonAPI.post(user, %{status: ".", visibility: "unlisted"})
+ {:ok, public} = CommonAPI.post(user, %{status: ".", visibility: "public"})
+
+ {:error, _} = CommonAPI.post(user, %{status: "nice", quote_id: direct.id})
+ {:error, _} = CommonAPI.post(user, %{status: "nice", quote_id: private.id})
+ {:ok, _} = CommonAPI.post(user, %{status: "nice", quote_id: unlisted.id})
+ {:ok, _} = CommonAPI.post(user, %{status: "nice", quote_id: public.id})
+ end
end
describe "reactions" do
From 6f11f11519f9c735f6b059c250f4bf01e09b305f Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Wed, 26 Jan 2022 11:49:31 -0600
Subject: [PATCH 168/823] StatusView: fix quote visibility
---
.../web/mastodon_api/views/status_view.ex | 2 +-
.../mastodon_api/views/status_view_test.exs | 41 +++++++++++++++++++
2 files changed, 42 insertions(+), 1 deletion(-)
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 06adfb221..7360d1093 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -315,7 +315,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
quote_activity = get_quote(activity, opts)
quote_post =
- if quote_activity do
+ if visible_for_user?(quote_activity, opts[:for]) do
quote_rendering_opts = Map.merge(opts, %{activity: quote_activity, show_quote: false})
render("show.json", quote_rendering_opts)
else
diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs
index f50b02799..f41ef580d 100644
--- a/test/pleroma/web/mastodon_api/views/status_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs
@@ -446,6 +446,47 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
assert status.pleroma.quote.id == to_string(quote_post.id)
end
+ test "quoted private post" do
+ user = insert(:user)
+
+ # Insert a private post
+ private = insert(:followers_only_note_activity, user: user)
+ private_object = Object.normalize(private)
+
+ # Create a public post quoting the private post
+ quote_private =
+ insert(:note_activity, note: insert(:note, data: %{"quoteUrl" => private_object.data["id"]}))
+
+ status = StatusView.render("show.json", %{activity: quote_private})
+
+ # The quote isn't rendered
+ refute status.pleroma.quote
+ assert status.pleroma.quote_url == private_object.data["id"]
+
+ # After following the user, the quote is rendered
+ follower = insert(:user)
+ CommonAPI.follow(follower, user)
+
+ status = StatusView.render("show.json", %{activity: quote_private, for: follower})
+ assert status.pleroma.quote.id == to_string(private.id)
+ end
+
+ test "quoted direct message" do
+ # Insert a direct message
+ direct = insert(:direct_note_activity)
+ direct_object = Object.normalize(direct)
+
+ # Create a public post quoting the direct message
+ quote_direct =
+ insert(:note_activity, note: insert(:note, data: %{"quoteUrl" => direct_object.data["id"]}))
+
+ status = StatusView.render("show.json", %{activity: quote_direct})
+
+ # The quote isn't rendered
+ refute status.pleroma.quote
+ assert status.pleroma.quote_url == direct_object.data["id"]
+ end
+
test "contains mentions" do
user = insert(:user)
mentioned = insert(:user)
From 74e0a4555f583a6962ad116bf6e54f06e42fe465 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Wed, 26 Jan 2022 11:52:50 -0600
Subject: [PATCH 169/823] StatusView: add `quote_visible` param
---
lib/pleroma/web/api_spec/schemas/status.ex | 4 ++++
lib/pleroma/web/mastodon_api/views/status_view.ex | 1 +
test/pleroma/web/mastodon_api/views/status_view_test.exs | 4 ++++
3 files changed, 9 insertions(+)
diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex
index f4ee9b38c..5d0eedb08 100644
--- a/lib/pleroma/web/api_spec/schemas/status.ex
+++ b/lib/pleroma/web/api_spec/schemas/status.ex
@@ -204,6 +204,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
nullable: true,
description: "URL of the quoted status"
},
+ quote_visible: %Schema{
+ type: :boolean,
+ description: "`true` if the quoted post is visible to the user"
+ },
local: %Schema{
type: :boolean,
description: "`true` if the post was made on the local instance"
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 7360d1093..2aa44b0f6 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -432,6 +432,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
in_reply_to_account_acct: reply_to_user && reply_to_user.nickname,
quote: quote_post,
quote_url: object.data["quoteUrl"],
+ quote_visible: visible_for_user?(quote_activity, opts[:for]),
content: %{"text/plain" => content_plaintext},
spoiler_text: %{"text/plain" => summary},
expires_at: expires_at,
diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs
index f41ef580d..ed0a87558 100644
--- a/test/pleroma/web/mastodon_api/views/status_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs
@@ -328,6 +328,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
in_reply_to_account_acct: nil,
quote: nil,
quote_url: nil,
+ quote_visible: false,
content: %{"text/plain" => HTML.strip_tags(object_data["content"])},
spoiler_text: %{"text/plain" => HTML.strip_tags(object_data["summary"])},
expires_at: nil,
@@ -462,6 +463,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
# The quote isn't rendered
refute status.pleroma.quote
assert status.pleroma.quote_url == private_object.data["id"]
+ refute status.pleroma.quote_visible
# After following the user, the quote is rendered
follower = insert(:user)
@@ -469,6 +471,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
status = StatusView.render("show.json", %{activity: quote_private, for: follower})
assert status.pleroma.quote.id == to_string(private.id)
+ assert status.pleroma.quote_visible
end
test "quoted direct message" do
@@ -485,6 +488,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
# The quote isn't rendered
refute status.pleroma.quote
assert status.pleroma.quote_url == direct_object.data["id"]
+ refute status.pleroma.quote_visible
end
test "contains mentions" do
From bee7e419597615ac6852942fe563166feba3fe73 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Thu, 27 Jan 2022 14:28:06 -0600
Subject: [PATCH 170/823] InlineQuotePolicy: don't add line breaks to markdown
posts
---
.../activity_pub/mrf/inline_quote_policy.ex | 12 ++++++++---
.../mrf/inline_quote_policy_test.exs | 21 ++++++++++++++++++-
2 files changed, 29 insertions(+), 4 deletions(-)
diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
index 0f1dc9f42..46013fc5e 100644
--- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
@@ -6,8 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do
@moduledoc "Force a quote line into the message content."
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
- defp build_inline_quote(prefix, url) do
- " #{prefix}: #{url} "
+ defp build_inline_quote(prefix, url, br) do
+ "#{String.duplicate(" ", br)}#{prefix}: #{url} "
end
defp filter_object(%{"quoteUrl" => quote_url} = object) do
@@ -17,7 +17,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do
object
else
prefix = Pleroma.Config.get([:mrf_inline_quote, :prefix])
- content = content <> build_inline_quote(prefix, quote_url)
+
+ inline_quote =
+ if String.ends_with?(content, "
"),
+ do: build_inline_quote(prefix, quote_url, 0),
+ else: build_inline_quote(prefix, quote_url, 2)
+
+ content = content <> inline_quote
Map.put(object, "content", content)
end
end
diff --git a/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs b/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
index 81dc06dda..8e75aaaab 100644
--- a/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
+++ b/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
@@ -9,6 +9,25 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicyTest do
test "adds quote URL to post content" do
quote_url = "https://gleasonator.com/objects/1234"
+ activity = %{
+ "type" => "Create",
+ "actor" => "https://gleasonator.com/users/alex",
+ "object" => %{
+ "type" => "Note",
+ "content" => "Nice post",
+ "quoteUrl" => quote_url
+ }
+ }
+
+ {:ok, %{"object" => %{"content" => filtered}}} = InlineQuotePolicy.filter(activity)
+
+ assert filtered ==
+ "Nice post RT: https://gleasonator.com/objects/1234 "
+ end
+
+ test "doesn't add line breaks to markdown posts" do
+ quote_url = "https://gleasonator.com/objects/1234"
+
activity = %{
"type" => "Create",
"actor" => "https://gleasonator.com/users/alex",
@@ -22,7 +41,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicyTest do
{:ok, %{"object" => %{"content" => filtered}}} = InlineQuotePolicy.filter(activity)
assert filtered ==
- "Nice post
RT: https://gleasonator.com/objects/1234 "
+ "Nice post
RT: https://gleasonator.com/objects/1234 "
end
test "ignores Misskey quote posts" do
From 93e4972b50f9bb0a07a7074fbab2aedbdc0cc4eb Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Thu, 27 Jan 2022 15:01:20 -0600
Subject: [PATCH 171/823] Add InlineQuotePolicy as a default MRF
---
config/config.exs | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/config/config.exs b/config/config.exs
index 56cc34db5..9149e925a 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -860,7 +860,11 @@ config :pleroma, :restrict_unauthenticated,
config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: false
config :pleroma, :mrf,
- policies: [Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy, Pleroma.Web.ActivityPub.MRF.TagPolicy],
+ policies: [
+ Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy,
+ Pleroma.Web.ActivityPub.MRF.TagPolicy,
+ Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy
+ ],
transparency: true,
transparency_exclusions: []
From cf8e4258830e3f362ab1e54238d622f6b2056502 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Fri, 28 Jan 2022 12:33:07 -0600
Subject: [PATCH 172/823] StatusView: return quote post inside a reblog
---
lib/pleroma/web/mastodon_api/views/status_view.ex | 10 ++++++----
.../web/mastodon_api/views/status_view_test.exs | 12 ++++++++++++
2 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 2aa44b0f6..ba4a8f3eb 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -668,13 +668,15 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
end
end
- def get_quote(_activity, %{show_quote: false}) do
- nil
- end
+ def get_quote(_activity, %{show_quote: false}), do: nil
def get_quote(activity, %{quoted_activities: quoted_activities}) do
object = Object.normalize(activity, fetch: false)
- quoted_activities[object.data["quoteUrl"]]
+
+ with nil <- quoted_activities[object.data["quoteUrl"]] do
+ # For when a quote post is inside an Announce
+ Activity.get_create_by_object_ap_id_with_object(object.data["quoteUrl"])
+ end
end
def get_quote(%{data: %{"object" => _object}} = activity, _) do
diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs
index ed0a87558..6d3a72970 100644
--- a/test/pleroma/web/mastodon_api/views/status_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs
@@ -491,6 +491,18 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
refute status.pleroma.quote_visible
end
+ test "repost of quote post" do
+ post = insert(:note_activity)
+ user = insert(:user)
+
+ {:ok, quote_post} = CommonAPI.post(user, %{status: "he", quote_id: post.id})
+ {:ok, repost} = CommonAPI.repeat(quote_post.id, user)
+
+ [status] = StatusView.render("index.json", %{activities: [repost], as: :activity})
+
+ assert status.reblog.pleroma.quote.id == to_string(post.id)
+ end
+
test "contains mentions" do
user = insert(:user)
mentioned = insert(:user)
From 3c8319fe9f7a7c793f8fdc347be2015190981e33 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Fri, 28 Jan 2022 14:06:32 -0600
Subject: [PATCH 173/823] Transmogrifier: federate quotes with _misskey_quote
field
---
lib/pleroma/web/activity_pub/transmogrifier.ex | 9 +++++++++
priv/static/schemas/litepub-0.1.jsonld | 2 ++
test/pleroma/web/activity_pub/transmogrifier_test.exs | 4 ++++
3 files changed, 15 insertions(+)
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index f5771e75e..163ae54fa 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -660,6 +660,14 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def set_reply_to_uri(obj), do: obj
+ # Misskey quotes
+ # Despite being underscored, it's potentially more reliable for interop.
+ def set_quote_url(%{"quoteUrl" => quote_url} = object) when is_binary(quote_url) do
+ Map.put(object, "_misskey_quote", quote_url)
+ end
+
+ def set_quote_url(obj), do: obj
+
@doc """
Serialized Mastodon-compatible `replies` collection containing _self-replies_.
Based on Mastodon's ActivityPub::NoteSerializer#replies.
@@ -714,6 +722,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
|> prepare_attachments
|> set_conversation
|> set_reply_to_uri
+ |> set_quote_url
|> set_replies
|> strip_internal_fields
|> strip_internal_tags
diff --git a/priv/static/schemas/litepub-0.1.jsonld b/priv/static/schemas/litepub-0.1.jsonld
index 5d8244a11..8559e744d 100644
--- a/priv/static/schemas/litepub-0.1.jsonld
+++ b/priv/static/schemas/litepub-0.1.jsonld
@@ -17,6 +17,7 @@
"ostatus": "http://ostatus.org#",
"schema": "http://schema.org#",
"toot": "http://joinmastodon.org/ns#",
+ "misskey": "https://misskey-hub.net/ns#",
"value": "schema:value",
"sensitive": "as:sensitive",
"litepub": "http://litepub.social/ns#",
@@ -27,6 +28,7 @@
"@type": "@id"
},
"quoteUrl": "as:quoteUrl",
+ "_misskey_quote": "misskey:_misskey_quote",
"oauthRegistrationEndpoint": {
"@id": "litepub:oauthRegistrationEndpoint",
"@type": "@id"
diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs
index 824398e38..8c7e0a4c9 100644
--- a/test/pleroma/web/activity_pub/transmogrifier_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs
@@ -382,7 +382,11 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
{:ok, modified} = Transmogrifier.prepare_outgoing(quote_post.data)
quoted_post = Object.normalize(quoted_post)
+
assert modified["object"]["quoteUrl"] == quoted_post.data["id"]
+
+ # Add Misskey's quote as a fallback
+ assert modified["object"]["_misskey_quote"] == quoted_post.data["id"]
end
end
From 817e308c0d9e42dfc39742c554e4c7a8da6f1c50 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Fri, 28 Jan 2022 15:55:52 -0600
Subject: [PATCH 174/823] Handle Fedibird's new quoteUri field
---
.../article_note_page_validator.ex | 8 ++-
.../web/activity_pub/transmogrifier.ex | 19 +++++--
priv/static/schemas/litepub-0.1.jsonld | 2 +
.../quote_post/fedibird_quote_uri.json | 54 +++++++++++++++++++
.../article_note_page_validator_test.exs | 10 ++++
.../web/activity_pub/transmogrifier_test.exs | 9 ++--
6 files changed, 92 insertions(+), 10 deletions(-)
create mode 100644 test/fixtures/quote_post/fedibird_quote_uri.json
diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
index 40bb67934..0b435b251 100644
--- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
@@ -78,7 +78,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
defp fix_quote_url(%{"quoteUrl" => _quote_url} = data), do: data
- # Fix for Fedibird
+ # Fedibird
+ # https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac
+ defp fix_quote_url(%{"quoteUri" => quote_url} = data) do
+ Map.put(data, "quoteUrl", quote_url)
+ end
+
+ # Old Fedibird (bug)
# https://github.com/fedibird/mastodon/issues/9
defp fix_quote_url(%{"quoteURL" => quote_url} = data) do
Map.put(data, "quoteUrl", quote_url)
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 163ae54fa..01e135fc1 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -180,7 +180,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
end
end
- # Fix for Fedibird
+ # Fedibird
+ # https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac
+ def fix_quote_url(%{"quoteUri" => quote_url} = object, options) do
+ object
+ |> Map.put("quoteUrl", quote_url)
+ |> fix_quote_url(options)
+ end
+
+ # Old Fedibird (bug)
# https://github.com/fedibird/mastodon/issues/9
def fix_quote_url(%{"quoteURL" => quote_url} = object, options) do
object
@@ -660,10 +668,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def set_reply_to_uri(obj), do: obj
- # Misskey quotes
- # Despite being underscored, it's potentially more reliable for interop.
def set_quote_url(%{"quoteUrl" => quote_url} = object) when is_binary(quote_url) do
- Map.put(object, "_misskey_quote", quote_url)
+ Map.merge(object, %{
+ # Fedibird quote
+ "quoteUri" => quote_url,
+ # Misskey quote
+ "_misskey_quote" => quote_url
+ })
end
def set_quote_url(obj), do: obj
diff --git a/priv/static/schemas/litepub-0.1.jsonld b/priv/static/schemas/litepub-0.1.jsonld
index 8559e744d..3d68e0714 100644
--- a/priv/static/schemas/litepub-0.1.jsonld
+++ b/priv/static/schemas/litepub-0.1.jsonld
@@ -18,6 +18,7 @@
"schema": "http://schema.org#",
"toot": "http://joinmastodon.org/ns#",
"misskey": "https://misskey-hub.net/ns#",
+ "fedibird": "http://fedibird.com/ns#",
"value": "schema:value",
"sensitive": "as:sensitive",
"litepub": "http://litepub.social/ns#",
@@ -28,6 +29,7 @@
"@type": "@id"
},
"quoteUrl": "as:quoteUrl",
+ "quoteUri": "fedibird:quoteUri",
"_misskey_quote": "misskey:_misskey_quote",
"oauthRegistrationEndpoint": {
"@id": "litepub:oauthRegistrationEndpoint",
diff --git a/test/fixtures/quote_post/fedibird_quote_uri.json b/test/fixtures/quote_post/fedibird_quote_uri.json
new file mode 100644
index 000000000..7c328fdb9
--- /dev/null
+++ b/test/fixtures/quote_post/fedibird_quote_uri.json
@@ -0,0 +1,54 @@
+{
+ "@context": [
+ "https://www.w3.org/ns/activitystreams",
+ {
+ "ostatus": "http://ostatus.org#",
+ "atomUri": "ostatus:atomUri",
+ "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+ "conversation": "ostatus:conversation",
+ "sensitive": "as:sensitive",
+ "toot": "http://joinmastodon.org/ns#",
+ "votersCount": "toot:votersCount",
+ "fedibird": "http://fedibird.com/ns#",
+ "quoteUri": "fedibird:quoteUri",
+ "expiry": "fedibird:expiry"
+ }
+ ],
+ "id": "https://fedibird.com/users/noellabo/statuses/107699335988346142",
+ "type": "Note",
+ "summary": null,
+ "inReplyTo": null,
+ "published": "2022-01-28T09:17:30Z",
+ "url": "https://fedibird.com/@noellabo/107699335988346142",
+ "attributedTo": "https://fedibird.com/users/noellabo",
+ "to": [
+ "https://www.w3.org/ns/activitystreams#Public"
+ ],
+ "cc": [
+ "https://fedibird.com/users/noellabo/followers"
+ ],
+ "sensitive": false,
+ "atomUri": "https://fedibird.com/users/noellabo/statuses/107699335988346142",
+ "inReplyToAtomUri": null,
+ "conversation": "tag:fedibird.com,2022-01-28:objectId=107699335988345290:objectType=Conversation",
+ "context": "https://fedibird.com/contexts/107699335988345290",
+ "quoteUri": "https://fedibird.com/users/yamako/statuses/107699333438289729",
+ "_misskey_quote": "https://fedibird.com/users/yamako/statuses/107699333438289729",
+ "_misskey_content": "美味しそう",
+ "content": "美味しそう QT: https:// fedibird.com/@yamako/107699333 438289729
",
+ "contentMap": {
+ "ja": "美味しそう QT: https:// fedibird.com/@yamako/107699333 438289729
"
+ },
+ "attachment": [],
+ "tag": [],
+ "replies": {
+ "id": "https://fedibird.com/users/noellabo/statuses/107699335988346142/replies",
+ "type": "Collection",
+ "first": {
+ "type": "CollectionPage",
+ "next": "https://fedibird.com/users/noellabo/statuses/107699335988346142/replies?only_other_accounts=true&page=true",
+ "partOf": "https://fedibird.com/users/noellabo/statuses/107699335988346142/replies",
+ "items": []
+ }
+ }
+}
diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
index dec2e28c9..a4ba38e6a 100644
--- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
+++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
@@ -127,6 +127,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest
assert cng.changes.quoteUrl == "https://misskey.io/notes/8vsn2izjwh"
end
+ test "Fedibird quote post with quoteUri field" do
+ insert(:user, ap_id: "https://fedibird.com/users/noellabo")
+
+ data = File.read!("test/fixtures/quote_post/fedibird_quote_uri.json") |> Jason.decode!()
+ cng = ArticleNotePageValidator.cast_and_validate(data)
+
+ assert cng.valid?
+ assert cng.changes.quoteUrl == "https://fedibird.com/users/yamako/statuses/107699333438289729"
+ end
+
test "Misskey quote post" do
insert(:user, ap_id: "https://misskey.io/users/7rkrarq81i")
diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs
index 8c7e0a4c9..1838f96bb 100644
--- a/test/pleroma/web/activity_pub/transmogrifier_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs
@@ -381,12 +381,11 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
{:ok, modified} = Transmogrifier.prepare_outgoing(quote_post.data)
- quoted_post = Object.normalize(quoted_post)
+ %{data: %{"id" => quote_id}} = Object.normalize(quoted_post)
- assert modified["object"]["quoteUrl"] == quoted_post.data["id"]
-
- # Add Misskey's quote as a fallback
- assert modified["object"]["_misskey_quote"] == quoted_post.data["id"]
+ assert modified["object"]["quoteUrl"] == quote_id
+ assert modified["object"]["quoteUri"] == quote_id
+ assert modified["object"]["_misskey_quote"] == quote_id
end
end
From 4075eecca0033e4487fa9d5d5bb75384597c3e79 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Fri, 28 Jan 2022 16:07:17 -0600
Subject: [PATCH 175/823] InlineQuotePolicy: improve the way Markdown quotes
are displayed by other software
---
.../web/activity_pub/mrf/inline_quote_policy.ex | 13 +++++++------
.../activity_pub/mrf/inline_quote_policy_test.exs | 4 ++--
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
index 46013fc5e..7de4935f2 100644
--- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
@@ -6,8 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do
@moduledoc "Force a quote line into the message content."
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
- defp build_inline_quote(prefix, url, br) do
- "#{String.duplicate(" ", br)}#{prefix}: #{url} "
+ defp build_inline_quote(prefix, url) do
+ " #{prefix}: #{url} "
end
defp filter_object(%{"quoteUrl" => quote_url} = object) do
@@ -18,12 +18,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do
else
prefix = Pleroma.Config.get([:mrf_inline_quote, :prefix])
- inline_quote =
+ content =
if String.ends_with?(content, ""),
- do: build_inline_quote(prefix, quote_url, 0),
- else: build_inline_quote(prefix, quote_url, 2)
+ do:
+ String.trim_trailing(content, "") <>
+ build_inline_quote(prefix, quote_url) <> "",
+ else: content <> build_inline_quote(prefix, quote_url)
- content = content <> inline_quote
Map.put(object, "content", content)
end
end
diff --git a/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs b/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
index 8e75aaaab..2291c1dac 100644
--- a/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
+++ b/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
@@ -22,7 +22,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicyTest do
{:ok, %{"object" => %{"content" => filtered}}} = InlineQuotePolicy.filter(activity)
assert filtered ==
- "Nice post RT: https://gleasonator.com/objects/1234 "
+ "Nice post RT: https://gleasonator.com/objects/1234 "
end
test "doesn't add line breaks to markdown posts" do
@@ -41,7 +41,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicyTest do
{:ok, %{"object" => %{"content" => filtered}}} = InlineQuotePolicy.filter(activity)
assert filtered ==
- "Nice post
RT: https://gleasonator.com/objects/1234 "
+ "Nice post RT: https://gleasonator.com/objects/1234
"
end
test "ignores Misskey quote posts" do
From 79fca39faf6d084eabb6be44a2263431943b8dd4 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Fri, 28 Jan 2022 17:53:19 -0600
Subject: [PATCH 176/823] Actually, don't send _misskey_quote anymore
---
lib/pleroma/web/activity_pub/transmogrifier.ex | 11 +++++------
priv/static/schemas/litepub-0.1.jsonld | 2 --
test/pleroma/web/activity_pub/transmogrifier_test.exs | 1 -
3 files changed, 5 insertions(+), 9 deletions(-)
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 01e135fc1..6c6cd712b 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -668,13 +668,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def set_reply_to_uri(obj), do: obj
+ @doc """
+ Fedibird compatibility
+ https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac
+ """
def set_quote_url(%{"quoteUrl" => quote_url} = object) when is_binary(quote_url) do
- Map.merge(object, %{
- # Fedibird quote
- "quoteUri" => quote_url,
- # Misskey quote
- "_misskey_quote" => quote_url
- })
+ Map.put(object, "quoteUri", quote_url)
end
def set_quote_url(obj), do: obj
diff --git a/priv/static/schemas/litepub-0.1.jsonld b/priv/static/schemas/litepub-0.1.jsonld
index 3d68e0714..b499a96f5 100644
--- a/priv/static/schemas/litepub-0.1.jsonld
+++ b/priv/static/schemas/litepub-0.1.jsonld
@@ -17,7 +17,6 @@
"ostatus": "http://ostatus.org#",
"schema": "http://schema.org#",
"toot": "http://joinmastodon.org/ns#",
- "misskey": "https://misskey-hub.net/ns#",
"fedibird": "http://fedibird.com/ns#",
"value": "schema:value",
"sensitive": "as:sensitive",
@@ -30,7 +29,6 @@
},
"quoteUrl": "as:quoteUrl",
"quoteUri": "fedibird:quoteUri",
- "_misskey_quote": "misskey:_misskey_quote",
"oauthRegistrationEndpoint": {
"@id": "litepub:oauthRegistrationEndpoint",
"@type": "@id"
diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs
index 1838f96bb..4a192cdc0 100644
--- a/test/pleroma/web/activity_pub/transmogrifier_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs
@@ -385,7 +385,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
assert modified["object"]["quoteUrl"] == quote_id
assert modified["object"]["quoteUri"] == quote_id
- assert modified["object"]["_misskey_quote"] == quote_id
end
end
From f9697e68c2b75a77575b9b7c89d08a5687bfd7b4 Mon Sep 17 00:00:00 2001
From: Alex Gleason
Date: Sun, 30 Jan 2022 10:57:29 -0600
Subject: [PATCH 177/823] InlineQuotePolicy: skip objects which already have an
.inline-quote span
---
.../activity_pub/mrf/inline_quote_policy.ex | 13 ++++-
.../quote_post/fedibird_quote_mismatched.json | 54 +++++++++++++++++++
.../mrf/inline_quote_policy_test.exs | 17 ++++++
3 files changed, 83 insertions(+), 1 deletion(-)
create mode 100644 test/fixtures/quote_post/fedibird_quote_mismatched.json
diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
index 7de4935f2..c78675caf 100644
--- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
@@ -10,10 +10,21 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do
" #{prefix}: #{url} "
end
+ defp has_inline_quote?(content, quote_url) do
+ cond do
+ # Does the quote URL exist in the content?
+ content =~ quote_url -> true
+ # Does the content already have a .quote-inline span?
+ content =~ "" -> true
+ # No inline quote found
+ true -> false
+ end
+ end
+
defp filter_object(%{"quoteUrl" => quote_url} = object) do
content = object["content"] || ""
- if content =~ quote_url do
+ if has_inline_quote?(content, quote_url) do
object
else
prefix = Pleroma.Config.get([:mrf_inline_quote, :prefix])
diff --git a/test/fixtures/quote_post/fedibird_quote_mismatched.json b/test/fixtures/quote_post/fedibird_quote_mismatched.json
new file mode 100644
index 000000000..8dee5daff
--- /dev/null
+++ b/test/fixtures/quote_post/fedibird_quote_mismatched.json
@@ -0,0 +1,54 @@
+{
+ "@context": [
+ "https://www.w3.org/ns/activitystreams",
+ {
+ "ostatus": "http://ostatus.org#",
+ "atomUri": "ostatus:atomUri",
+ "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+ "conversation": "ostatus:conversation",
+ "sensitive": "as:sensitive",
+ "toot": "http://joinmastodon.org/ns#",
+ "votersCount": "toot:votersCount",
+ "fedibird": "http://fedibird.com/ns#",
+ "quoteUri": "fedibird:quoteUri",
+ "expiry": "fedibird:expiry"
+ }
+ ],
+ "id": "https://fedibird.com/users/noellabo/statuses/107712183700212249",
+ "type": "Note",
+ "summary": null,
+ "inReplyTo": null,
+ "published": "2022-01-30T15:44:50Z",
+ "url": "https://fedibird.com/@noellabo/107712183700212249",
+ "attributedTo": "https://fedibird.com/users/noellabo",
+ "to": [
+ "https://www.w3.org/ns/activitystreams#Public"
+ ],
+ "cc": [
+ "https://fedibird.com/users/noellabo/followers"
+ ],
+ "sensitive": false,
+ "atomUri": "https://fedibird.com/users/noellabo/statuses/107712183700212249",
+ "inReplyToAtomUri": null,
+ "conversation": "tag:fedibird.com,2022-01-30:objectId=107712183700170473:objectType=Conversation",
+ "context": "https://fedibird.com/contexts/107712183700170473",
+ "quoteUri": "https://unnerv.jp/users/UN_NERV/statuses/107712176849067434",
+ "_misskey_quote": "https://unnerv.jp/users/UN_NERV/statuses/107712176849067434",
+ "_misskey_content": "揺れていたようだ",
+ "content": "揺れていたようだ QT: https:// unnerv.jp/@UN_NERV/10771217684 9067434
",
+ "contentMap": {
+ "ja": "揺れていたようだ QT: https:// unnerv.jp/@UN_NERV/10771217684 9067434
"
+ },
+ "attachment": [],
+ "tag": [],
+ "replies": {
+ "id": "https://fedibird.com/users/noellabo/statuses/107712183700212249/replies",
+ "type": "Collection",
+ "first": {
+ "type": "CollectionPage",
+ "next": "https://fedibird.com/users/noellabo/statuses/107712183700212249/replies?only_other_accounts=true&page=true",
+ "partOf": "https://fedibird.com/users/noellabo/statuses/107712183700212249/replies",
+ "items": []
+ }
+ }
+}
diff --git a/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs b/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
index 2291c1dac..44ee91d4b 100644
--- a/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
+++ b/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
@@ -72,4 +72,21 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicyTest do
{:ok, filtered} = InlineQuotePolicy.filter(activity)
assert filtered == activity
end
+
+ test "skips objects which already have an .inline-quote span" do
+ object =
+ File.read!("test/fixtures/quote_post/fedibird_quote_mismatched.json") |> Jason.decode!()
+
+ # Normally the ObjectValidator will fix this before it reaches MRF
+ object = Map.put(object, "quoteUrl", object["quoteUri"])
+
+ activity = %{
+ "type" => "Create",
+ "actor" => "https://fedibird.com/users/noellabo",
+ "object" => object
+ }
+
+ {:ok, filtered} = InlineQuotePolicy.filter(activity)
+ assert filtered == activity
+ end
end
From b0a7e795e799d3c8d750ab909657ec7b3d0bfd58 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Mon, 10 Jul 2023 17:57:09 -0400
Subject: [PATCH 178/823] Unify logic for normalizing quoteUri
---
.../article_note_page_validator.ex | 23 +---------
.../object_validators/common_fixes.ex | 21 ++++++++++
.../web/activity_pub/transmogrifier.ex | 42 ++++++-------------
priv/scrubbers/default.ex | 7 +++-
4 files changed, 40 insertions(+), 53 deletions(-)
diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
index 0b435b251..1b5b2e8fb 100644
--- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
@@ -76,27 +76,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
def fix_attachments(data), do: data
- defp fix_quote_url(%{"quoteUrl" => _quote_url} = data), do: data
-
- # Fedibird
- # https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac
- defp fix_quote_url(%{"quoteUri" => quote_url} = data) do
- Map.put(data, "quoteUrl", quote_url)
- end
-
- # Old Fedibird (bug)
- # https://github.com/fedibird/mastodon/issues/9
- defp fix_quote_url(%{"quoteURL" => quote_url} = data) do
- Map.put(data, "quoteUrl", quote_url)
- end
-
- # Misskey fallback
- defp fix_quote_url(%{"_misskey_quote" => quote_url} = data) do
- Map.put(data, "quoteUrl", quote_url)
- end
-
- defp fix_quote_url(data), do: data
-
defp fix(data) do
data
|> CommonFixes.fix_actor()
@@ -105,7 +84,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
|> fix_tag()
|> fix_replies()
|> fix_attachments()
- |> fix_quote_url()
+ |> CommonFixes.fix_quote_url()
|> Transmogrifier.fix_emoji()
|> Transmogrifier.fix_content_map()
end
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
index add46d561..cc2ad9116 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
@@ -76,4 +76,25 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
Map.put(data, "to", to)
end
+
+ def fix_quote_url(%{"quoteUrl" => _quote_url} = data), do: data
+
+ # Fedibird
+ # https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac
+ def fix_quote_url(%{"quoteUri" => quote_url} = data) do
+ Map.put(data, "quoteUrl", quote_url)
+ end
+
+ # Old Fedibird (bug)
+ # https://github.com/fedibird/mastodon/issues/9
+ def fix_quote_url(%{"quoteURL" => quote_url} = data) do
+ Map.put(data, "quoteUrl", quote_url)
+ end
+
+ # Misskey fallback
+ def fix_quote_url(%{"_misskey_quote" => quote_url} = data) do
+ Map.put(data, "quoteUrl", quote_url)
+ end
+
+ def fix_quote_url(data), do: data
end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 6c6cd712b..86d3ac60f 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -166,45 +166,27 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def fix_in_reply_to(object, _options), do: object
- def fix_quote_url(object, options \\ [])
+ def fix_quote_url_and_maybe_fetch(object, options \\ []) do
+ quote_url =
+ case Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes.fix_quote_url(object) do
+ %{"quoteUrl" => quote_url} -> quote_url
+ _ -> nil
+ end
- def fix_quote_url(%{"quoteUrl" => quote_url} = object, options)
- when not is_nil(quote_url) do
- with {:ok, quoted_object} <- get_obj_helper(quote_url, options),
+ with {:quoting?, true} <- {:quoting?, not is_nil(quote_url)},
+ {:ok, quoted_object} <- get_obj_helper(quote_url, options),
%Activity{} <- Activity.get_create_by_object_ap_id(quoted_object.data["id"]) do
Map.put(object, "quoteUrl", quoted_object.data["id"])
else
+ {:quoting?, _} ->
+ object
+
e ->
Logger.warn("Couldn't fetch #{inspect(quote_url)}, error: #{inspect(e)}")
object
end
end
- # Fedibird
- # https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac
- def fix_quote_url(%{"quoteUri" => quote_url} = object, options) do
- object
- |> Map.put("quoteUrl", quote_url)
- |> fix_quote_url(options)
- end
-
- # Old Fedibird (bug)
- # https://github.com/fedibird/mastodon/issues/9
- def fix_quote_url(%{"quoteURL" => quote_url} = object, options) do
- object
- |> Map.put("quoteUrl", quote_url)
- |> fix_quote_url(options)
- end
-
- # Misskey fallback
- def fix_quote_url(%{"_misskey_quote" => quote_url} = object, options) do
- object
- |> Map.put("quoteUrl", quote_url)
- |> fix_quote_url(options)
- end
-
- def fix_quote_url(object, _options), do: object
-
defp prepare_in_reply_to(in_reply_to) do
cond do
is_bitstring(in_reply_to) ->
@@ -493,7 +475,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
|> strip_internal_fields()
|> fix_type(fetch_options)
|> fix_in_reply_to(fetch_options)
- |> fix_quote_url(fetch_options)
+ |> fix_quote_url_and_maybe_fetch(fetch_options)
data = Map.put(data, "object", object)
options = Keyword.put(options, :local, false)
diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex
index 56324a9fa..4e7950547 100644
--- a/priv/scrubbers/default.ex
+++ b/priv/scrubbers/default.ex
@@ -60,7 +60,12 @@ defmodule Pleroma.HTML.Scrubber.Default do
Meta.allow_tag_with_these_attributes(:u, ["lang"])
Meta.allow_tag_with_these_attributes(:ul, ["lang"])
- Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card", "recipients-inline", "quote-inline"])
+ Meta.allow_tag_with_this_attribute_values(:span, "class", [
+ "h-card",
+ "recipients-inline",
+ "quote-inline"
+ ])
+
Meta.allow_tag_with_these_attributes(:span, ["lang"])
Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"])
From 9bcec87aba5ce4de6b61b5a95d6832da9dfa0fd8 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Mon, 10 Jul 2023 18:27:23 -0400
Subject: [PATCH 179/823] Allow local quote and private self-quote
---
lib/pleroma/web/common_api/activity_draft.ex | 14 +++++++++++++-
.../pleroma/web/common_api/activity_draft_test.exs | 9 ++++++++-
2 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex
index d4875765c..32921aa5a 100644
--- a/lib/pleroma/web/common_api/activity_draft.ex
+++ b/lib/pleroma/web/common_api/activity_draft.ex
@@ -167,9 +167,21 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
end
end
+ defp can_quote?(_draft, _object, visibility) when visibility in ~w(public unlisted local) do
+ true
+ end
+
+ defp can_quote?(draft, object, "private") do
+ draft.user.ap_id == object.data["actor"]
+ end
+
+ defp can_quote?(_, _, _) do
+ false
+ end
+
defp quoting_visibility(%{quote_post: %Activity{}} = draft) do
with %Object{} = object <- Object.normalize(draft.quote_post, fetch: false),
- visibility when visibility in ~w(public unlisted) <- Visibility.get_visibility(object) do
+ true <- can_quote?(draft, object, Visibility.get_visibility(object)) do
draft
else
_ -> add_error(draft, dgettext("errors", "Cannot quote private message"))
diff --git a/test/pleroma/web/common_api/activity_draft_test.exs b/test/pleroma/web/common_api/activity_draft_test.exs
index 8a09fc710..02bc6cf3b 100644
--- a/test/pleroma/web/common_api/activity_draft_test.exs
+++ b/test/pleroma/web/common_api/activity_draft_test.exs
@@ -12,15 +12,22 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraftTest do
test "create/2 with a quote post" do
user = insert(:user)
+ another_user = insert(:user)
{:ok, direct} = CommonAPI.post(user, %{status: ".", visibility: "direct"})
{:ok, private} = CommonAPI.post(user, %{status: ".", visibility: "private"})
{:ok, unlisted} = CommonAPI.post(user, %{status: ".", visibility: "unlisted"})
+ {:ok, local} = CommonAPI.post(user, %{status: ".", visibility: "local"})
{:ok, public} = CommonAPI.post(user, %{status: ".", visibility: "public"})
{:error, _} = ActivityDraft.create(user, %{status: "nice", quote_id: direct.id})
- {:error, _} = ActivityDraft.create(user, %{status: "nice", quote_id: private.id})
+ {:ok, _} = ActivityDraft.create(user, %{status: "nice", quote_id: private.id})
+ {:error, _} = ActivityDraft.create(another_user, %{status: "nice", quote_id: private.id})
{:ok, _} = ActivityDraft.create(user, %{status: "nice", quote_id: unlisted.id})
+ {:ok, _} = ActivityDraft.create(another_user, %{status: "nice", quote_id: unlisted.id})
+ {:ok, _} = ActivityDraft.create(user, %{status: "nice", quote_id: local.id})
+ {:ok, _} = ActivityDraft.create(another_user, %{status: "nice", quote_id: local.id})
{:ok, _} = ActivityDraft.create(user, %{status: "nice", quote_id: public.id})
+ {:ok, _} = ActivityDraft.create(another_user, %{status: "nice", quote_id: public.id})
end
end
From d244c9d2984d21887f50737597fc03d2d0dd1601 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Mon, 10 Jul 2023 18:28:13 -0400
Subject: [PATCH 180/823] Add changelog
---
changelog.d/quote.add | 1 +
1 file changed, 1 insertion(+)
create mode 100644 changelog.d/quote.add
diff --git a/changelog.d/quote.add b/changelog.d/quote.add
new file mode 100644
index 000000000..1c368ae75
--- /dev/null
+++ b/changelog.d/quote.add
@@ -0,0 +1 @@
+Implement quotes
From 762794eed9e6fc8a03d1416e2a6e080b00df9e10 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Mon, 10 Jul 2023 19:43:18 -0400
Subject: [PATCH 181/823] Fix CommonAPITest
---
test/pleroma/web/common_api_test.exs | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index c4eba8b9c..70a5b6fed 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -825,16 +825,23 @@ defmodule Pleroma.Web.CommonAPITest do
test "quote posting visibility" do
user = insert(:user)
+ another_user = insert(:user)
{:ok, direct} = CommonAPI.post(user, %{status: ".", visibility: "direct"})
{:ok, private} = CommonAPI.post(user, %{status: ".", visibility: "private"})
{:ok, unlisted} = CommonAPI.post(user, %{status: ".", visibility: "unlisted"})
+ {:ok, local} = CommonAPI.post(user, %{status: ".", visibility: "local"})
{:ok, public} = CommonAPI.post(user, %{status: ".", visibility: "public"})
{:error, _} = CommonAPI.post(user, %{status: "nice", quote_id: direct.id})
- {:error, _} = CommonAPI.post(user, %{status: "nice", quote_id: private.id})
+ {:ok, _} = CommonAPI.post(user, %{status: "nice", quote_id: private.id})
+ {:error, _} = CommonAPI.post(another_user, %{status: "nice", quote_id: private.id})
{:ok, _} = CommonAPI.post(user, %{status: "nice", quote_id: unlisted.id})
+ {:ok, _} = CommonAPI.post(another_user, %{status: "nice", quote_id: unlisted.id})
+ {:ok, _} = CommonAPI.post(user, %{status: "nice", quote_id: local.id})
+ {:ok, _} = CommonAPI.post(another_user, %{status: "nice", quote_id: local.id})
{:ok, _} = CommonAPI.post(user, %{status: "nice", quote_id: public.id})
+ {:ok, _} = CommonAPI.post(another_user, %{status: "nice", quote_id: public.id})
end
end
From 163e5637335f9454688d3cc83530f82fc640a5b9 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Wed, 12 Jul 2023 09:30:43 -0400
Subject: [PATCH 182/823] Allow more flexibility in InlineQuotePolicy
---
config/config.exs | 2 +-
config/description.exs | 18 ++++++++++++++
docs/configuration/cheatsheet.md | 2 +-
.../activity_pub/mrf/inline_quote_policy.ex | 12 ++++++----
priv/scrubbers/default.ex | 1 +
.../mrf/inline_quote_policy_test.exs | 24 +++++++++++++++++--
6 files changed, 50 insertions(+), 9 deletions(-)
diff --git a/config/config.exs b/config/config.exs
index 9149e925a..e8ae31542 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -434,7 +434,7 @@ config :pleroma, :mrf_object_age,
config :pleroma, :mrf_follow_bot, follower_nickname: nil
-config :pleroma, :mrf_inline_quote, prefix: "RT"
+config :pleroma, :mrf_inline_quote, template: "RT: {url}"
config :pleroma, :rich_media,
enabled: true,
diff --git a/config/description.exs b/config/description.exs
index d18649ae8..079d187d5 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -2994,6 +2994,24 @@ config :pleroma, :config_description, [
}
]
},
+ %{
+ group: :pleroma,
+ key: :mrf_inline_quote,
+ tab: :mrf,
+ related_policy: "Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy",
+ label: "MRF Inline Quote Policy",
+ type: :group,
+ description: "Force quote url to appear in post content.",
+ children: [
+ %{
+ key: :template,
+ type: :string,
+ description:
+ "The template to append to the post. `{url}` will be replaced with the actual link to the quoted post.",
+ suggestions: ["RT: {url}"]
+ }
+ ]
+ },
%{
group: :pleroma,
key: :modules,
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 32cc5811a..a17f8735a 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -269,7 +269,7 @@ Notes:
* `federated_timeline_removal_shortcode`: A list of patterns which result in message with emojis whose shortcodes match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses. Each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html).
#### :mrf_inline_quote
-* `prefix`: Prefix before the link (default: `RT`)
+* `template`: The template to append to the post. `{url}` will be replaced with the actual link to the quoted post. Default: `RT: {url}`
### :activitypub
* `unfollow_blocked`: Whether blocks result in people getting unfollowed
diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
index c78675caf..a0eefefc0 100644
--- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
@@ -6,8 +6,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do
@moduledoc "Force a quote line into the message content."
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
- defp build_inline_quote(prefix, url) do
- " #{prefix}: #{url} "
+ defp build_inline_quote(template, url) do
+ quote_line = String.replace(template, "{url}", "#{url} ")
+
+ " #{quote_line} "
end
defp has_inline_quote?(content, quote_url) do
@@ -27,14 +29,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do
if has_inline_quote?(content, quote_url) do
object
else
- prefix = Pleroma.Config.get([:mrf_inline_quote, :prefix])
+ template = Pleroma.Config.get([:mrf_inline_quote, :template])
content =
if String.ends_with?(content, ""),
do:
String.trim_trailing(content, "") <>
- build_inline_quote(prefix, quote_url) <> "",
- else: content <> build_inline_quote(prefix, quote_url)
+ build_inline_quote(template, quote_url) <> "",
+ else: content <> build_inline_quote(template, quote_url)
Map.put(object, "content", content)
end
diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex
index 4e7950547..24a76263b 100644
--- a/priv/scrubbers/default.ex
+++ b/priv/scrubbers/default.ex
@@ -38,6 +38,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
Meta.allow_tag_with_these_attributes(:abbr, ["title", "lang"])
Meta.allow_tag_with_these_attributes(:b, ["lang"])
+ Meta.allow_tag_with_these_attributes(:bdi, [])
Meta.allow_tag_with_these_attributes(:blockquote, ["lang"])
Meta.allow_tag_with_these_attributes(:br, ["lang"])
Meta.allow_tag_with_these_attributes(:code, ["lang"])
diff --git a/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs b/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
index 44ee91d4b..d5762766f 100644
--- a/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
+++ b/test/pleroma/web/activity_pub/mrf/inline_quote_policy_test.exs
@@ -22,7 +22,27 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicyTest do
{:ok, %{"object" => %{"content" => filtered}}} = InlineQuotePolicy.filter(activity)
assert filtered ==
- "Nice post RT: https://gleasonator.com/objects/1234 "
+ "Nice postRT: https://gleasonator.com/objects/1234 "
+ end
+
+ test "adds quote URL to post content, custom template" do
+ clear_config([:mrf_inline_quote, :template], "{url}'s quoting")
+ quote_url = "https://gleasonator.com/objects/1234"
+
+ activity = %{
+ "type" => "Create",
+ "actor" => "https://gleasonator.com/users/alex",
+ "object" => %{
+ "type" => "Note",
+ "content" => "Nice post",
+ "quoteUrl" => quote_url
+ }
+ }
+
+ {:ok, %{"object" => %{"content" => filtered}}} = InlineQuotePolicy.filter(activity)
+
+ assert filtered ==
+ "Nice posthttps://gleasonator.com/objects/1234 's quoting "
end
test "doesn't add line breaks to markdown posts" do
@@ -41,7 +61,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicyTest do
{:ok, %{"object" => %{"content" => filtered}}} = InlineQuotePolicy.filter(activity)
assert filtered ==
- "Nice post RT: https://gleasonator.com/objects/1234
"
+ "Nice postRT: https://gleasonator.com/objects/1234
"
end
test "ignores Misskey quote posts" do
From e9cd004ba1b904d92b1c07446bbf03dc070cce6a Mon Sep 17 00:00:00 2001
From: tusooa
Date: Wed, 12 Jul 2023 11:09:10 -0400
Subject: [PATCH 183/823] Parse object link as quoteUrl
---
lib/pleroma/constants.ex | 7 +++++
.../audio_image_video_validator.ex | 1 +
.../object_validators/common_fixes.ex | 27 +++++++++++++++++++
.../object_validators/question_validator.ex | 1 +
.../quote_post/fep-e232-tag-example.json | 17 ++++++++++++
.../article_note_page_validator_test.exs | 12 +++++++++
6 files changed, 65 insertions(+)
create mode 100644 test/fixtures/quote_post/fep-e232-tag-example.json
diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex
index 6befc6897..9d764ec25 100644
--- a/lib/pleroma/constants.ex
+++ b/lib/pleroma/constants.ex
@@ -83,4 +83,11 @@ defmodule Pleroma.Constants do
)
const(upload_object_types, do: ["Document", "Image"])
+
+ const(activity_json_mime_types,
+ do: [
+ "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"",
+ "application/activity+json"
+ ]
+ )
end
diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_image_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_image_video_validator.ex
index 79ff76104..65ac6bb93 100644
--- a/lib/pleroma/web/activity_pub/object_validators/audio_image_video_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/audio_image_video_validator.ex
@@ -99,6 +99,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioImageVideoValidator do
data
|> CommonFixes.fix_actor()
|> CommonFixes.fix_object_defaults()
+ |> CommonFixes.fix_quote_url()
|> Transmogrifier.fix_emoji()
|> fix_url()
|> fix_content()
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
index cc2ad9116..65b8d9a2c 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
@@ -10,6 +10,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.Utils
+ require Pleroma.Constants
+
def cast_and_filter_recipients(message, field, follower_collection, field_fallback \\ []) do
{:ok, data} = ObjectValidators.Recipients.cast(message[field] || field_fallback)
@@ -96,5 +98,30 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
Map.put(data, "quoteUrl", quote_url)
end
+ def fix_quote_url(%{"tag" => [_ | _] = tags} = data) do
+ tag = Enum.find(tags, &is_object_link_tag/1)
+
+ if not is_nil(tag) do
+ data
+ |> Map.put("quoteUrl", tag["href"])
+ else
+ data
+ end
+ end
+
def fix_quote_url(data), do: data
+
+ # https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md
+ defp is_object_link_tag(
+ %{
+ "type" => "Link",
+ "mediaType" => media_type,
+ "href" => href
+ } = tag
+ )
+ when media_type in Pleroma.Constants.activity_json_mime_types() and is_binary(href) do
+ true
+ end
+
+ defp is_object_link_tag(_), do: false
end
diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
index ce3305142..621085e6c 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -62,6 +62,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
data
|> CommonFixes.fix_actor()
|> CommonFixes.fix_object_defaults()
+ |> CommonFixes.fix_quote_url()
|> Transmogrifier.fix_emoji()
|> fix_closed()
end
diff --git a/test/fixtures/quote_post/fep-e232-tag-example.json b/test/fixtures/quote_post/fep-e232-tag-example.json
new file mode 100644
index 000000000..23c7fb5ac
--- /dev/null
+++ b/test/fixtures/quote_post/fep-e232-tag-example.json
@@ -0,0 +1,17 @@
+{
+ "@context": "https://www.w3.org/ns/activitystreams",
+ "type": "Note",
+ "content": "This is a quote: RE: https://server.example/objects/123",
+ "tag": [
+ {
+ "type": "Link",
+ "mediaType": "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"",
+ "href": "https://server.example/objects/123",
+ "name": "RE: https://server.example/objects/123"
+ }
+ ],
+ "id": "https://server.example/objects/1",
+ "to": "https://server.example/users/1",
+ "attributedTo": "https://server.example/users/1",
+ "actor": "https://server.example/users/1"
+}
diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
index a4ba38e6a..73141cac1 100644
--- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
+++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
@@ -146,4 +146,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest
assert cng.valid?
assert cng.changes.quoteUrl == "https://misskey.io/notes/8vs6wxufd0"
end
+
+ test "Parse tag as quote" do
+ # https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md
+
+ insert(:user, ap_id: "https://server.example/users/1")
+
+ data = File.read!("test/fixtures/quote_post/fep-e232-tag-example.json") |> Jason.decode!()
+ cng = ArticleNotePageValidator.cast_and_validate(data)
+
+ assert cng.valid?
+ assert cng.changes.quoteUrl == "https://server.example/objects/123"
+ end
end
From 479a6f11dbbba0c945c08883956ffab198f91688 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Wed, 12 Jul 2023 14:08:24 -0400
Subject: [PATCH 184/823] Keep incoming Link tag
---
.../object_validators/tag_validator.ex | 14 +++++++++++++-
.../article_note_page_validator_test.exs | 7 +++++++
2 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex
index cfd510c19..47cf7b415 100644
--- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex
@@ -9,15 +9,20 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do
import Ecto.Changeset
+ require Pleroma.Constants
+
@primary_key false
embedded_schema do
# Common
field(:type, :string)
field(:name, :string)
- # Mention, Hashtag
+ # Mention, Hashtag, Link
field(:href, ObjectValidators.Uri)
+ # Link
+ field(:mediaType, :string)
+
# Emoji
embeds_one :icon, IconObjectValidator, primary_key: false do
field(:type, :string)
@@ -68,6 +73,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do
|> validate_required([:type, :name, :icon])
end
+ def changeset(struct, %{"type" => "Link"} = data) do
+ struct
+ |> cast(data, [:type, :name, :mediaType, :href])
+ |> validate_inclusion(:mediaType, Pleroma.Constants.activity_json_mime_types())
+ |> validate_required([:type, :href, :mediaType])
+ end
+
def changeset(struct, %{"type" => _} = data) do
struct
|> cast(data, [])
diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
index 73141cac1..4703c3801 100644
--- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
+++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
@@ -157,5 +157,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest
assert cng.valid?
assert cng.changes.quoteUrl == "https://server.example/objects/123"
+
+ assert Enum.at(cng.changes.tag, 0).changes == %{
+ type: "Link",
+ mediaType: "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"",
+ href: "https://server.example/objects/123",
+ name: "RE: https://server.example/objects/123"
+ }
end
end
From e349e92a441840bbbdbf13cacd307e65f85a38ff Mon Sep 17 00:00:00 2001
From: tusooa
Date: Wed, 12 Jul 2023 14:27:29 -0400
Subject: [PATCH 185/823] Add mrf to force link tag of quoting posts
---
docs/configuration/cheatsheet.md | 1 +
lib/pleroma/constants.ex | 4 +
.../mrf/quote_to_link_tag_policy.ex | 49 +++++++++++++
.../object_validators/common_fixes.ex | 16 ++--
.../mrf/quote_to_link_tag_policy_test.exs | 73 +++++++++++++++++++
5 files changed, 134 insertions(+), 9 deletions(-)
create mode 100644 lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex
create mode 100644 test/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy_test.exs
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index a17f8735a..a4cae4dbb 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -161,6 +161,7 @@ To add configuration to your config file, you can copy it from the base config.
* `Pleroma.Web.ActivityPub.MRF.KeywordPolicy`: Rejects or removes from the federated timeline or replaces keywords. (See [`:mrf_keyword`](#mrf_keyword)).
* `Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent`: Forces every mentioned user to be reflected in the post content.
* `Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy`: Forces quote post URLs to be reflected in the message content inline.
+ * `Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicy`: Force a Link tag for posts quoting another post. (may break outgoing federation of quote posts with older Pleroma versions)
* `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
* `transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.
diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex
index 9d764ec25..ed60bcc37 100644
--- a/lib/pleroma/constants.ex
+++ b/lib/pleroma/constants.ex
@@ -84,6 +84,10 @@ defmodule Pleroma.Constants do
const(upload_object_types, do: ["Document", "Image"])
+ const(activity_json_canonical_mime_type,
+ do: "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""
+ )
+
const(activity_json_mime_types,
do: [
"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"",
diff --git a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex
new file mode 100644
index 000000000..f1c573d1b
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex
@@ -0,0 +1,49 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2023 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicy do
+ @moduledoc "Force a Link tag for posts quoting another post. (may break outgoing federation of quote posts with older Pleroma versions)"
+ @behaviour Pleroma.Web.ActivityPub.MRF.Policy
+
+ alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
+
+ require Pleroma.Constants
+
+ @impl Pleroma.Web.ActivityPub.MRF.Policy
+ def filter(%{"object" => %{"quoteUrl" => _} = object} = activity) do
+ {:ok, Map.put(activity, "object", filter_object(object))}
+ end
+
+ @impl Pleroma.Web.ActivityPub.MRF.Policy
+ def filter(object), do: {:ok, object}
+
+ @impl Pleroma.Web.ActivityPub.MRF.Policy
+ def describe, do: {:ok, %{}}
+
+ @impl Pleroma.Web.ActivityPub.MRF.Policy
+ def history_awareness, do: :auto
+
+ defp filter_object(%{"quoteUrl" => quote_url} = object) do
+ tags = object["tag"] || []
+
+ if Enum.any?(tags, fn tag ->
+ CommonFixes.is_object_link_tag(tag) and tag["href"] == quote_url
+ end) do
+ object
+ else
+ object
+ |> Map.put(
+ "tag",
+ tags ++
+ [
+ %{
+ "type" => "Link",
+ "mediaType" => Pleroma.Constants.activity_json_canonical_mime_type(),
+ "href" => quote_url
+ }
+ ]
+ )
+ end
+ end
+end
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
index 65b8d9a2c..4d9be0bdd 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
@@ -112,16 +112,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
def fix_quote_url(data), do: data
# https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md
- defp is_object_link_tag(
- %{
- "type" => "Link",
- "mediaType" => media_type,
- "href" => href
- } = tag
- )
- when media_type in Pleroma.Constants.activity_json_mime_types() and is_binary(href) do
+ def is_object_link_tag(%{
+ "type" => "Link",
+ "mediaType" => media_type,
+ "href" => href
+ })
+ when media_type in Pleroma.Constants.activity_json_mime_types() and is_binary(href) do
true
end
- defp is_object_link_tag(_), do: false
+ def is_object_link_tag(_), do: false
end
diff --git a/test/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy_test.exs b/test/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy_test.exs
new file mode 100644
index 000000000..96b49b6a0
--- /dev/null
+++ b/test/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy_test.exs
@@ -0,0 +1,73 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2023 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicyTest do
+ alias Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicy
+
+ use Pleroma.DataCase
+
+ require Pleroma.Constants
+
+ test "Add quote url to Link tag" do
+ quote_url = "https://gleasonator.com/objects/1234"
+
+ activity = %{
+ "type" => "Create",
+ "actor" => "https://gleasonator.com/users/alex",
+ "object" => %{
+ "type" => "Note",
+ "content" => "Nice post",
+ "quoteUrl" => quote_url
+ }
+ }
+
+ {:ok, %{"object" => object}} = QuoteToLinkTagPolicy.filter(activity)
+
+ assert object["tag"] == [
+ %{
+ "type" => "Link",
+ "href" => quote_url,
+ "mediaType" => Pleroma.Constants.activity_json_canonical_mime_type()
+ }
+ ]
+ end
+
+ test "Add quote url to Link tag, append to the end" do
+ quote_url = "https://gleasonator.com/objects/1234"
+
+ activity = %{
+ "type" => "Create",
+ "actor" => "https://gleasonator.com/users/alex",
+ "object" => %{
+ "type" => "Note",
+ "content" => "Nice post",
+ "quoteUrl" => quote_url,
+ "tag" => [%{"type" => "Hashtag", "name" => "#foo"}]
+ }
+ }
+
+ {:ok, %{"object" => object}} = QuoteToLinkTagPolicy.filter(activity)
+
+ assert [_, tag] = object["tag"]
+
+ assert tag == %{
+ "type" => "Link",
+ "href" => quote_url,
+ "mediaType" => Pleroma.Constants.activity_json_canonical_mime_type()
+ }
+ end
+
+ test "Bypass posts without quoteUrl" do
+ activity = %{
+ "type" => "Create",
+ "actor" => "https://gleasonator.com/users/alex",
+ "object" => %{
+ "type" => "Note",
+ "content" => "Nice post"
+ }
+ }
+
+ assert {:ok, ^activity} = QuoteToLinkTagPolicy.filter(activity)
+ end
+end
From 8b98a98dfb4ab3db9b4f2347713d86ed9c87b61b Mon Sep 17 00:00:00 2001
From: tusooa
Date: Wed, 12 Jul 2023 14:37:12 -0400
Subject: [PATCH 186/823] Make InlineQuotePolicy history aware
---
lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
index a0eefefc0..aaa209aa1 100644
--- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
@@ -53,6 +53,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do
@impl true
def describe, do: {:ok, %{}}
+ @impl Pleroma.Web.ActivityPub.MRF.Policy
+ def history_awareness, do: :auto
+
@impl true
def config_description do
%{
From 8596f926543126efdb4b8cfe70beab6812824398 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Wed, 12 Jul 2023 14:58:20 -0400
Subject: [PATCH 187/823] Fix TransmogrifierTest
---
test/pleroma/web/activity_pub/transmogrifier_test.exs | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs
index 4a192cdc0..5e58d75db 100644
--- a/test/pleroma/web/activity_pub/transmogrifier_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs
@@ -123,7 +123,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
assert activity.data["context"] == object.data["context"]
end
- test "it drops link tags" do
+ test "it keeps link tags" do
insert(:user, ap_id: "https://example.org/users/alice")
message = File.read!("test/fixtures/fep-e232.json") |> Jason.decode!()
@@ -131,10 +131,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
assert {:ok, activity} = Transmogrifier.handle_incoming(message)
object = Object.normalize(activity)
- assert length(object.data["tag"]) == 1
-
- tag = object.data["tag"] |> List.first()
- assert tag["type"] == "Mention"
+ assert [%{"type" => "Mention"}, %{"type" => "Link"}] = object.data["tag"]
end
test "it accepts quote posts" do
From 87353e5ad12799d12507253fe9a0363fd9f0c817 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Wed, 12 Jul 2023 22:07:16 -0400
Subject: [PATCH 188/823] Fix config descriptions for mrf inline quote
---
config/description.exs | 18 ------------------
.../activity_pub/mrf/inline_quote_policy.ex | 12 +++++++-----
2 files changed, 7 insertions(+), 23 deletions(-)
diff --git a/config/description.exs b/config/description.exs
index 079d187d5..d18649ae8 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -2994,24 +2994,6 @@ config :pleroma, :config_description, [
}
]
},
- %{
- group: :pleroma,
- key: :mrf_inline_quote,
- tab: :mrf,
- related_policy: "Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy",
- label: "MRF Inline Quote Policy",
- type: :group,
- description: "Force quote url to appear in post content.",
- children: [
- %{
- key: :template,
- type: :string,
- description:
- "The template to append to the post. `{url}` will be replaced with the actual link to the quoted post.",
- suggestions: ["RT: {url}"]
- }
- ]
- },
%{
group: :pleroma,
key: :modules,
diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
index aaa209aa1..171b22c5e 100644
--- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
@@ -61,14 +61,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do
%{
key: :mrf_inline_quote,
related_policy: "Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy",
- label: "MRF Inline Quote",
- description: "Force quote post URLs inline",
+ label: "MRF Inline Quote Policy",
+ type: :group,
+ description: "Force quote url to appear in post content.",
children: [
%{
- key: :prefix,
+ key: :template,
type: :string,
- description: "Prefix before the link",
- suggestions: ["RT", "QT", "RE", "RN"]
+ description:
+ "The template to append to the post. `{url}` will be replaced with the actual link to the quoted post.",
+ suggestions: ["RT: {url}"]
}
]
}
From 875b46d97d910ffd2c33ac26ed8dfe38f7672f52 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Wed, 12 Jul 2023 23:29:23 -0400
Subject: [PATCH 189/823] Do not mention original poster when quoting
---
lib/pleroma/web/common_api/activity_draft.ex | 3 ---
test/pleroma/web/common_api_test.exs | 4 ++--
2 files changed, 2 insertions(+), 5 deletions(-)
diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex
index 32921aa5a..ca1329284 100644
--- a/lib/pleroma/web/common_api/activity_draft.ex
+++ b/lib/pleroma/web/common_api/activity_draft.ex
@@ -139,9 +139,6 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
defp quote_post(%{params: %{quote_id: id}} = draft) when not_empty_string(id) do
case Activity.get_by_id_with_object(id) do
- %Activity{actor: actor_ap_id} = activity when not_empty_string(actor_ap_id) ->
- %__MODULE__{draft | quote_post: activity, mentions: [actor_ap_id]}
-
%Activity{} = activity ->
%__MODULE__{draft | quote_post: activity}
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index 70a5b6fed..a98b16d4b 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -808,8 +808,8 @@ defmodule Pleroma.Web.CommonAPITest do
assert quote_post.data["quoteUrl"] == quoted.data["id"]
- # The OP is mentioned
- assert quoted.data["actor"] in quote_post.data["to"]
+ # The OP is not mentioned
+ refute quoted.data["actor"] in quote_post.data["to"]
end
test "quote posting with explicit addressing doesn't mention the OP" do
From a8b2f9205d16465a3b11d3802c966db3da908c5d Mon Sep 17 00:00:00 2001
From: tusooa
Date: Wed, 12 Jul 2023 23:47:31 -0400
Subject: [PATCH 190/823] Expose quote_id parameter on the api
---
lib/pleroma/web/api_spec/schemas/status.ex | 5 +++++
lib/pleroma/web/mastodon_api/views/status_view.ex | 10 ++++++++++
.../web/mastodon_api/views/status_view_test.exs | 3 +++
3 files changed, 18 insertions(+)
diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex
index 5d0eedb08..07f03134a 100644
--- a/lib/pleroma/web/api_spec/schemas/status.ex
+++ b/lib/pleroma/web/api_spec/schemas/status.ex
@@ -198,6 +198,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
nullable: true,
description: "Quoted status (if any)"
},
+ quote_id: %Schema{
+ nullable: true,
+ allOf: [FlakeID],
+ description: "ID of the status being quoted, if any"
+ },
quote_url: %Schema{
type: :string,
format: :uri,
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index ba4a8f3eb..3d3039751 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -312,6 +312,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
# Here the implicit index of the current content is 0
chrono_order = history_len - 1
+ quote_id = get_quote_id(activity)
+
quote_activity = get_quote(activity, opts)
quote_post =
@@ -431,6 +433,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
context: object.data["context"],
in_reply_to_account_acct: reply_to_user && reply_to_user.nickname,
quote: quote_post,
+ quote_id: quote_id,
quote_url: object.data["quoteUrl"],
quote_visible: visible_for_user?(quote_activity, opts[:for]),
content: %{"text/plain" => content_plaintext},
@@ -689,6 +692,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
end
end
+ defp get_quote_id(activity) do
+ case get_quote(activity, %{}) do
+ %Activity{id: id} -> id
+ _ -> nil
+ end
+ end
+
def render_content(%{data: %{"name" => name}} = object) when not is_nil(name) and name != "" do
url = object.data["url"] || object.data["id"]
diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs
index 6d3a72970..221244d4e 100644
--- a/test/pleroma/web/mastodon_api/views/status_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs
@@ -327,6 +327,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
context: object_data["context"],
in_reply_to_account_acct: nil,
quote: nil,
+ quote_id: nil,
quote_url: nil,
quote_visible: false,
content: %{"text/plain" => HTML.strip_tags(object_data["content"])},
@@ -435,10 +436,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
status = StatusView.render("show.json", %{activity: quoted_quote_post})
assert status.pleroma.quote.id == to_string(quote_post.id)
+ assert status.pleroma.quote_id == to_string(quote_post.id)
assert status.pleroma.quote_url == Object.normalize(quote_post).data["id"]
# Quotes don't go more than one level deep
refute status.pleroma.quote.pleroma.quote
+ assert status.pleroma.quote.pleroma.quote_id == to_string(post.id)
assert status.pleroma.quote.pleroma.quote_url == Object.normalize(post).data["id"]
# In an index
From 08608afca5566f712acdc14b7c43976d6d071106 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Wed, 12 Jul 2023 23:56:54 -0400
Subject: [PATCH 191/823] Fix quote_visible attribute
---
.../web/mastodon_api/views/status_view.ex | 19 +++++++------------
.../mastodon_api/views/status_view_test.exs | 2 ++
2 files changed, 9 insertions(+), 12 deletions(-)
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 3d3039751..d070262cc 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -312,12 +312,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
# Here the implicit index of the current content is 0
chrono_order = history_len - 1
- quote_id = get_quote_id(activity)
-
quote_activity = get_quote(activity, opts)
+ quote_id =
+ case quote_activity do
+ %Activity{id: id} -> id
+ _ -> nil
+ end
+
quote_post =
- if visible_for_user?(quote_activity, opts[:for]) do
+ if visible_for_user?(quote_activity, opts[:for]) and opts[:show_quote] != false do
quote_rendering_opts = Map.merge(opts, %{activity: quote_activity, show_quote: false})
render("show.json", quote_rendering_opts)
else
@@ -671,8 +675,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
end
end
- def get_quote(_activity, %{show_quote: false}), do: nil
-
def get_quote(activity, %{quoted_activities: quoted_activities}) do
object = Object.normalize(activity, fetch: false)
@@ -692,13 +694,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
end
end
- defp get_quote_id(activity) do
- case get_quote(activity, %{}) do
- %Activity{id: id} -> id
- _ -> nil
- end
- end
-
def render_content(%{data: %{"name" => name}} = object) when not is_nil(name) and name != "" do
url = object.data["url"] || object.data["id"]
diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs
index 221244d4e..baa9b32f5 100644
--- a/test/pleroma/web/mastodon_api/views/status_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs
@@ -438,11 +438,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
assert status.pleroma.quote.id == to_string(quote_post.id)
assert status.pleroma.quote_id == to_string(quote_post.id)
assert status.pleroma.quote_url == Object.normalize(quote_post).data["id"]
+ assert status.pleroma.quote_visible
# Quotes don't go more than one level deep
refute status.pleroma.quote.pleroma.quote
assert status.pleroma.quote.pleroma.quote_id == to_string(post.id)
assert status.pleroma.quote.pleroma.quote_url == Object.normalize(post).data["id"]
+ assert status.pleroma.quote.pleroma.quote_visible
# In an index
[status] = StatusView.render("index.json", %{activities: [quoted_quote_post], as: :activity})
From 2f6fc6a7ab0e757abfe9ec535842a12b887d2fe6 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier"
Date: Sun, 24 Sep 2023 22:52:41 +0200
Subject: [PATCH 192/823] TwitterAPI: Return proper error when healthcheck is
disabled
---
changelog.d/healthcheck-disabled-error.fix | 1 +
lib/pleroma/web/twitter_api/controllers/util_controller.ex | 5 ++++-
test/pleroma/web/twitter_api/util_controller_test.exs | 2 +-
3 files changed, 6 insertions(+), 2 deletions(-)
create mode 100644 changelog.d/healthcheck-disabled-error.fix
diff --git a/changelog.d/healthcheck-disabled-error.fix b/changelog.d/healthcheck-disabled-error.fix
new file mode 100644
index 000000000..984384a52
--- /dev/null
+++ b/changelog.d/healthcheck-disabled-error.fix
@@ -0,0 +1 @@
+TwitterAPI: Return proper error when healthcheck is disabled
diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
index d5a24ae6c..ca8a98960 100644
--- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex
+++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
@@ -345,13 +345,16 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
end
def healthcheck(conn, _params) do
- with true <- Config.get([:instance, :healthcheck]),
+ with {:cfg, true} <- {:cfg, Config.get([:instance, :healthcheck])},
%{healthy: true} = info <- Healthcheck.system_info() do
json(conn, info)
else
%{healthy: false} = info ->
service_unavailable(conn, info)
+ {:cfg, false} ->
+ service_unavailable(conn, %{"error" => "Healthcheck disabled"})
+
_ ->
service_unavailable(conn, %{})
end
diff --git a/test/pleroma/web/twitter_api/util_controller_test.exs b/test/pleroma/web/twitter_api/util_controller_test.exs
index a4da23635..d06ae71aa 100644
--- a/test/pleroma/web/twitter_api/util_controller_test.exs
+++ b/test/pleroma/web/twitter_api/util_controller_test.exs
@@ -106,7 +106,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do
|> get("/api/pleroma/healthcheck")
|> json_response_and_validate_schema(503)
- assert response == %{}
+ assert response == %{"error" => "Healthcheck disabled"}
end
test "returns 200 when healthcheck enabled and all ok", %{conn: conn} do
From 2b5636bf127b1987736c281d346a5771c8168c09 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Fri, 31 Mar 2023 21:47:37 -0400
Subject: [PATCH 193/823] Allow unified streaming endpoint
---
.../web/mastodon_api/websocket_handler.ex | 17 ++++++++++++-----
lib/pleroma/web/streamer.ex | 8 ++++++--
.../integration/mastodon_websocket_test.exs | 5 ++++-
test/pleroma/web/streamer_test.exs | 4 ++++
4 files changed, 26 insertions(+), 8 deletions(-)
diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index 88444106d..97a1f1401 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -32,8 +32,15 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
req
end
+ topics =
+ if topic do
+ [topic]
+ else
+ []
+ end
+
{:cowboy_websocket, req,
- %{user: user, topic: topic, oauth_token: oauth_token, count: 0, timer: nil},
+ %{user: user, topics: topics, oauth_token: oauth_token, count: 0, timer: nil},
%{idle_timeout: @timeout}}
else
{:error, :bad_topic} ->
@@ -50,10 +57,10 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
def websocket_init(state) do
Logger.debug(
- "#{__MODULE__} accepted websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topic #{state.topic}"
+ "#{__MODULE__} accepted websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topics #{state.topics}"
)
- Streamer.add_socket(state.topic, state.oauth_token)
+ Enum.each(state.topics, fn topic -> Streamer.add_socket(topic, state.oauth_token) end)
{:ok, %{state | timer: timer()}}
end
@@ -109,10 +116,10 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
def terminate(reason, _req, state) do
Logger.debug(
- "#{__MODULE__} terminating websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topic #{state.topic || "?"}: #{inspect(reason)}"
+ "#{__MODULE__} terminating websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topics #{state.topics || "?"}: #{inspect(reason)}"
)
- Streamer.remove_socket(state.topic)
+ Enum.each(state.topics, fn topic -> Streamer.remove_socket(topic) end)
:ok
end
diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex
index b9a04cc76..6b5fdbf0c 100644
--- a/lib/pleroma/web/streamer.ex
+++ b/lib/pleroma/web/streamer.ex
@@ -59,10 +59,14 @@ defmodule Pleroma.Web.Streamer do
end
@doc "Expand and authorizes a stream"
- @spec get_topic(stream :: String.t(), User.t() | nil, Token.t() | nil, Map.t()) ::
- {:ok, topic :: String.t()} | {:error, :bad_topic}
+ @spec get_topic(stream :: String.t() | nil, User.t() | nil, Token.t() | nil, Map.t()) ::
+ {:ok, topic :: String.t() | nil} | {:error, :bad_topic}
def get_topic(stream, user, oauth_token, params \\ %{})
+ def get_topic(nil = _stream, _user, _oauth_token, _params) do
+ {:ok, nil}
+ end
+
# Allow all public steams if the instance allows unauthenticated access.
# Otherwise, only allow users with valid oauth tokens.
def get_topic(stream, user, oauth_token, _params) when stream in @public_streams do
diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs
index 9be0445c0..3b120c10e 100644
--- a/test/pleroma/integration/mastodon_websocket_test.exs
+++ b/test/pleroma/integration/mastodon_websocket_test.exs
@@ -33,7 +33,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
test "refuses invalid requests" do
capture_log(fn ->
- assert {:error, %WebSockex.RequestError{code: 404}} = start_socket()
assert {:error, %WebSockex.RequestError{code: 404}} = start_socket("?stream=ncjdk")
Process.sleep(30)
end)
@@ -49,6 +48,10 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
end)
end
+ test "allows unified stream" do
+ assert {:ok, _} = start_socket()
+ end
+
test "allows public streams without authentication" do
assert {:ok, _} = start_socket("?stream=public")
assert {:ok, _} = start_socket("?stream=public:local")
diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs
index 7ab0e379b..da97b87d8 100644
--- a/test/pleroma/web/streamer_test.exs
+++ b/test/pleroma/web/streamer_test.exs
@@ -22,6 +22,10 @@ defmodule Pleroma.Web.StreamerTest do
setup do: clear_config([:instance, :skip_thread_containment])
describe "get_topic/_ (unauthenticated)" do
+ test "allows no stream" do
+ assert {:ok, nil} = Streamer.get_topic(nil, nil, nil)
+ end
+
test "allows public" do
assert {:ok, "public"} = Streamer.get_topic("public", nil, nil)
assert {:ok, "public:local"} = Streamer.get_topic("public:local", nil, nil)
From 273cda63ad79b61f4d37e4b7603694908e894e4f Mon Sep 17 00:00:00 2001
From: tusooa
Date: Fri, 31 Mar 2023 22:55:52 -0400
Subject: [PATCH 194/823] Allow subscribing to streams
---
.../web/mastodon_api/websocket_handler.ex | 74 +++++++++++++++
lib/pleroma/web/views/streamer_view.ex | 18 ++++
.../integration/mastodon_websocket_test.exs | 90 +++++++++++++++++++
3 files changed, 182 insertions(+)
diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index 97a1f1401..a42a9a63c 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
alias Pleroma.User
alias Pleroma.Web.OAuth.Token
alias Pleroma.Web.Streamer
+ alias Pleroma.Web.StreamerView
@behaviour :cowboy_websocket
@@ -73,6 +74,16 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
# We only receive pings for now
def websocket_handle(:ping, state), do: {:ok, state}
+ def websocket_handle({:text, text}, state) do
+ with {:ok, %{} = event} <- Jason.decode(text) do
+ handle_client_event(event, state)
+ else
+ _ ->
+ Logger.error("#{__MODULE__} received non-JSON event: #{inspect(text)}")
+ {:ok, state}
+ end
+ end
+
def websocket_handle(frame, state) do
Logger.error("#{__MODULE__} received frame: #{inspect(frame)}")
{:ok, state}
@@ -144,4 +155,67 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
defp timer do
Process.send_after(self(), :tick, @tick)
end
+
+ defp handle_client_event(%{"type" => "subscribe", "stream" => _topic} = params, state) do
+ with {_, {:ok, topic}} <-
+ {:topic, Streamer.get_topic(params["stream"], state.user, state.oauth_token, params)},
+ {_, false} <- {:subscribed, topic in state.topics} do
+ Streamer.add_socket(topic, state.oauth_token)
+
+ {[
+ {:text,
+ StreamerView.render("pleroma_respond.json", %{type: "subscribe", result: "success"})}
+ ], %{state | topics: [topic | state.topics]}}
+ else
+ {:subscribed, true} ->
+ {[
+ {:text,
+ StreamerView.render("pleroma_respond.json", %{type: "subscribe", result: "ignored"})}
+ ], state}
+
+ {:topic, {:error, error}} ->
+ {[
+ {:text,
+ StreamerView.render("pleroma_respond.json", %{
+ type: "subscribe",
+ result: "error",
+ error: error
+ })}
+ ], state}
+ end
+ end
+
+ defp handle_client_event(%{"type" => "unsubscribe", "stream" => _topic} = params, state) do
+ with {_, {:ok, topic}} <-
+ {:topic, Streamer.get_topic(params["stream"], state.user, state.oauth_token, params)},
+ {_, true} <- {:subscribed, topic in state.topics} do
+ Streamer.remove_socket(topic)
+
+ {[
+ {:text,
+ StreamerView.render("pleroma_respond.json", %{type: "unsubscribe", result: "success"})}
+ ], %{state | topics: List.delete(state.topics, topic)}}
+ else
+ {:subscribed, false} ->
+ {[
+ {:text,
+ StreamerView.render("pleroma_respond.json", %{type: "unsubscribe", result: "ignored"})}
+ ], state}
+
+ {:topic, {:error, error}} ->
+ {[
+ {:text,
+ StreamerView.render("pleroma_respond.json", %{
+ type: "unsubscribe",
+ result: "error",
+ error: error
+ })}
+ ], state}
+ end
+ end
+
+ defp handle_client_event(params, state) do
+ Logger.error("#{__MODULE__} received unknown event: #{inspect(params)}")
+ {[], state}
+ end
end
diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex
index 6a55242b0..19f098783 100644
--- a/lib/pleroma/web/views/streamer_view.ex
+++ b/lib/pleroma/web/views/streamer_view.ex
@@ -135,4 +135,22 @@ defmodule Pleroma.Web.StreamerView do
}
|> Jason.encode!()
end
+
+ def render("pleroma_respond.json", %{type: type, result: result} = params) do
+ %{
+ event: "pleroma.respond",
+ payload:
+ %{
+ result: result,
+ type: type
+ }
+ |> Map.merge(maybe_error(params))
+ |> Jason.encode!()
+ }
+ |> Jason.encode!()
+ end
+
+ defp maybe_error(%{error: :bad_topic}), do: %{error: "bad_topic"}
+ defp maybe_error(%{error: :unauthorized}), do: %{error: "unauthorized"}
+ defp maybe_error(_), do: %{}
end
diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs
index 3b120c10e..9db0f714f 100644
--- a/test/pleroma/integration/mastodon_websocket_test.exs
+++ b/test/pleroma/integration/mastodon_websocket_test.exs
@@ -31,6 +31,13 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
WebsocketClient.start_link(self(), path, headers)
end
+ defp decode_json(json) do
+ with {:ok, %{"event" => event, "payload" => payload_text}} <- Jason.decode(json),
+ {:ok, payload} <- Jason.decode(payload_text) do
+ {:ok, %{"event" => event, "payload" => payload}}
+ end
+ end
+
test "refuses invalid requests" do
capture_log(fn ->
assert {:error, %WebSockex.RequestError{code: 404}} = start_socket("?stream=ncjdk")
@@ -79,6 +86,89 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
assert json == view_json
end
+ describe "subscribing via WebSocket" do
+ test "can subscribe" do
+ user = insert(:user)
+ {:ok, pid} = start_socket()
+ WebsocketClient.send_text(pid, %{type: "subscribe", stream: "public"} |> Jason.encode!())
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma.respond",
+ "payload" => %{"type" => "subscribe", "result" => "success"}
+ }} = decode_json(raw_json)
+
+ {:ok, activity} = CommonAPI.post(user, %{status: "nice echo chamber"})
+
+ assert_receive {:text, raw_json}, 1_000
+ assert {:ok, json} = Jason.decode(raw_json)
+
+ assert "update" == json["event"]
+ assert json["payload"]
+ assert {:ok, json} = Jason.decode(json["payload"])
+
+ view_json =
+ Pleroma.Web.MastodonAPI.StatusView.render("show.json", activity: activity, for: nil)
+ |> Jason.encode!()
+ |> Jason.decode!()
+
+ assert json == view_json
+ end
+
+ test "won't double subscribe" do
+ user = insert(:user)
+ {:ok, pid} = start_socket()
+ WebsocketClient.send_text(pid, %{type: "subscribe", stream: "public"} |> Jason.encode!())
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma.respond",
+ "payload" => %{"type" => "subscribe", "result" => "success"}
+ }} = decode_json(raw_json)
+
+ WebsocketClient.send_text(pid, %{type: "subscribe", stream: "public"} |> Jason.encode!())
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma.respond",
+ "payload" => %{"type" => "subscribe", "result" => "ignored"}
+ }} = decode_json(raw_json)
+
+ {:ok, _activity} = CommonAPI.post(user, %{status: "nice echo chamber"})
+
+ assert_receive {:text, _}, 1_000
+ refute_receive {:text, _}, 1_000
+ end
+
+ test "can unsubscribe" do
+ user = insert(:user)
+ {:ok, pid} = start_socket()
+ WebsocketClient.send_text(pid, %{type: "subscribe", stream: "public"} |> Jason.encode!())
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma.respond",
+ "payload" => %{"type" => "subscribe", "result" => "success"}
+ }} = decode_json(raw_json)
+
+ WebsocketClient.send_text(pid, %{type: "unsubscribe", stream: "public"} |> Jason.encode!())
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma.respond",
+ "payload" => %{"type" => "unsubscribe", "result" => "success"}
+ }} = decode_json(raw_json)
+
+ {:ok, _activity} = CommonAPI.post(user, %{status: "nice echo chamber"})
+ refute_receive {:text, _}, 1_000
+ end
+ end
+
describe "with a valid user token" do
setup do
{:ok, app} =
From 21395aa5090f2a53bdbe0ef5fac46693d16025ed Mon Sep 17 00:00:00 2001
From: tusooa
Date: Fri, 31 Mar 2023 23:19:57 -0400
Subject: [PATCH 195/823] Allow authenticating via client-sent events
---
.../web/mastodon_api/websocket_handler.ex | 36 +++++++++
lib/pleroma/web/views/streamer_view.ex | 1 +
.../integration/mastodon_websocket_test.exs | 81 +++++++++++++++++++
3 files changed, 118 insertions(+)
diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index a42a9a63c..6233c3340 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -214,6 +214,42 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
end
end
+ defp handle_client_event(
+ %{"type" => "pleroma.authenticate", "token" => access_token} = _params,
+ state
+ ) do
+ with {:auth, nil, nil} <- {:auth, state.user, state.oauth_token},
+ {:ok, user, oauth_token} <- authenticate_request(access_token, nil) do
+ {[
+ {:text,
+ StreamerView.render("pleroma_respond.json", %{
+ type: "pleroma.authenticate",
+ result: "success"
+ })}
+ ], %{state | user: user, oauth_token: oauth_token}}
+ else
+ {:auth, _, _} ->
+ {[
+ {:text,
+ StreamerView.render("pleroma_respond.json", %{
+ type: "pleroma.authenticate",
+ result: "error",
+ error: :already_authenticated
+ })}
+ ], state}
+
+ _ ->
+ {[
+ {:text,
+ StreamerView.render("pleroma_respond.json", %{
+ type: "pleroma.authenticate",
+ result: "error",
+ error: :unauthorized
+ })}
+ ], state}
+ end
+ end
+
defp handle_client_event(params, state) do
Logger.error("#{__MODULE__} received unknown event: #{inspect(params)}")
{[], state}
diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex
index 19f098783..0cdcb1918 100644
--- a/lib/pleroma/web/views/streamer_view.ex
+++ b/lib/pleroma/web/views/streamer_view.ex
@@ -152,5 +152,6 @@ defmodule Pleroma.Web.StreamerView do
defp maybe_error(%{error: :bad_topic}), do: %{error: "bad_topic"}
defp maybe_error(%{error: :unauthorized}), do: %{error: "unauthorized"}
+ defp maybe_error(%{error: :already_authenticated}), do: %{error: "already_authenticated"}
defp maybe_error(_), do: %{}
end
diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs
index 9db0f714f..827c7b5b0 100644
--- a/test/pleroma/integration/mastodon_websocket_test.exs
+++ b/test/pleroma/integration/mastodon_websocket_test.exs
@@ -224,6 +224,87 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
end)
end
+ test "accepts valid token on client-sent event", %{token: token} do
+ assert {:ok, pid} = start_socket()
+
+ WebsocketClient.send_text(
+ pid,
+ %{type: "pleroma.authenticate", token: token.token} |> Jason.encode!()
+ )
+
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma.respond",
+ "payload" => %{"type" => "pleroma.authenticate", "result" => "success"}
+ }} = decode_json(raw_json)
+
+ WebsocketClient.send_text(pid, %{type: "subscribe", stream: "user"} |> Jason.encode!())
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma.respond",
+ "payload" => %{"type" => "subscribe", "result" => "success"}
+ }} = decode_json(raw_json)
+ end
+
+ test "rejects invalid token on client-sent event" do
+ assert {:ok, pid} = start_socket()
+
+ WebsocketClient.send_text(
+ pid,
+ %{type: "pleroma.authenticate", token: "Something else"} |> Jason.encode!()
+ )
+
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma.respond",
+ "payload" => %{
+ "type" => "pleroma.authenticate",
+ "result" => "error",
+ "error" => "unauthorized"
+ }
+ }} = decode_json(raw_json)
+ end
+
+ test "rejects new authenticate request if already logged-in", %{token: token} do
+ assert {:ok, pid} = start_socket()
+
+ WebsocketClient.send_text(
+ pid,
+ %{type: "pleroma.authenticate", token: token.token} |> Jason.encode!()
+ )
+
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma.respond",
+ "payload" => %{"type" => "pleroma.authenticate", "result" => "success"}
+ }} = decode_json(raw_json)
+
+ WebsocketClient.send_text(
+ pid,
+ %{type: "pleroma.authenticate", token: "Something else"} |> Jason.encode!()
+ )
+
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma.respond",
+ "payload" => %{
+ "type" => "pleroma.authenticate",
+ "result" => "error",
+ "error" => "already_authenticated"
+ }
+ }} = decode_json(raw_json)
+ end
+
test "disconnect when token is revoked", %{app: app, user: user, token: token} do
assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}")
assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}")
From 7d005e8c93b22dc3d7be1a66dd2d404b7f54306a Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 01:25:13 -0400
Subject: [PATCH 196/823] Return stream attribute in server-sent events
---
lib/pleroma/constants.ex | 4 ++
.../web/mastodon_api/websocket_handler.ex | 4 +-
lib/pleroma/web/streamer.ex | 27 +++++----
lib/pleroma/web/views/streamer_view.ex | 42 +++++++++++---
.../integration/mastodon_websocket_test.exs | 33 +++++++++++
test/pleroma/web/streamer_test.exs | 58 +++++++++----------
6 files changed, 119 insertions(+), 49 deletions(-)
diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex
index ed60bcc37..77bc4bfac 100644
--- a/lib/pleroma/constants.ex
+++ b/lib/pleroma/constants.ex
@@ -94,4 +94,8 @@ defmodule Pleroma.Constants do
"application/activity+json"
]
)
+
+ const(public_streams,
+ do: ["public", "public:local", "public:media", "public:local:media"]
+ )
end
diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index 6233c3340..2707673ba 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -89,11 +89,11 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
{:ok, state}
end
- def websocket_info({:render_with_user, view, template, item}, state) do
+ def websocket_info({:render_with_user, view, template, item, topic}, state) do
user = %User{} = User.get_cached_by_ap_id(state.user.ap_id)
unless Streamer.filtered_by_user?(user, item) do
- websocket_info({:text, view.render(template, item, user)}, %{state | user: user})
+ websocket_info({:text, view.render(template, item, user, topic)}, %{state | user: user})
else
{:ok, state}
end
diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex
index 6b5fdbf0c..70e46617a 100644
--- a/lib/pleroma/web/streamer.ex
+++ b/lib/pleroma/web/streamer.ex
@@ -4,6 +4,7 @@
defmodule Pleroma.Web.Streamer do
require Logger
+ require Pleroma.Constants
alias Pleroma.Activity
alias Pleroma.Chat.MessageReference
@@ -24,7 +25,7 @@ defmodule Pleroma.Web.Streamer do
def registry, do: @registry
- @public_streams ["public", "public:local", "public:media", "public:local:media"]
+ @public_streams Pleroma.Constants.public_streams()
@local_streams ["public:local", "public:local:media"]
@user_streams ["user", "user:notification", "direct", "user:pleroma_chat"]
@@ -223,8 +224,8 @@ defmodule Pleroma.Web.Streamer do
end
defp do_stream("follow_relationship", item) do
- text = StreamerView.render("follow_relationships_update.json", item)
user_topic = "user:#{item.follower.id}"
+ text = StreamerView.render("follow_relationships_update.json", item, user_topic)
Logger.debug("Trying to push follow relationship update to #{user_topic}\n\n")
@@ -270,9 +271,11 @@ defmodule Pleroma.Web.Streamer do
defp do_stream(topic, %Notification{} = item)
when topic in ["user", "user:notification"] do
- Registry.dispatch(@registry, "#{topic}:#{item.user_id}", fn list ->
+ user_topic = "#{topic}:#{item.user_id}"
+
+ Registry.dispatch(@registry, user_topic, fn list ->
Enum.each(list, fn {pid, _auth} ->
- send(pid, {:render_with_user, StreamerView, "notification.json", item})
+ send(pid, {:render_with_user, StreamerView, "notification.json", item, user_topic})
end)
end)
end
@@ -281,7 +284,7 @@ defmodule Pleroma.Web.Streamer do
when topic in ["user", "user:pleroma_chat"] do
topic = "#{topic}:#{user.id}"
- text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref})
+ text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref}, topic)
Registry.dispatch(@registry, topic, fn list ->
Enum.each(list, fn {pid, _auth} ->
@@ -309,7 +312,7 @@ defmodule Pleroma.Web.Streamer do
end
defp push_to_socket(topic, %Participation{} = participation) do
- rendered = StreamerView.render("conversation.json", participation)
+ rendered = StreamerView.render("conversation.json", participation, topic)
Registry.dispatch(@registry, topic, fn list ->
Enum.each(list, fn {pid, _} ->
@@ -337,12 +340,15 @@ defmodule Pleroma.Web.Streamer do
Pleroma.Activity.get_create_by_object_ap_id(item.object.data["id"])
|> Map.put(:object, item.object)
- anon_render = StreamerView.render("status_update.json", create_activity)
+ anon_render = StreamerView.render("status_update.json", create_activity, topic)
Registry.dispatch(@registry, topic, fn list ->
Enum.each(list, fn {pid, auth?} ->
if auth? do
- send(pid, {:render_with_user, StreamerView, "status_update.json", create_activity})
+ send(
+ pid,
+ {:render_with_user, StreamerView, "status_update.json", create_activity, topic}
+ )
else
send(pid, {:text, anon_render})
end
@@ -351,12 +357,13 @@ defmodule Pleroma.Web.Streamer do
end
defp push_to_socket(topic, item) do
- anon_render = StreamerView.render("update.json", item)
+ Logger.debug("topic=#{topic}")
+ anon_render = StreamerView.render("update.json", item, topic)
Registry.dispatch(@registry, topic, fn list ->
Enum.each(list, fn {pid, auth?} ->
if auth? do
- send(pid, {:render_with_user, StreamerView, "update.json", item})
+ send(pid, {:render_with_user, StreamerView, "update.json", item, topic})
else
send(pid, {:text, anon_render})
end
diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex
index 0cdcb1918..f591da9a6 100644
--- a/lib/pleroma/web/views/streamer_view.ex
+++ b/lib/pleroma/web/views/streamer_view.ex
@@ -11,8 +11,11 @@ defmodule Pleroma.Web.StreamerView do
alias Pleroma.User
alias Pleroma.Web.MastodonAPI.NotificationView
- def render("update.json", %Activity{} = activity, %User{} = user) do
+ require Pleroma.Constants
+
+ def render("update.json", %Activity{} = activity, %User{} = user, topic) do
%{
+ stream: render("stream.json", %{topic: topic}),
event: "update",
payload:
Pleroma.Web.MastodonAPI.StatusView.render(
@@ -25,8 +28,9 @@ defmodule Pleroma.Web.StreamerView do
|> Jason.encode!()
end
- def render("status_update.json", %Activity{} = activity, %User{} = user) do
+ def render("status_update.json", %Activity{} = activity, %User{} = user, topic) do
%{
+ stream: render("stream.json", %{topic: topic}),
event: "status.update",
payload:
Pleroma.Web.MastodonAPI.StatusView.render(
@@ -39,8 +43,9 @@ defmodule Pleroma.Web.StreamerView do
|> Jason.encode!()
end
- def render("notification.json", %Notification{} = notify, %User{} = user) do
+ def render("notification.json", %Notification{} = notify, %User{} = user, topic) do
%{
+ stream: render("stream.json", %{topic: topic}),
event: "notification",
payload:
NotificationView.render(
@@ -52,8 +57,9 @@ defmodule Pleroma.Web.StreamerView do
|> Jason.encode!()
end
- def render("update.json", %Activity{} = activity) do
+ def render("update.json", %Activity{} = activity, topic) do
%{
+ stream: render("stream.json", %{topic: topic}),
event: "update",
payload:
Pleroma.Web.MastodonAPI.StatusView.render(
@@ -65,8 +71,9 @@ defmodule Pleroma.Web.StreamerView do
|> Jason.encode!()
end
- def render("status_update.json", %Activity{} = activity) do
+ def render("status_update.json", %Activity{} = activity, topic) do
%{
+ stream: render("stream.json", %{topic: topic}),
event: "status.update",
payload:
Pleroma.Web.MastodonAPI.StatusView.render(
@@ -78,7 +85,7 @@ defmodule Pleroma.Web.StreamerView do
|> Jason.encode!()
end
- def render("chat_update.json", %{chat_message_reference: cm_ref}) do
+ def render("chat_update.json", %{chat_message_reference: cm_ref}, topic) do
# Explicitly giving the cmr for the object here, so we don't accidentally
# send a later 'last_message' that was inserted between inserting this and
# streaming it out
@@ -93,6 +100,7 @@ defmodule Pleroma.Web.StreamerView do
)
%{
+ stream: render("stream.json", %{topic: topic}),
event: "pleroma:chat_update",
payload:
representation
@@ -101,8 +109,9 @@ defmodule Pleroma.Web.StreamerView do
|> Jason.encode!()
end
- def render("follow_relationships_update.json", item) do
+ def render("follow_relationships_update.json", item, topic) do
%{
+ stream: render("stream.json", %{topic: topic}),
event: "pleroma:follow_relationships_update",
payload:
%{
@@ -123,8 +132,9 @@ defmodule Pleroma.Web.StreamerView do
|> Jason.encode!()
end
- def render("conversation.json", %Participation{} = participation) do
+ def render("conversation.json", %Participation{} = participation, topic) do
%{
+ stream: render("stream.json", %{topic: topic}),
event: "conversation",
payload:
Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{
@@ -150,6 +160,22 @@ defmodule Pleroma.Web.StreamerView do
|> Jason.encode!()
end
+ def render("stream.json", %{topic: "user:pleroma_chat:" <> _}), do: ["user:pleroma_chat"]
+ def render("stream.json", %{topic: "user:notification:" <> _}), do: ["user:notification"]
+ def render("stream.json", %{topic: "user:" <> _}), do: ["user"]
+ def render("stream.json", %{topic: "direct:" <> _}), do: ["direct"]
+ def render("stream.json", %{topic: "list:" <> id}), do: ["list", id]
+ def render("stream.json", %{topic: "hashtag:" <> tag}), do: ["hashtag", tag]
+
+ def render("stream.json", %{topic: "public:remote:media:" <> instance}),
+ do: ["public:remote:media", instance]
+
+ def render("stream.json", %{topic: "public:remote:" <> instance}),
+ do: ["public:remote", instance]
+
+ def render("stream.json", %{topic: stream}) when stream in Pleroma.Constants.public_streams(),
+ do: [stream]
+
defp maybe_error(%{error: :bad_topic}), do: %{error: "bad_topic"}
defp maybe_error(%{error: :unauthorized}), do: %{error: "unauthorized"}
defp maybe_error(%{error: :already_authenticated}), do: %{error: "already_authenticated"}
diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs
index 827c7b5b0..d2e98df3b 100644
--- a/test/pleroma/integration/mastodon_websocket_test.exs
+++ b/test/pleroma/integration/mastodon_websocket_test.exs
@@ -116,6 +116,39 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
assert json == view_json
end
+ test "can subscribe to multiple streams" do
+ user = insert(:user)
+ {:ok, pid} = start_socket()
+ WebsocketClient.send_text(pid, %{type: "subscribe", stream: "public"} |> Jason.encode!())
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma.respond",
+ "payload" => %{"type" => "subscribe", "result" => "success"}
+ }} = decode_json(raw_json)
+
+ WebsocketClient.send_text(pid, %{type: "subscribe", stream: "hashtag", tag: "mew"} |> Jason.encode!())
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma.respond",
+ "payload" => %{"type" => "subscribe", "result" => "success"}
+ }} = decode_json(raw_json)
+
+ {:ok, _activity} = CommonAPI.post(user, %{status: "nice echo chamber #mew"})
+
+ assert_receive {:text, raw_json}, 1_000
+ assert {:ok, %{"stream" => stream1}} = Jason.decode(raw_json)
+ assert_receive {:text, raw_json}, 1_000
+ assert {:ok, %{"stream" => stream2}} = Jason.decode(raw_json)
+
+ streams = [stream1, stream2]
+ assert ["hashtag", "mew"] in streams
+ assert ["public"] in streams
+ end
+
test "won't double subscribe" do
user = insert(:user)
{:ok, pid} = start_socket()
diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs
index da97b87d8..cc57a2989 100644
--- a/test/pleroma/web/streamer_test.exs
+++ b/test/pleroma/web/streamer_test.exs
@@ -246,7 +246,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user", user, oauth_token)
{:ok, activity} = CommonAPI.post(user, %{status: "hey"})
- assert_receive {:render_with_user, _, _, ^activity}
+ assert_receive {:render_with_user, _, _, ^activity, _}
refute Streamer.filtered_by_user?(user, activity)
end
@@ -257,7 +257,7 @@ defmodule Pleroma.Web.StreamerTest do
{:ok, activity} = CommonAPI.post(other_user, %{status: "hey"})
{:ok, announce} = CommonAPI.repeat(activity.id, user)
- assert_receive {:render_with_user, Pleroma.Web.StreamerView, "update.json", ^announce}
+ assert_receive {:render_with_user, Pleroma.Web.StreamerView, "update.json", ^announce, _}
refute Streamer.filtered_by_user?(user, announce)
end
@@ -310,7 +310,7 @@ defmodule Pleroma.Web.StreamerTest do
{:ok, %Pleroma.Activity{data: _data, local: false} = announce} =
Pleroma.Web.ActivityPub.Transmogrifier.handle_incoming(data)
- assert_receive {:render_with_user, Pleroma.Web.StreamerView, "update.json", ^announce}
+ assert_receive {:render_with_user, Pleroma.Web.StreamerView, "update.json", ^announce, _}
refute Streamer.filtered_by_user?(user, announce)
end
@@ -322,7 +322,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user", user, oauth_token)
Streamer.stream("user", notify)
- assert_receive {:render_with_user, _, _, ^notify}
+ assert_receive {:render_with_user, _, _, ^notify, _}
refute Streamer.filtered_by_user?(user, notify)
end
@@ -334,7 +334,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
Streamer.stream("user:notification", notify)
- assert_receive {:render_with_user, _, _, ^notify}
+ assert_receive {:render_with_user, _, _, ^notify, _}
refute Streamer.filtered_by_user?(user, notify)
end
@@ -355,7 +355,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user:pleroma_chat", user, oauth_token)
Streamer.stream("user:pleroma_chat", {user, cm_ref})
- text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref})
+ text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref}, "user:pleroma_chat:#{user.id}")
assert text =~ "hey cirno"
assert_receive {:text, ^text}
@@ -373,7 +373,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user", user, oauth_token)
Streamer.stream("user", {user, cm_ref})
- text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref})
+ text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref}, "user:#{user.id}")
assert text =~ "hey cirno"
assert_receive {:text, ^text}
@@ -394,7 +394,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
Streamer.stream("user:notification", notify)
- assert_receive {:render_with_user, _, _, ^notify}
+ assert_receive {:render_with_user, _, _, ^notify, _}
refute Streamer.filtered_by_user?(user, notify)
end
@@ -440,7 +440,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
{:ok, favorite_activity} = CommonAPI.favorite(user2, activity.id)
- assert_receive {:render_with_user, _, "notification.json", notif}
+ assert_receive {:render_with_user, _, "notification.json", notif, _}
assert notif.activity.id == favorite_activity.id
refute Streamer.filtered_by_user?(user, notif)
end
@@ -469,7 +469,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
{:ok, _follower, _followed, follow_activity} = CommonAPI.follow(user2, user)
- assert_receive {:render_with_user, _, "notification.json", notif}
+ assert_receive {:render_with_user, _, "notification.json", notif, _}
assert notif.activity.id == follow_activity.id
refute Streamer.filtered_by_user?(user, notif)
end
@@ -534,7 +534,7 @@ defmodule Pleroma.Web.StreamerTest do
{:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew"})
create = Pleroma.Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"])
- assert_receive {:render_with_user, _, "status_update.json", ^create}
+ assert_receive {:render_with_user, _, "status_update.json", ^create, _}
refute Streamer.filtered_by_user?(user, edited)
end
@@ -545,7 +545,7 @@ defmodule Pleroma.Web.StreamerTest do
{:ok, edited} = CommonAPI.update(user, activity, %{status: "mew mew"})
create = Pleroma.Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"])
- assert_receive {:render_with_user, _, "status_update.json", ^create}
+ assert_receive {:render_with_user, _, "status_update.json", ^create, _}
refute Streamer.filtered_by_user?(user, edited)
end
end
@@ -558,7 +558,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("public", user, oauth_token)
{:ok, activity} = CommonAPI.post(other_user, %{status: "Test"})
- assert_receive {:render_with_user, _, _, ^activity}
+ assert_receive {:render_with_user, _, _, ^activity, _}
refute Streamer.filtered_by_user?(other_user, activity)
end
@@ -658,7 +658,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("public", user, oauth_token)
Streamer.stream("public", activity)
- assert_receive {:render_with_user, _, _, ^activity}
+ assert_receive {:render_with_user, _, _, ^activity, _}
assert Streamer.filtered_by_user?(user, activity)
end
@@ -680,7 +680,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("public", user, oauth_token)
Streamer.stream("public", activity)
- assert_receive {:render_with_user, _, _, ^activity}
+ assert_receive {:render_with_user, _, _, ^activity, _}
refute Streamer.filtered_by_user?(user, activity)
end
@@ -703,7 +703,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("public", user, oauth_token)
Streamer.stream("public", activity)
- assert_receive {:render_with_user, _, _, ^activity}
+ assert_receive {:render_with_user, _, _, ^activity, _}
refute Streamer.filtered_by_user?(user, activity)
end
end
@@ -717,7 +717,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("public", user, oauth_token)
{:ok, activity} = CommonAPI.post(blocked_user, %{status: "Test"})
- assert_receive {:render_with_user, _, _, ^activity}
+ assert_receive {:render_with_user, _, _, ^activity, _}
assert Streamer.filtered_by_user?(user, activity)
end
@@ -734,17 +734,17 @@ defmodule Pleroma.Web.StreamerTest do
{:ok, activity_one} = CommonAPI.post(friend, %{status: "hey! @#{blockee.nickname}"})
- assert_receive {:render_with_user, _, _, ^activity_one}
+ assert_receive {:render_with_user, _, _, ^activity_one, _}
assert Streamer.filtered_by_user?(blocker, activity_one)
{:ok, activity_two} = CommonAPI.post(blockee, %{status: "hey! @#{friend.nickname}"})
- assert_receive {:render_with_user, _, _, ^activity_two}
+ assert_receive {:render_with_user, _, _, ^activity_two, _}
assert Streamer.filtered_by_user?(blocker, activity_two)
{:ok, activity_three} = CommonAPI.post(blockee, %{status: "hey! @#{blocker.nickname}"})
- assert_receive {:render_with_user, _, _, ^activity_three}
+ assert_receive {:render_with_user, _, _, ^activity_three, _}
assert Streamer.filtered_by_user?(blocker, activity_three)
end
end
@@ -805,7 +805,7 @@ defmodule Pleroma.Web.StreamerTest do
visibility: "private"
})
- assert_receive {:render_with_user, _, _, ^activity}
+ assert_receive {:render_with_user, _, _, ^activity, _}
refute Streamer.filtered_by_user?(user_a, activity)
end
end
@@ -823,7 +823,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user", user1, user1_token)
{:ok, announce_activity} = CommonAPI.repeat(create_activity.id, user2)
- assert_receive {:render_with_user, _, _, ^announce_activity}
+ assert_receive {:render_with_user, _, _, ^announce_activity, _}
assert Streamer.filtered_by_user?(user1, announce_activity)
end
@@ -839,7 +839,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user", user1, user1_token)
{:ok, _announce_activity} = CommonAPI.repeat(create_activity.id, user2)
- assert_receive {:render_with_user, _, "notification.json", notif}
+ assert_receive {:render_with_user, _, "notification.json", notif, _}
assert Streamer.filtered_by_user?(user1, notif)
end
@@ -855,7 +855,7 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user", user1, user1_token)
{:ok, _favorite_activity} = CommonAPI.favorite(user2, create_activity.id)
- assert_receive {:render_with_user, _, "notification.json", notif}
+ assert_receive {:render_with_user, _, "notification.json", notif, _}
refute Streamer.filtered_by_user?(user1, notif)
end
end
@@ -870,7 +870,7 @@ defmodule Pleroma.Web.StreamerTest do
{:ok, activity} = CommonAPI.post(user, %{status: "super hot take"})
{:ok, _} = CommonAPI.add_mute(user2, activity)
- assert_receive {:render_with_user, _, _, ^activity}
+ assert_receive {:render_with_user, _, _, ^activity, _}
assert Streamer.filtered_by_user?(user2, activity)
end
end
@@ -912,7 +912,7 @@ defmodule Pleroma.Web.StreamerTest do
})
create_activity_id = create_activity.id
- assert_receive {:render_with_user, _, _, ^create_activity}
+ assert_receive {:render_with_user, _, _, ^create_activity, _}
assert_receive {:text, received_conversation1}
assert %{"event" => "conversation", "payload" => _} = Jason.decode!(received_conversation1)
@@ -947,8 +947,8 @@ defmodule Pleroma.Web.StreamerTest do
visibility: "direct"
})
- assert_receive {:render_with_user, _, _, ^create_activity}
- assert_receive {:render_with_user, _, _, ^create_activity2}
+ assert_receive {:render_with_user, _, _, ^create_activity, _}
+ assert_receive {:render_with_user, _, _, ^create_activity2, _}
assert_receive {:text, received_conversation1}
assert %{"event" => "conversation", "payload" => _} = Jason.decode!(received_conversation1)
assert_receive {:text, received_conversation1}
@@ -977,7 +977,7 @@ defmodule Pleroma.Web.StreamerTest do
receive do
{StreamerTest, :ready} ->
- assert_receive {:render_with_user, _, "update.json", _}
+ assert_receive {:render_with_user, _, "update.json", _, _}
receive do
{StreamerTest, :revoked} -> finalize.()
From a348c2e4dd0551a603509501d718d9e0b995be90 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 01:29:11 -0400
Subject: [PATCH 197/823] Use pleroma: instead of pleroma. for ws events
---
.../web/mastodon_api/websocket_handler.ex | 8 ++--
lib/pleroma/web/views/streamer_view.ex | 2 +-
.../integration/mastodon_websocket_test.exs | 46 ++++++++++---------
test/pleroma/web/streamer_test.exs | 14 +++++-
4 files changed, 42 insertions(+), 28 deletions(-)
diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index 2707673ba..07c2b62e3 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -215,7 +215,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
end
defp handle_client_event(
- %{"type" => "pleroma.authenticate", "token" => access_token} = _params,
+ %{"type" => "pleroma:authenticate", "token" => access_token} = _params,
state
) do
with {:auth, nil, nil} <- {:auth, state.user, state.oauth_token},
@@ -223,7 +223,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
{[
{:text,
StreamerView.render("pleroma_respond.json", %{
- type: "pleroma.authenticate",
+ type: "pleroma:authenticate",
result: "success"
})}
], %{state | user: user, oauth_token: oauth_token}}
@@ -232,7 +232,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
{[
{:text,
StreamerView.render("pleroma_respond.json", %{
- type: "pleroma.authenticate",
+ type: "pleroma:authenticate",
result: "error",
error: :already_authenticated
})}
@@ -242,7 +242,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
{[
{:text,
StreamerView.render("pleroma_respond.json", %{
- type: "pleroma.authenticate",
+ type: "pleroma:authenticate",
result: "error",
error: :unauthorized
})}
diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex
index f591da9a6..f97570b0a 100644
--- a/lib/pleroma/web/views/streamer_view.ex
+++ b/lib/pleroma/web/views/streamer_view.ex
@@ -148,7 +148,7 @@ defmodule Pleroma.Web.StreamerView do
def render("pleroma_respond.json", %{type: type, result: result} = params) do
%{
- event: "pleroma.respond",
+ event: "pleroma:respond",
payload:
%{
result: result,
diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs
index d2e98df3b..21e7ca2b0 100644
--- a/test/pleroma/integration/mastodon_websocket_test.exs
+++ b/test/pleroma/integration/mastodon_websocket_test.exs
@@ -95,7 +95,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
assert {:ok,
%{
- "event" => "pleroma.respond",
+ "event" => "pleroma:respond",
"payload" => %{"type" => "subscribe", "result" => "success"}
}} = decode_json(raw_json)
@@ -124,16 +124,20 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
assert {:ok,
%{
- "event" => "pleroma.respond",
+ "event" => "pleroma:respond",
"payload" => %{"type" => "subscribe", "result" => "success"}
}} = decode_json(raw_json)
- WebsocketClient.send_text(pid, %{type: "subscribe", stream: "hashtag", tag: "mew"} |> Jason.encode!())
+ WebsocketClient.send_text(
+ pid,
+ %{type: "subscribe", stream: "hashtag", tag: "mew"} |> Jason.encode!()
+ )
+
assert_receive {:text, raw_json}, 1_000
assert {:ok,
%{
- "event" => "pleroma.respond",
+ "event" => "pleroma:respond",
"payload" => %{"type" => "subscribe", "result" => "success"}
}} = decode_json(raw_json)
@@ -157,7 +161,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
assert {:ok,
%{
- "event" => "pleroma.respond",
+ "event" => "pleroma:respond",
"payload" => %{"type" => "subscribe", "result" => "success"}
}} = decode_json(raw_json)
@@ -166,7 +170,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
assert {:ok,
%{
- "event" => "pleroma.respond",
+ "event" => "pleroma:respond",
"payload" => %{"type" => "subscribe", "result" => "ignored"}
}} = decode_json(raw_json)
@@ -184,7 +188,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
assert {:ok,
%{
- "event" => "pleroma.respond",
+ "event" => "pleroma:respond",
"payload" => %{"type" => "subscribe", "result" => "success"}
}} = decode_json(raw_json)
@@ -193,7 +197,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
assert {:ok,
%{
- "event" => "pleroma.respond",
+ "event" => "pleroma:respond",
"payload" => %{"type" => "unsubscribe", "result" => "success"}
}} = decode_json(raw_json)
@@ -262,15 +266,15 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
WebsocketClient.send_text(
pid,
- %{type: "pleroma.authenticate", token: token.token} |> Jason.encode!()
+ %{type: "pleroma:authenticate", token: token.token} |> Jason.encode!()
)
assert_receive {:text, raw_json}, 1_000
assert {:ok,
%{
- "event" => "pleroma.respond",
- "payload" => %{"type" => "pleroma.authenticate", "result" => "success"}
+ "event" => "pleroma:respond",
+ "payload" => %{"type" => "pleroma:authenticate", "result" => "success"}
}} = decode_json(raw_json)
WebsocketClient.send_text(pid, %{type: "subscribe", stream: "user"} |> Jason.encode!())
@@ -278,7 +282,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
assert {:ok,
%{
- "event" => "pleroma.respond",
+ "event" => "pleroma:respond",
"payload" => %{"type" => "subscribe", "result" => "success"}
}} = decode_json(raw_json)
end
@@ -288,16 +292,16 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
WebsocketClient.send_text(
pid,
- %{type: "pleroma.authenticate", token: "Something else"} |> Jason.encode!()
+ %{type: "pleroma:authenticate", token: "Something else"} |> Jason.encode!()
)
assert_receive {:text, raw_json}, 1_000
assert {:ok,
%{
- "event" => "pleroma.respond",
+ "event" => "pleroma:respond",
"payload" => %{
- "type" => "pleroma.authenticate",
+ "type" => "pleroma:authenticate",
"result" => "error",
"error" => "unauthorized"
}
@@ -309,29 +313,29 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
WebsocketClient.send_text(
pid,
- %{type: "pleroma.authenticate", token: token.token} |> Jason.encode!()
+ %{type: "pleroma:authenticate", token: token.token} |> Jason.encode!()
)
assert_receive {:text, raw_json}, 1_000
assert {:ok,
%{
- "event" => "pleroma.respond",
- "payload" => %{"type" => "pleroma.authenticate", "result" => "success"}
+ "event" => "pleroma:respond",
+ "payload" => %{"type" => "pleroma:authenticate", "result" => "success"}
}} = decode_json(raw_json)
WebsocketClient.send_text(
pid,
- %{type: "pleroma.authenticate", token: "Something else"} |> Jason.encode!()
+ %{type: "pleroma:authenticate", token: "Something else"} |> Jason.encode!()
)
assert_receive {:text, raw_json}, 1_000
assert {:ok,
%{
- "event" => "pleroma.respond",
+ "event" => "pleroma:respond",
"payload" => %{
- "type" => "pleroma.authenticate",
+ "type" => "pleroma:authenticate",
"result" => "error",
"error" => "already_authenticated"
}
diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs
index cc57a2989..d85358fd4 100644
--- a/test/pleroma/web/streamer_test.exs
+++ b/test/pleroma/web/streamer_test.exs
@@ -355,7 +355,12 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user:pleroma_chat", user, oauth_token)
Streamer.stream("user:pleroma_chat", {user, cm_ref})
- text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref}, "user:pleroma_chat:#{user.id}")
+ text =
+ StreamerView.render(
+ "chat_update.json",
+ %{chat_message_reference: cm_ref},
+ "user:pleroma_chat:#{user.id}"
+ )
assert text =~ "hey cirno"
assert_receive {:text, ^text}
@@ -373,7 +378,12 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user", user, oauth_token)
Streamer.stream("user", {user, cm_ref})
- text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref}, "user:#{user.id}")
+ text =
+ StreamerView.render(
+ "chat_update.json",
+ %{chat_message_reference: cm_ref},
+ "user:#{user.id}"
+ )
assert text =~ "hey cirno"
assert_receive {:text, ^text}
From 2d430679468a4c6a9b5c365a53f007cfa28679d9 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 02:24:30 -0400
Subject: [PATCH 198/823] Document the streaming endpoint
---
.../API/differences_in_mastoapi_responses.md | 116 ++++++++++++++++++
1 file changed, 116 insertions(+)
diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md
index 4007c63c8..ef6b3b3be 100644
--- a/docs/development/API/differences_in_mastoapi_responses.md
+++ b/docs/development/API/differences_in_mastoapi_responses.md
@@ -357,6 +357,122 @@ The message payload consist of:
- `follower_count`: follower count
- `following_count`: following count
+### Authenticating via `sec-websocket-protocol` header
+
+Pleroma allows to authenticate via the `sec-websocket-protocol` header, for example, if your access token is `your-access-token`, you can authenticate using the following:
+
+```
+sec-websocket-protocol: your-access-token
+```
+
+### Authenticating after connection via `pleroma:authenticate` event
+
+Pleroma allows to authenticate after connection is established, via the `pleroma:authenticate` event. For example, if your access token is `your-access-token`, you can send the following after the connection is established:
+
+```
+{"type": "pleroma:authenticate", "token": "your-access-token"}
+```
+
+### Response to client-sent events
+
+Pleroma will respond to client-sent events that it recognizes. Supported event types are:
+
+- `subscribe`
+- `unsubscribe`
+- `pleroma:authenticate`
+
+The reply will be in the following format:
+
+```
+{
+ "event": "pleroma:respond",
+ "payload": "{\"type\": \"\", \"result\": \"\", \"error\": \"\"}"
+}
+```
+
+Result of the action can be either `success`, `ignored` or `error`. If it is `error`, the `error` property will contain the error code. Otherwise, the `error` property will not be present. Below are some examples:
+
+```
+{
+ "event": "pleroma:respond",
+ "payload": "{\"type\": \"pleroma:authenticate\", \"result\": \"success\"}"
+}
+
+{
+ "event": "pleroma:respond",
+ "payload": "{\"type\": \"subscribe\", \"result\": \"ignored\"}"
+}
+
+{
+ "event": "pleroma:respond",
+ "payload": "{\"type\": \"unsubscribe\", \"result\": \"error\", \"error\": \"bad_topic\"}"
+}
+```
+
+If the sent event is not of a type that Pleroma supports, it will not reply.
+
+### The `stream` attribute of a server-sent event
+
+Technically, this is in Mastodon, but its documentation does nothing to specify its format.
+
+This attribute appears on every event type except `pleroma:respond` and `delete`. It helps clients determine where they should display the new statuses.
+
+The value of the attribute is an array containing one or two elements. The first element is the type of the stream. The second is the identifier related to that specific stream, if applicable.
+
+For the following stream types, there is a second element in the array:
+
+- `list`: The second element is the id of the list.
+- `hashtag`: The second element is the name of the hashtag.
+- `public:remote:media` and `public:remote`: The second element is the domain of the corresponding instance.
+
+For all other stream types, there is no second element.
+
+Some examples of valid `stream` values:
+
+- `list:1`: List of id 1.
+- `hashtag:mew`: The hashtag #mew.
+- `user:notifications`: Notifications for the current user.
+- `user`: Home timeline.
+- `public:remote:mew.moe`: Public posts from the instance mew.moe .
+
+### The unified streaming endpoint
+
+If you do not specify a stream to connect to when requesting `/api/v1/streaming`, you will enter a connection that subscribes to no streams. After the connection is established, you can authenticate and then subscribe to different streams.
+
+### List of supported streams
+
+Below is a list of supported streams by Pleroma. To make a single-stream WebSocket connection, append the string specified in "Query style" to the streaming endpoint url.
+To subscribe to a stream after the connection is established, merge the JSON object specified in "Subscribe style" with `{"type": "subscribe"}`. To unsubscribe, merge it with `{"type": "unsubscribe"}`.
+
+For example, to receive updates on the list 1, you can connect to `/api/v1/streaming/?stream=list&list=1`, or send
+
+```
+{"type": "subscribe", "stream": "list", "list": 1}
+```
+
+upon establishing the websocket connection.
+
+To unsubscribe to list 1, send
+
+```
+{"type": "unsubscribe", "stream": "list", "list": 1}
+```
+
+Note that if you specify a stream that requires a logged-in user in the query string (for example, `user` or `list`), you have to specify the access token when you are trying to establish the connection, i.e. in the query string or via the `sec-websocket-protocol` header.
+
+- `list`
+ - Query style: `?stream=list&list=`
+ - Subscribe style: `{"stream": "list", "list": }`
+- `public`, `public:local`, `public:media`, `public:local:media`, `user`, `user:pleroma_chat`, `user:notifications`, `direct`
+ - Query style: `?stream=`
+ - Subscribe style: `{"stream": ""}`
+- `hashtag`
+ - Query style: `?stream=hashtag&tag=`
+ - Subscribe style: `{"stream": "hashtag", "tag": ""}`
+- `public:remote`, `public:remote:media`
+ - Query style: `?stream=&instance=`
+ - Subscribe style: `{"stream": "", "instance": ""}`
+
## User muting and thread muting
Both user muting and thread muting can be done for only a certain time by adding an `expires_in` parameter to the API calls and giving the expiration time in seconds.
From 9572be1e5fe0201cd069271c7025ef233e8ddf00 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 02:35:12 -0400
Subject: [PATCH 199/823] Add tests for list streams
---
.../integration/mastodon_websocket_test.exs | 37 +++++++++++++++++++
1 file changed, 37 insertions(+)
diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs
index 21e7ca2b0..8129645ec 100644
--- a/test/pleroma/integration/mastodon_websocket_test.exs
+++ b/test/pleroma/integration/mastodon_websocket_test.exs
@@ -342,6 +342,43 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
}} = decode_json(raw_json)
end
+ test "accepts the 'list' stream", %{token: token, user: user} do
+ posting_user = insert(:user)
+
+ {:ok, list} = Pleroma.List.create("test", user)
+ Pleroma.List.follow(list, posting_user)
+
+ assert {:ok, _} = start_socket("?stream=list&access_token=#{token.token}&list=#{list.id}")
+
+ assert {:ok, pid} = start_socket("?access_token=#{token.token}")
+
+ WebsocketClient.send_text(
+ pid,
+ %{type: "subscribe", stream: "list", list: list.id} |> Jason.encode!()
+ )
+
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma:respond",
+ "payload" => %{"type" => "subscribe", "result" => "success"}
+ }} = decode_json(raw_json)
+
+ WebsocketClient.send_text(
+ pid,
+ %{type: "subscribe", stream: "list", list: to_string(list.id)} |> Jason.encode!()
+ )
+
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma:respond",
+ "payload" => %{"type" => "subscribe", "result" => "ignored"}
+ }} = decode_json(raw_json)
+ end
+
test "disconnect when token is revoked", %{app: app, user: user, token: token} do
assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}")
assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}")
From 7f12ad6dccfe4c81fa7e0d4e66c43bedadbf4c6a Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 02:37:19 -0400
Subject: [PATCH 200/823] Fix docs wording
---
.../API/differences_in_mastoapi_responses.md | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md
index ef6b3b3be..48a9c104c 100644
--- a/docs/development/API/differences_in_mastoapi_responses.md
+++ b/docs/development/API/differences_in_mastoapi_responses.md
@@ -421,7 +421,7 @@ The value of the attribute is an array containing one or two elements. The first
For the following stream types, there is a second element in the array:
-- `list`: The second element is the id of the list.
+- `list`: The second element is the id of the list, as a string.
- `hashtag`: The second element is the name of the hashtag.
- `public:remote:media` and `public:remote`: The second element is the domain of the corresponding instance.
@@ -429,11 +429,11 @@ For all other stream types, there is no second element.
Some examples of valid `stream` values:
-- `list:1`: List of id 1.
-- `hashtag:mew`: The hashtag #mew.
-- `user:notifications`: Notifications for the current user.
-- `user`: Home timeline.
-- `public:remote:mew.moe`: Public posts from the instance mew.moe .
+- `["list", "1"]`: List of id 1.
+- `["hashtag", "mew"]`: The hashtag #mew.
+- `["user:notifications"]`: Notifications for the current user.
+- `["user"]`: Home timeline.
+- `["public:remote", "mew.moe"]`: Public posts from the instance mew.moe .
### The unified streaming endpoint
@@ -447,7 +447,7 @@ To subscribe to a stream after the connection is established, merge the JSON obj
For example, to receive updates on the list 1, you can connect to `/api/v1/streaming/?stream=list&list=1`, or send
```
-{"type": "subscribe", "stream": "list", "list": 1}
+{"type": "subscribe", "stream": "list", "list": "1"}
```
upon establishing the websocket connection.
@@ -455,14 +455,14 @@ upon establishing the websocket connection.
To unsubscribe to list 1, send
```
-{"type": "unsubscribe", "stream": "list", "list": 1}
+{"type": "unsubscribe", "stream": "list", "list": "1"}
```
Note that if you specify a stream that requires a logged-in user in the query string (for example, `user` or `list`), you have to specify the access token when you are trying to establish the connection, i.e. in the query string or via the `sec-websocket-protocol` header.
- `list`
- Query style: `?stream=list&list=`
- - Subscribe style: `{"stream": "list", "list": }`
+ - Subscribe style: `{"stream": "list", "list": ""}`
- `public`, `public:local`, `public:media`, `public:local:media`, `user`, `user:pleroma_chat`, `user:notifications`, `direct`
- Query style: `?stream=`
- Subscribe style: `{"stream": ""}`
From 949c4f01c6d5686ff1852e6439616c2069557ef0 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 02:38:16 -0400
Subject: [PATCH 201/823] Fix NotificationTest
---
test/pleroma/notification_test.exs | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs
index e55aa3a08..71af9acb8 100644
--- a/test/pleroma/notification_test.exs
+++ b/test/pleroma/notification_test.exs
@@ -252,7 +252,7 @@ defmodule Pleroma.NotificationTest do
task =
Task.async(fn ->
{:ok, _topic} = Streamer.get_topic_and_add_socket("user", user, oauth_token)
- assert_receive {:render_with_user, _, _, _}, 4_000
+ assert_receive {:render_with_user, _, _, _, _}, 4_000
end)
task_user_notification =
@@ -260,7 +260,7 @@ defmodule Pleroma.NotificationTest do
{:ok, _topic} =
Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
- assert_receive {:render_with_user, _, _, _}, 4_000
+ assert_receive {:render_with_user, _, _, _, _}, 4_000
end)
activity = insert(:note_activity)
From eebc605bc25deead55c305d703c06ddb9d9b1107 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 08:27:43 -0400
Subject: [PATCH 202/823] Clear up debug statement
---
lib/pleroma/web/streamer.ex | 1 -
1 file changed, 1 deletion(-)
diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex
index 70e46617a..48ca82421 100644
--- a/lib/pleroma/web/streamer.ex
+++ b/lib/pleroma/web/streamer.ex
@@ -357,7 +357,6 @@ defmodule Pleroma.Web.Streamer do
end
defp push_to_socket(topic, item) do
- Logger.debug("topic=#{topic}")
anon_render = StreamerView.render("update.json", item, topic)
Registry.dispatch(@registry, topic, fn list ->
From 050227f11898f402f5888d53e6460b704bcd0a8b Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 08:39:38 -0400
Subject: [PATCH 203/823] Add test to cover error: bad_topic
---
test/pleroma/integration/mastodon_websocket_test.exs | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs
index 8129645ec..0e8413749 100644
--- a/test/pleroma/integration/mastodon_websocket_test.exs
+++ b/test/pleroma/integration/mastodon_websocket_test.exs
@@ -180,6 +180,18 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
refute_receive {:text, _}, 1_000
end
+ test "rejects invalid streams" do
+ {:ok, pid} = start_socket()
+ WebsocketClient.send_text(pid, %{type: "subscribe", stream: "nonsense"} |> Jason.encode!())
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "pleroma:respond",
+ "payload" => %{"type" => "subscribe", "result" => "error", "error" => "bad_topic"}
+ }} = decode_json(raw_json)
+ end
+
test "can unsubscribe" do
user = insert(:user)
{:ok, pid} = start_socket()
From 4cf109d3c4093d788e6b2de229a9e4034146a5be Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 08:47:46 -0400
Subject: [PATCH 204/823] Add test to cover rendering update with user
---
.../integration/mastodon_websocket_test.exs | 27 +++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs
index 0e8413749..5ec681ce3 100644
--- a/test/pleroma/integration/mastodon_websocket_test.exs
+++ b/test/pleroma/integration/mastodon_websocket_test.exs
@@ -406,5 +406,32 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
assert_receive {:close, _}
refute_receive {:close, _}
end
+
+ test "receives private statuses", %{user: reading_user, token: token} do
+ user = insert(:user)
+ CommonAPI.follow(reading_user, user)
+
+ {:ok, _} = start_socket("?stream=user&access_token=#{token.token}")
+
+ {:ok, activity} =
+ CommonAPI.post(user, %{status: "nice echo chamber", visibility: "private"})
+
+ assert_receive {:text, raw_json}, 1_000
+ assert {:ok, json} = Jason.decode(raw_json)
+
+ assert "update" == json["event"]
+ assert json["payload"]
+ assert {:ok, json} = Jason.decode(json["payload"])
+
+ view_json =
+ Pleroma.Web.MastodonAPI.StatusView.render("show.json",
+ activity: activity,
+ for: reading_user
+ )
+ |> Jason.encode!()
+ |> Jason.decode!()
+
+ assert json == view_json
+ end
end
end
From 26f5caebae1581f1dcdc6d1352840d27009e061c Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 08:56:53 -0400
Subject: [PATCH 205/823] Add test to cover notifications streaming
---
.../integration/mastodon_websocket_test.exs | 25 +++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs
index 5ec681ce3..96648264e 100644
--- a/test/pleroma/integration/mastodon_websocket_test.exs
+++ b/test/pleroma/integration/mastodon_websocket_test.exs
@@ -433,5 +433,30 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
assert json == view_json
end
+
+ test "receives notifications", %{user: reading_user, token: token} do
+ user = insert(:user)
+ CommonAPI.follow(reading_user, user)
+
+ {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}")
+
+ {:ok, %Pleroma.Activity{id: activity_id} = _activity} =
+ CommonAPI.post(user, %{
+ status: "nice echo chamber @#{reading_user.nickname}",
+ visibility: "private"
+ })
+
+ assert_receive {:text, raw_json}, 1_000
+
+ assert {:ok,
+ %{
+ "event" => "notification",
+ "payload" => %{
+ "status" => %{
+ "id" => ^activity_id
+ }
+ }
+ }} = decode_json(raw_json)
+ end
end
end
From 314360e5e3a4bdd13f152e94050cb8562e2ea0ed Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 09:02:43 -0400
Subject: [PATCH 206/823] Add test to cover edit streaming
---
.../integration/mastodon_websocket_test.exs | 28 +++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs
index 96648264e..7d5dfc255 100644
--- a/test/pleroma/integration/mastodon_websocket_test.exs
+++ b/test/pleroma/integration/mastodon_websocket_test.exs
@@ -434,6 +434,34 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
assert json == view_json
end
+ test "receives edits", %{user: reading_user, token: token} do
+ user = insert(:user)
+ CommonAPI.follow(reading_user, user)
+
+ {:ok, _} = start_socket("?stream=user&access_token=#{token.token}")
+
+ {:ok, activity} =
+ CommonAPI.post(user, %{status: "nice echo chamber", visibility: "private"})
+
+ assert_receive {:text, _raw_json}, 1_000
+
+ {:ok, _} = CommonAPI.update(user, activity, %{status: "mew mew", visibility: "private"})
+
+ assert_receive {:text, raw_json}, 1_000
+
+ activity = Pleroma.Activity.normalize(activity)
+
+ view_json =
+ Pleroma.Web.MastodonAPI.StatusView.render("show.json",
+ activity: activity,
+ for: reading_user
+ )
+ |> Jason.encode!()
+ |> Jason.decode!()
+
+ assert {:ok, %{"event" => "status.update", "payload" => ^view_json}} = decode_json(raw_json)
+ end
+
test "receives notifications", %{user: reading_user, token: token} do
user = insert(:user)
CommonAPI.follow(reading_user, user)
From 844d1a14e0b4aabbb61a6693fa6dd3a0aa0dbc5b Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 15:31:56 -0400
Subject: [PATCH 207/823] Start writing api docs for streaming endpoint
---
lib/pleroma/web/api_spec.ex | 10 +-
.../operations/streaming_operation.ex | 186 ++++++++++++++++++
2 files changed, 195 insertions(+), 1 deletion(-)
create mode 100644 lib/pleroma/web/api_spec/operations/streaming_operation.ex
diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex
index 2d56dc643..163226ce5 100644
--- a/lib/pleroma/web/api_spec.ex
+++ b/lib/pleroma/web/api_spec.ex
@@ -10,6 +10,14 @@ defmodule Pleroma.Web.ApiSpec do
@behaviour OpenApi
+ defp streaming_paths do
+ %{
+ "/api/v1/streaming" => %OpenApiSpex.PathItem{
+ get: Pleroma.Web.ApiSpec.StreamingOperation.streaming_operation()
+ }
+ }
+ end
+
@impl OpenApi
def spec(opts \\ []) do
%OpenApi{
@@ -45,7 +53,7 @@ defmodule Pleroma.Web.ApiSpec do
}
},
# populate the paths from a phoenix router
- paths: OpenApiSpex.Paths.from_router(Router),
+ paths: Map.merge(streaming_paths(), OpenApiSpex.Paths.from_router(Router)),
components: %OpenApiSpex.Components{
parameters: %{
"accountIdOrNickname" =>
diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex
new file mode 100644
index 000000000..1ef7d72ef
--- /dev/null
+++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex
@@ -0,0 +1,186 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.StreamingOperation do
+ alias OpenApiSpex.Operation
+ alias OpenApiSpex.Response
+ alias OpenApiSpex.Schema
+ alias Pleroma.Web.ApiSpec.Helpers
+ alias Pleroma.Web.ApiSpec.Schemas.Status
+
+ @spec open_api_operation(atom) :: Operation.t()
+ def open_api_operation(action) do
+ operation = String.to_existing_atom("#{action}_operation")
+ apply(__MODULE__, operation, [])
+ end
+
+ @spec streaming_operation() :: Operation.t()
+ def streaming_operation do
+ %Operation{
+ tags: ["Timelines"],
+ summary: "Establish streaming connection",
+ description: "Receive statuses in real-time via WebSocket.",
+ security: [%{"oAuth" => ["read:statuses", "read:notifications"]}],
+ parameters: [
+ Operation.parameter(:connection, :header, %Schema{type: :string}, "connection header",
+ required: true
+ ),
+ Operation.parameter(:upgrade, :header, %Schema{type: :string}, "upgrade header",
+ required: true
+ ),
+ Operation.parameter(
+ :"sec-websocket-key",
+ :header,
+ %Schema{type: :string},
+ "sec-websocket-key header",
+ required: true
+ ),
+ Operation.parameter(
+ :"sec-websocket-version",
+ :header,
+ %Schema{type: :string},
+ "sec-websocket-version header",
+ required: true
+ )
+ ],
+ responses: %{
+ 101 => switching_protocols_response(),
+ 200 =>
+ Operation.response(
+ "Server-sent events",
+ "application/json",
+ server_sent_events()
+ )
+ }
+ }
+ end
+
+ defp switching_protocols_response do
+ %Response{
+ description: "Switching protocols",
+ headers: %{
+ "connection" => %OpenApiSpex.Header{required: true},
+ "upgrade" => %OpenApiSpex.Header{required: true},
+ "sec-websocket-accept" => %OpenApiSpex.Header{required: true}
+ }
+ }
+ end
+
+ defp server_sent_events do
+ %Schema{
+ oneOf: [
+ update_event(),
+ status_update_event(),
+ pleroma_respond_event()
+ ]
+ }
+ end
+
+ defp stream do
+ %Schema{
+ type: :array,
+ title: "Stream",
+ description: """
+ The stream identifier.
+ The first item is the name of the stream. If the stream needs a differentiator, the second item will be the corresponding identifier.
+ Currently, for the following stream types, there is a second element in the array:
+
+ - `list`: The second element is the id of the list, as a string.
+ - `hashtag`: The second element is the name of the hashtag.
+ - `public:remote:media` and `public:remote`: The second element is the domain of the corresponding instance.
+ """,
+ maxItems: 2,
+ minItems: 1,
+ items: %Schema{type: :string},
+ example: ["hashtag", "mew"]
+ }
+ end
+
+ defp get_schema(%Schema{} = schema), do: schema
+ defp get_schema(schema), do: schema.schema
+
+ defp server_sent_event_helper(name, description, type, payload, opts \\ []) do
+ payload_type = opts[:payload_type] || :json
+ has_stream = opts[:has_stream] || true
+
+ stream_properties =
+ if has_stream do
+ %{stream: stream()}
+ else
+ %{}
+ end
+
+ stream_example = if has_stream, do: %{"stream" => get_schema(stream()).example}, else: %{}
+
+ stream_required = if has_stream, do: [:stream], else: []
+
+ %Schema{
+ type: :object,
+ title: name,
+ description: description,
+ required: [:event, :payload] ++ stream_required,
+ properties:
+ %{
+ event: %Schema{
+ title: "Event type",
+ description: "Type of the event.",
+ type: :string,
+ required: true,
+ enum: [type]
+ },
+ payload:
+ if payload_type == :json do
+ %Schema{
+ title: "Event payload",
+ description: "JSON-encoded string of #{get_schema(payload).title}",
+ allOf: [payload]
+ }
+ else
+ payload
+ end
+ }
+ |> Map.merge(stream_properties),
+ example:
+ %{
+ "event" => type,
+ "payload" => get_schema(payload).example |> Jason.encode!()
+ }
+ |> Map.merge(stream_example)
+ }
+ end
+
+ defp update_event do
+ server_sent_event_helper("New status", "A newly-posted status.", "update", Status)
+ end
+
+ defp status_update_event do
+ server_sent_event_helper("Edit", "A status that was just edited", "status.update", Status)
+ end
+
+ defp pleroma_respond_event do
+ server_sent_event_helper(
+ "Server response",
+ "A response to a client-sent event.",
+ "pleroma:respond",
+ %Schema{
+ type: :object,
+ title: "Results",
+ required: [:result],
+ properties: %{
+ result: %Schema{
+ type: :string,
+ title: "Result of the request",
+ enum: ["success", "error", "ignored"]
+ },
+ error: %Schema{
+ type: :string,
+ title: "Error code",
+ description: "An error identifier. Only appears if `result` is `error`."
+ }
+ },
+ example: %{"result" => "success"}
+ }
+ )
+ end
+end
From dcef33f5f0c93883a634d12dc662b83d7ef6abfa Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 16:07:27 -0400
Subject: [PATCH 208/823] Document server-sent events of streaming
---
.../operations/streaming_operation.ex | 106 ++++++++++++++++++
1 file changed, 106 insertions(+)
diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex
index 1ef7d72ef..cdef41603 100644
--- a/lib/pleroma/web/api_spec/operations/streaming_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex
@@ -7,6 +7,10 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
alias OpenApiSpex.Response
alias OpenApiSpex.Schema
alias Pleroma.Web.ApiSpec.Helpers
+ alias Pleroma.Web.ApiSpec.NotificationOperation
+ alias Pleroma.Web.ApiSpec.Schemas.Chat
+ alias Pleroma.Web.ApiSpec.Schemas.Conversation
+ alias Pleroma.Web.ApiSpec.Schemas.FlakeID
alias Pleroma.Web.ApiSpec.Schemas.Status
@spec open_api_operation(atom) :: Operation.t()
@@ -22,6 +26,7 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
summary: "Establish streaming connection",
description: "Receive statuses in real-time via WebSocket.",
security: [%{"oAuth" => ["read:statuses", "read:notifications"]}],
+ operationId: "WebsocketHandler.streaming",
parameters: [
Operation.parameter(:connection, :header, %Schema{type: :string}, "connection header",
required: true
@@ -72,6 +77,11 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
oneOf: [
update_event(),
status_update_event(),
+ notification_event(),
+ chat_update_event(),
+ follow_relationships_update_event(),
+ conversation_event(),
+ delete_event(),
pleroma_respond_event()
]
}
@@ -158,6 +168,102 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
server_sent_event_helper("Edit", "A status that was just edited", "status.update", Status)
end
+ defp notification_event do
+ server_sent_event_helper(
+ "Notification",
+ "A new notification.",
+ "notification",
+ NotificationOperation.notification()
+ )
+ end
+
+ defp follow_relationships_update_event do
+ server_sent_event_helper(
+ "Follow relationships update",
+ "An update to follow relationships.",
+ "pleroma:follow_relationships_update",
+ %Schema{
+ type: :object,
+ title: "Follow relationships update",
+ required: [:state, :follower, :following],
+ properties: %{
+ state: %Schema{
+ type: :string,
+ description: "Follow state of the relationship.",
+ enum: ["follow_pending", "follow_accept", "follow_reject", "unfollow"]
+ },
+ follower: %Schema{
+ type: :object,
+ description: "Information about the follower.",
+ required: [:id, :follower_count, :following_count],
+ properties: %{
+ id: FlakeID,
+ follower_count: %Schema{type: :integer},
+ following_count: %Schema{type: :integer}
+ }
+ },
+ following: %Schema{
+ type: :object,
+ description: "Information about the following person.",
+ required: [:id, :follower_count, :following_count],
+ properties: %{
+ id: FlakeID,
+ follower_count: %Schema{type: :integer},
+ following_count: %Schema{type: :integer}
+ }
+ }
+ },
+ example: %{
+ "state" => "follow_pending",
+ "follower" => %{
+ "id" => "someUser1",
+ "follower_count" => 1,
+ "following_count" => 1
+ },
+ "following" => %{
+ "id" => "someUser2",
+ "follower_count" => 1,
+ "following_count" => 1
+ }
+ }
+ }
+ )
+ end
+
+ defp chat_update_event do
+ server_sent_event_helper(
+ "Chat update",
+ "A new chat message.",
+ "pleroma:chat_update",
+ Chat
+ )
+ end
+
+ defp conversation_event do
+ server_sent_event_helper(
+ "Conversation",
+ "An update about a conversation",
+ "conversation",
+ Conversation
+ )
+ end
+
+ defp delete_event do
+ server_sent_event_helper(
+ "Delete",
+ "A status that was just deleted.",
+ "delete",
+ %Schema{
+ type: :string,
+ title: "Status id",
+ description: "Id of the deleted status",
+ allOf: [FlakeID],
+ example: "some-opaque-id"
+ },
+ payload_type: :string
+ )
+ end
+
defp pleroma_respond_event do
server_sent_event_helper(
"Server response",
From 8829dcaee42b3ad1ee50f95b0586b22118771785 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 16:33:22 -0400
Subject: [PATCH 209/823] Document client-sent events in streaming
---
.../operations/streaming_operation.ex | 126 +++++++++++++++++-
1 file changed, 125 insertions(+), 1 deletion(-)
diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex
index cdef41603..18674c9a1 100644
--- a/lib/pleroma/web/api_spec/operations/streaming_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex
@@ -6,13 +6,14 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
alias OpenApiSpex.Operation
alias OpenApiSpex.Response
alias OpenApiSpex.Schema
- alias Pleroma.Web.ApiSpec.Helpers
alias Pleroma.Web.ApiSpec.NotificationOperation
alias Pleroma.Web.ApiSpec.Schemas.Chat
alias Pleroma.Web.ApiSpec.Schemas.Conversation
alias Pleroma.Web.ApiSpec.Schemas.FlakeID
alias Pleroma.Web.ApiSpec.Schemas.Status
+ require Pleroma.Constants
+
@spec open_api_operation(atom) :: Operation.t()
def open_api_operation(action) do
operation = String.to_existing_atom("#{action}_operation")
@@ -49,6 +50,7 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
required: true
)
],
+ requestBody: request_body("Client-sent events", client_sent_events()),
responses: %{
101 => switching_protocols_response(),
200 =>
@@ -289,4 +291,126 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
}
)
end
+
+ defp client_sent_events do
+ %Schema{
+ oneOf: [
+ subscribe_event(),
+ unsubscribe_event(),
+ authenticate_event()
+ ]
+ }
+ end
+
+ defp request_body(description, schema, opts \\ []) do
+ %OpenApiSpex.RequestBody{
+ description: description,
+ content: %{
+ "application/json" => %OpenApiSpex.MediaType{
+ schema: schema,
+ example: opts[:example],
+ examples: opts[:examples]
+ }
+ }
+ }
+ end
+
+ defp client_sent_event_helper(name, description, type, properties, opts) do
+ required = opts[:required] || []
+
+ %Schema{
+ type: :object,
+ title: name,
+ required: [:type] ++ required,
+ description: description,
+ properties:
+ %{
+ type: %Schema{type: :string, enum: [type], description: "Type of the event."}
+ }
+ |> Map.merge(properties),
+ example: opts[:example]
+ }
+ end
+
+ defp subscribe_event do
+ client_sent_event_helper(
+ "Subscribe",
+ "Subscribe to a stream.",
+ "subscribe",
+ stream_specifier(),
+ required: [:stream],
+ example: %{"type" => "subscribe", "stream" => "list", "list" => "1"}
+ )
+ end
+
+ defp unsubscribe_event do
+ client_sent_event_helper(
+ "Unsubscribe",
+ "Unsubscribe from a stream.",
+ "subscribe",
+ stream_specifier(),
+ required: [:stream],
+ example: %{
+ "type" => "unsubscribe",
+ "stream" => "public:remote:media",
+ "instance" => "example.org"
+ }
+ )
+ end
+
+ defp authenticate_event do
+ client_sent_event_helper(
+ "Authenticate",
+ "Authenticate via an access token.",
+ "pleroma:authenticate",
+ %{
+ token: %Schema{
+ type: :string,
+ description: "An OAuth access token with corresponding permissions.",
+ example: "some token"
+ }
+ },
+ required: [:token]
+ )
+ end
+
+ defp stream_specifier do
+ %{
+ stream: %Schema{
+ type: :string,
+ description: "The name of the stream.",
+ enum:
+ Pleroma.Constants.public_streams() ++
+ [
+ "public:remote",
+ "public:remote:media",
+ "user",
+ "user:pleroma_chat",
+ "user:notification",
+ "direct",
+ "list",
+ "hashtag"
+ ]
+ },
+ list: %Schema{
+ type: :string,
+ title: "List id",
+ description: "The id of the list. Required when `stream` is `list`.",
+ example: "some-id"
+ },
+ tag: %Schema{
+ type: :string,
+ title: "Hashtag name",
+ description: "The name of the hashtag. Required when `stream` is `hashtag`.",
+ example: "mew"
+ },
+ instance: %Schema{
+ type: :string,
+ title: "Domain name",
+ description:
+ "Domain name of the instance. Required when `stream` is `public:remote` or `public:remote:media`.",
+ example: "example.org"
+ }
+ }
+ end
end
From f393a15dd1217a7f6aec9e9acc7b983e7b165a91 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 16:46:32 -0400
Subject: [PATCH 210/823] Fix some specs about server-sent events in streaming
---
.../operations/streaming_operation.ex | 49 ++++++++++++-------
1 file changed, 32 insertions(+), 17 deletions(-)
diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex
index 18674c9a1..fa48b9613 100644
--- a/lib/pleroma/web/api_spec/operations/streaming_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex
@@ -113,8 +113,8 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
defp get_schema(schema), do: schema.schema
defp server_sent_event_helper(name, description, type, payload, opts \\ []) do
- payload_type = opts[:payload_type] || :json
- has_stream = opts[:has_stream] || true
+ payload_type = Keyword.get(opts, :payload_type, :json)
+ has_stream = Keyword.get(opts, :has_stream, true)
stream_properties =
if has_stream do
@@ -127,6 +127,24 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
stream_required = if has_stream, do: [:stream], else: []
+ payload_schema =
+ if payload_type == :json do
+ %Schema{
+ title: "Event payload",
+ description: "JSON-encoded string of #{get_schema(payload).title}",
+ allOf: [payload]
+ }
+ else
+ payload
+ end
+
+ payload_example =
+ if payload_type == :json do
+ get_schema(payload).example |> Jason.encode!()
+ else
+ get_schema(payload).example
+ end
+
%Schema{
type: :object,
title: name,
@@ -141,22 +159,13 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
required: true,
enum: [type]
},
- payload:
- if payload_type == :json do
- %Schema{
- title: "Event payload",
- description: "JSON-encoded string of #{get_schema(payload).title}",
- allOf: [payload]
- }
- else
- payload
- end
+ payload: payload_schema
}
|> Map.merge(stream_properties),
example:
%{
"event" => type,
- "payload" => get_schema(payload).example |> Jason.encode!()
+ "payload" => payload_example
}
|> Map.merge(stream_example)
}
@@ -262,7 +271,8 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
allOf: [FlakeID],
example: "some-opaque-id"
},
- payload_type: :string
+ payload_type: :string,
+ has_stream: false
)
end
@@ -274,7 +284,7 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
%Schema{
type: :object,
title: "Results",
- required: [:result],
+ required: [:result, :type],
properties: %{
result: %Schema{
type: :string,
@@ -285,10 +295,15 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
type: :string,
title: "Error code",
description: "An error identifier. Only appears if `result` is `error`."
+ },
+ type: %Schema{
+ type: :string,
+ description: "Type of the request."
}
},
- example: %{"result" => "success"}
- }
+ example: %{"result" => "success", "type" => "pleroma:authenticate"}
+ },
+ has_stream: false
)
end
From c13f0a8460853d8fe9aa04cb5d57ea06b692a3bf Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 17:00:35 -0400
Subject: [PATCH 211/823] Add meta-info and query strings to streaming doc
---
.../operations/streaming_operation.ex | 89 +++++++++++++------
1 file changed, 61 insertions(+), 28 deletions(-)
diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex
index fa48b9613..4c4888d8e 100644
--- a/lib/pleroma/web/api_spec/operations/streaming_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex
@@ -25,31 +25,46 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
%Operation{
tags: ["Timelines"],
summary: "Establish streaming connection",
- description: "Receive statuses in real-time via WebSocket.",
+ description: """
+ Receive statuses in real-time via WebSocket.
+
+ You can specify the access token on the query string or through the `sec-websocket-protocol` header. Using
+ the query string to authenticate is considered unsafe and should not be used unless you have to (e.g. to maintain
+ your client's compatibility with Mastodon).
+
+ You may specify a stream on the query string. If you do so and you are connecting to a stream that requires logged-in users,
+ you must specify the access token at the time of the connection (i.e. via query string or header).
+
+ Otherwise, you have the option to authenticate after you have established the connection through client-sent events.
+
+ The "Request body" section below describes what events clients can send through WebSocket, and the "Responses" section
+ describes what events server will send through WebSocket.
+ """,
security: [%{"oAuth" => ["read:statuses", "read:notifications"]}],
operationId: "WebsocketHandler.streaming",
- parameters: [
- Operation.parameter(:connection, :header, %Schema{type: :string}, "connection header",
- required: true
- ),
- Operation.parameter(:upgrade, :header, %Schema{type: :string}, "upgrade header",
- required: true
- ),
- Operation.parameter(
- :"sec-websocket-key",
- :header,
- %Schema{type: :string},
- "sec-websocket-key header",
- required: true
- ),
- Operation.parameter(
- :"sec-websocket-version",
- :header,
- %Schema{type: :string},
- "sec-websocket-version header",
- required: true
- )
- ],
+ parameters:
+ [
+ Operation.parameter(:connection, :header, %Schema{type: :string}, "connection header",
+ required: true
+ ),
+ Operation.parameter(:upgrade, :header, %Schema{type: :string}, "upgrade header",
+ required: true
+ ),
+ Operation.parameter(
+ :"sec-websocket-key",
+ :header,
+ %Schema{type: :string},
+ "sec-websocket-key header",
+ required: true
+ ),
+ Operation.parameter(
+ :"sec-websocket-version",
+ :header,
+ %Schema{type: :string},
+ "sec-websocket-version header",
+ required: true
+ )
+ ] ++ stream_params() ++ access_token_params(),
requestBody: request_body("Client-sent events", client_sent_events()),
responses: %{
101 => switching_protocols_response(),
@@ -63,6 +78,20 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
}
end
+ defp stream_params do
+ stream_specifier()
+ |> Enum.map(fn {name, schema} ->
+ Operation.parameter(name, :query, schema, get_schema(schema).description)
+ end)
+ end
+
+ defp access_token_params do
+ [
+ Operation.parameter(:access_token, :query, token(), token().description),
+ Operation.parameter(:"sec-websocket-protocol", :header, token(), token().description)
+ ]
+ end
+
defp switching_protocols_response do
%Response{
description: "Switching protocols",
@@ -379,16 +408,20 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
"Authenticate via an access token.",
"pleroma:authenticate",
%{
- token: %Schema{
- type: :string,
- description: "An OAuth access token with corresponding permissions.",
- example: "some token"
- }
+ token: token()
},
required: [:token]
)
end
+ defp token do
+ %Schema{
+ type: :string,
+ description: "An OAuth access token with corresponding permissions.",
+ example: "some token"
+ }
+ end
+
defp stream_specifier do
%{
stream: %Schema{
From 32de0683c4069f5877722dce0b4f5a9a6825b7c7 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 17:15:58 -0400
Subject: [PATCH 212/823] Fix unsubscribe event type in streaming doc
---
lib/pleroma/web/api_spec/operations/streaming_operation.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex
index 4c4888d8e..ae3aeb4ab 100644
--- a/lib/pleroma/web/api_spec/operations/streaming_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex
@@ -391,7 +391,7 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
client_sent_event_helper(
"Unsubscribe",
"Unsubscribe from a stream.",
- "subscribe",
+ "unsubscribe",
stream_specifier(),
required: [:stream],
example: %{
From 840dd01035581a37613f695facdd99fbf6ac8319 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Apr 2023 18:33:43 -0400
Subject: [PATCH 213/823] Fix duplicated schema names
---
lib/pleroma/web/api_spec/operations/streaming_operation.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex
index ae3aeb4ab..b580bc2f0 100644
--- a/lib/pleroma/web/api_spec/operations/streaming_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex
@@ -281,7 +281,7 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do
defp conversation_event do
server_sent_event_helper(
- "Conversation",
+ "Conversation update",
"An update about a conversation",
"conversation",
Conversation
From 3e7d2e29b369535a9a942a4090cde9a21892f8c1 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 1 Jul 2023 23:07:07 -0400
Subject: [PATCH 214/823] Add changelog
---
changelog.d/unified-streaming.add | 1 +
1 file changed, 1 insertion(+)
create mode 100644 changelog.d/unified-streaming.add
diff --git a/changelog.d/unified-streaming.add b/changelog.d/unified-streaming.add
new file mode 100644
index 000000000..84821fcc8
--- /dev/null
+++ b/changelog.d/unified-streaming.add
@@ -0,0 +1 @@
+Add unified streaming endpoint
From eb33a03d0ad8571e3f0f3e0c5e9af39158c72a09 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Tue, 18 Jul 2023 18:13:49 -0400
Subject: [PATCH 215/823] Explain the encode-decode roundtrip
---
test/pleroma/integration/mastodon_websocket_test.exs | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs
index 7d5dfc255..a2c20f0a6 100644
--- a/test/pleroma/integration/mastodon_websocket_test.exs
+++ b/test/pleroma/integration/mastodon_websocket_test.exs
@@ -38,6 +38,13 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
end
end
+ # Turns atom keys to strings
+ defp atom_key_to_string(json) do
+ json
+ |> Jason.encode!()
+ |> Jason.decode!()
+ end
+
test "refuses invalid requests" do
capture_log(fn ->
assert {:error, %WebSockex.RequestError{code: 404}} = start_socket("?stream=ncjdk")
@@ -80,8 +87,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
view_json =
Pleroma.Web.MastodonAPI.StatusView.render("show.json", activity: activity, for: nil)
- |> Jason.encode!()
- |> Jason.decode!()
+ |> atom_key_to_string()
assert json == view_json
end
From b748efe66a099b66300f2beda42f5639911bab4a Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sat, 29 Jul 2023 12:55:43 -0400
Subject: [PATCH 216/823] Fix mentioning punycode domains when using Markdown
---
changelog.d/punycode-mention.fix | 1 +
lib/pleroma/formatter.ex | 2 +-
test/pleroma/web/common_api_test.exs | 12 ++++++++++++
3 files changed, 14 insertions(+), 1 deletion(-)
create mode 100644 changelog.d/punycode-mention.fix
diff --git a/changelog.d/punycode-mention.fix b/changelog.d/punycode-mention.fix
new file mode 100644
index 000000000..f013c2dac
--- /dev/null
+++ b/changelog.d/punycode-mention.fix
@@ -0,0 +1 @@
+Fix mentioning punycode domains when using Markdown
diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index a46c3e381..11d5af2fb 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -124,7 +124,7 @@ defmodule Pleroma.Formatter do
end
def markdown_to_html(text) do
- Earmark.as_html!(text, %Earmark.Options{compact_output: true})
+ Earmark.as_html!(text, %Earmark.Options{compact_output: true, smartypants: false})
end
def html_escape({text, mentions, hashtags}, type) do
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index a98b16d4b..b21dd4e23 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -843,6 +843,18 @@ defmodule Pleroma.Web.CommonAPITest do
{:ok, _} = CommonAPI.post(user, %{status: "nice", quote_id: public.id})
{:ok, _} = CommonAPI.post(another_user, %{status: "nice", quote_id: public.id})
end
+
+ test "it properly mentions punycode domain" do
+ user = insert(:user)
+
+ _mentioned_user =
+ insert(:user, ap_id: "https://xn--i2raa.com/users/yyy", nickname: "yyy@xn--i2raa.com")
+
+ {:ok, activity} =
+ CommonAPI.post(user, %{status: "hey @yyy@xn--i2raa.com", content_type: "text/markdown"})
+
+ assert "https://xn--i2raa.com/users/yyy" in Object.normalize(activity).data["to"]
+ end
end
describe "reactions" do
From df0b56576a6863db9f1ca2c7987f2a465bd1f1c3 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sun, 24 Sep 2023 22:46:30 -0400
Subject: [PATCH 217/823] Fix other quotation mark conversion tests
---
test/pleroma/web/common_api/utils_test.exs | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs
index 4ce039d64..27b1da1e3 100644
--- a/test/pleroma/web/common_api/utils_test.exs
+++ b/test/pleroma/web/common_api/utils_test.exs
@@ -200,7 +200,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
{result, _, []} = Utils.format_input(code, "text/markdown")
assert result ==
- ~s[@mario @luigi yo what’s up?
]
+ ~s[@mario @luigi yo what's up?
]
end
test "remote mentions" do
@@ -211,7 +211,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
{result, _, []} = Utils.format_input(code, "text/markdown")
assert result ==
- ~s[@mario @luigi yo what’s up?
]
+ ~s[@mario @luigi yo what's up?
]
end
test "raw HTML" do
@@ -229,7 +229,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
test "blockquote" do
code = ~s[> whoms't are you quoting?]
{result, [], []} = Utils.format_input(code, "text/markdown")
- assert result == "whoms’t are you quoting?
"
+ assert result == "whoms't are you quoting?
"
end
test "code" do
From 5ff3783d018476bad954881e39f714fac0630436 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Tue, 22 Nov 2022 23:34:23 +0100
Subject: [PATCH 218/823] Use correct domain for fqn and InstanceView
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
lib/pleroma/user.ex | 2 +-
lib/pleroma/web/mastodon_api/views/instance_view.ex | 2 +-
lib/pleroma/web/web_finger.ex | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index ce125d608..a0d4aca66 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -2252,7 +2252,7 @@ defmodule Pleroma.User do
if String.contains?(user.nickname, "@") do
user.nickname
else
- %{host: host} = URI.parse(user.ap_id)
+ host = Pleroma.Web.WebFinger.domain()
user.nickname <> "@" <> host
end
end
diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index 1b01d7371..aef02a418 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -14,7 +14,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
instance = Config.get(:instance)
%{
- uri: Pleroma.Web.Endpoint.url(),
+ uri: Pleroma.Web.WebFinger.domain(),
title: Keyword.get(instance, :name),
description: Keyword.get(instance, :description),
short_description: Keyword.get(instance, :short_description),
diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex
index f95dc2458..b28fad8d1 100644
--- a/lib/pleroma/web/web_finger.ex
+++ b/lib/pleroma/web/web_finger.ex
@@ -96,7 +96,7 @@ defmodule Pleroma.Web.WebFinger do
|> XmlBuilder.to_doc()
end
- defp domain do
+ def domain do
Pleroma.Config.get([__MODULE__, :domain]) || Pleroma.Web.Endpoint.host()
end
From c03852fbc78f8b02aef2189409f393a339ac113f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Wed, 23 Nov 2022 00:13:06 +0100
Subject: [PATCH 219/823] update tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
test/pleroma/user_test.exs | 20 ++++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs
index 7f60b959a..a8bd509a9 100644
--- a/test/pleroma/user_test.exs
+++ b/test/pleroma/user_test.exs
@@ -2678,13 +2678,25 @@ defmodule Pleroma.UserTest do
end
describe "full_nickname/1" do
- test "returns fully qualified nickname for local and remote users" do
- local_user =
- insert(:user, nickname: "local_user", ap_id: "https://somehost.com/users/local_user")
+ test "returns fully qualified nickname for local users" do
+ local_user = insert(:user, nickname: "local_user")
+ assert User.full_nickname(local_user) == "local_user@localhost"
+ end
+
+ test "returns fully qualified nickname for local users when using different domain for webfinger" do
+ clear_config([Pleroma.Web.WebFinger, :domain], "plemora.dev")
+
+ host = Pleroma.Web.Endpoint.host()
+
+ local_user = insert(:user, nickname: "local_user")
+
+ assert User.full_nickname(local_user) == "local_user@plemora.dev"
+ end
+
+ test "returns fully qualified nickname for remote users" do
remote_user = insert(:user, nickname: "remote@host.com", local: false)
- assert User.full_nickname(local_user) == "local_user@somehost.com"
assert User.full_nickname(remote_user) == "remote@host.com"
end
From a2a69709b51692be307940c79d0befdd3c9678bb Mon Sep 17 00:00:00 2001
From: tusooa
Date: Tue, 24 Oct 2023 19:57:31 -0400
Subject: [PATCH 220/823] Bump version to 2.6.0
---
CHANGELOG.md | 38 +++++++++++++++++--
changelog.d/2023-06-deps-update.skip | 0
changelog.d/3126.fix | 1 -
changelog.d/3739.skip | 0
changelog.d/3801.fix | 1 -
changelog.d/3831.skip | 0
changelog.d/3848.add | 1 -
changelog.d/3870.skip | 0
changelog.d/3872.remove | 1 -
changelog.d/3873.fix | 1 -
changelog.d/3874.remove | 1 -
changelog.d/3876.skip | 0
changelog.d/3877.skip | 0
changelog.d/3878.skip | 0
changelog.d/3879.fix | 1 -
changelog.d/3880.remove | 1 -
changelog.d/3882.add | 1 -
changelog.d/3883.fix | 1 -
changelog.d/3884.fix | 1 -
changelog.d/3885.fix | 1 -
changelog.d/3888.fix | 1 -
changelog.d/3891.fix | 1 -
changelog.d/3893.skip | 0
changelog.d/3897.add | 1 -
changelog.d/3899.skip | 0
changelog.d/3901.security | 1 -
changelog.d/3902.skip | 0
changelog.d/3909.skip | 0
.../akkoma-xml-remote-entities.security | 1 -
changelog.d/amd64-runner.skip | 0
changelog.d/attachment-type-check.fix | 1 -
changelog.d/changelog-improve.skip | 0
.../check-attachment-attribution.security | 1 -
changelog.d/delete-status-of-banned-user.fix | 1 -
changelog.d/deprecate-scrobbles.remove | 1 -
.../disable-xml-entity-resolution.security | 1 -
changelog.d/distro-docs-elixir-1.11.skip | 0
changelog.d/dockerfile-config-perms.fix | 1 -
changelog.d/emoji-pack-sanitization.security | 1 -
changelog.d/emoji-policy.add | 1 -
...d-collection-shouldnt-break-user-fetch.fix | 1 -
changelog.d/fix-object-test.fix | 1 -
changelog.d/gentoo_otp.skip | 0
changelog.d/gentoo_otp_hotfix.skip | 0
changelog.d/gentoo_otp_intro.skip | 0
.../handle-report-from-deactivated-user.fix | 1 -
changelog.d/lint.skip | 0
changelog.d/media-altdomain.skip | 0
changelog.d/no_new_privs.add | 1 -
changelog.d/otp_perms.security | 1 -
changelog.d/pipeline-triggers.skip | 0
...revent-bypassing-authorized-fetch-mode.fix | 1 -
changelog.d/punycode-mention.fix | 1 -
changelog.d/quote.add | 1 -
changelog.d/testfix-system-config-use.skip | 0
changelog.d/unified-streaming.add | 1 -
.../update-credentials-limit-error.fix | 1 -
mix.exs | 2 +-
58 files changed, 35 insertions(+), 40 deletions(-)
delete mode 100644 changelog.d/2023-06-deps-update.skip
delete mode 100644 changelog.d/3126.fix
delete mode 100644 changelog.d/3739.skip
delete mode 100644 changelog.d/3801.fix
delete mode 100644 changelog.d/3831.skip
delete mode 100644 changelog.d/3848.add
delete mode 100644 changelog.d/3870.skip
delete mode 100644 changelog.d/3872.remove
delete mode 100644 changelog.d/3873.fix
delete mode 100644 changelog.d/3874.remove
delete mode 100644 changelog.d/3876.skip
delete mode 100644 changelog.d/3877.skip
delete mode 100644 changelog.d/3878.skip
delete mode 100644 changelog.d/3879.fix
delete mode 100644 changelog.d/3880.remove
delete mode 100644 changelog.d/3882.add
delete mode 100644 changelog.d/3883.fix
delete mode 100644 changelog.d/3884.fix
delete mode 100644 changelog.d/3885.fix
delete mode 100644 changelog.d/3888.fix
delete mode 100644 changelog.d/3891.fix
delete mode 100644 changelog.d/3893.skip
delete mode 100644 changelog.d/3897.add
delete mode 100644 changelog.d/3899.skip
delete mode 100644 changelog.d/3901.security
delete mode 100644 changelog.d/3902.skip
delete mode 100644 changelog.d/3909.skip
delete mode 100644 changelog.d/akkoma-xml-remote-entities.security
delete mode 100644 changelog.d/amd64-runner.skip
delete mode 100644 changelog.d/attachment-type-check.fix
delete mode 100644 changelog.d/changelog-improve.skip
delete mode 100644 changelog.d/check-attachment-attribution.security
delete mode 100644 changelog.d/delete-status-of-banned-user.fix
delete mode 100644 changelog.d/deprecate-scrobbles.remove
delete mode 100644 changelog.d/disable-xml-entity-resolution.security
delete mode 100644 changelog.d/distro-docs-elixir-1.11.skip
delete mode 100644 changelog.d/dockerfile-config-perms.fix
delete mode 100644 changelog.d/emoji-pack-sanitization.security
delete mode 100644 changelog.d/emoji-policy.add
delete mode 100644 changelog.d/featured-collection-shouldnt-break-user-fetch.fix
delete mode 100644 changelog.d/fix-object-test.fix
delete mode 100644 changelog.d/gentoo_otp.skip
delete mode 100644 changelog.d/gentoo_otp_hotfix.skip
delete mode 100644 changelog.d/gentoo_otp_intro.skip
delete mode 100644 changelog.d/handle-report-from-deactivated-user.fix
delete mode 100644 changelog.d/lint.skip
delete mode 100644 changelog.d/media-altdomain.skip
delete mode 100644 changelog.d/no_new_privs.add
delete mode 100644 changelog.d/otp_perms.security
delete mode 100644 changelog.d/pipeline-triggers.skip
delete mode 100644 changelog.d/prevent-bypassing-authorized-fetch-mode.fix
delete mode 100644 changelog.d/punycode-mention.fix
delete mode 100644 changelog.d/quote.add
delete mode 100644 changelog.d/testfix-system-config-use.skip
delete mode 100644 changelog.d/unified-streaming.add
delete mode 100644 changelog.d/update-credentials-limit-error.fix
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 65acfad3e..211e611ab 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,19 +4,49 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
-## Unreleased
-
-### Changed
+## 2.6.0
+### Security
+- Preload: Make generated JSON html-safe. It already was html safe because it only consists of config data that is base64 encoded, but this will keep it safe it that ever changes.
+- CommonAPI: Prevent users from accessing media of other users by creating a status with reused attachment ID
+- Disable XML entity resolution completely to fix a dos vulnerability
### Added
- Support for Image activities, namely from Hubzilla
+- Add OAuth scope descriptions
+- Allow lang attribute in status text
+- OnlyMedia Upload Filter
+- Implement MRF policy to reject or delist according to emojis
+- (hardening) Add no_new_privs=yes to OpenRC service files
+- Implement quotes
+- Add unified streaming endpoint
### Fixed
-
- rel="me" was missing its cache
+- MediaProxy responses now return a sandbox CSP header
+- Filter context activities using Visibility.visible_for_user?
+- UploadedMedia: Add missing disposition_type to Content-Disposition
+- fix not being able to fetch flash file from remote instance
+- Fix abnormal behaviour when refetching a poll
+- Allow non-HTTP(s) URIs in "url" fields for compatibility with "FEP-fffd: Proxy Objects"
+- Fix opengraph and twitter card meta tags
+- ForceMentionsInContent: fix double mentions for Mastodon/Misskey posts
+- OEmbed HTML tags are now filtered
+- Restrict attachments to only uploaded files only
+- Fix error 404 when deleting status of a banned user
+- Fix config ownership in dockerfile to pass restriction test
+- Fix user fetch completely broken if featured collection is not in a supported form
+- Correctly handle the situation when a poll has both "anyOf" and "oneOf" but one of them being empty
+- Fix handling report from a deactivated user
+- Prevent using the .json format to bypass authorized fetch mode
+- Fix mentioning punycode domains when using Markdown
+- Show more informative errors when profile exceeds char limits
### Removed
- BREAKING: Support for passwords generated with `crypt(3)` (Gnu Social migration artifact)
+- remove BBS/SSH feature, replaced by an external bridge.
+- Remove a few unused indexes.
+- Cleanup OStatus-era user upgrades and ap_enabled indicator
+- Deprecate Pleroma's audio scrobbling
## 2.5.4
diff --git a/changelog.d/2023-06-deps-update.skip b/changelog.d/2023-06-deps-update.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/3126.fix b/changelog.d/3126.fix
deleted file mode 100644
index 91d396c89..000000000
--- a/changelog.d/3126.fix
+++ /dev/null
@@ -1 +0,0 @@
-MediaProxy responses now return a sandbox CSP header
diff --git a/changelog.d/3739.skip b/changelog.d/3739.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/3801.fix b/changelog.d/3801.fix
deleted file mode 100644
index 8c2ec0199..000000000
--- a/changelog.d/3801.fix
+++ /dev/null
@@ -1 +0,0 @@
-Filter context activities using Visibility.visible_for_user?
diff --git a/changelog.d/3831.skip b/changelog.d/3831.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/3848.add b/changelog.d/3848.add
deleted file mode 100644
index d7b1b0a84..000000000
--- a/changelog.d/3848.add
+++ /dev/null
@@ -1 +0,0 @@
-Add OAuth scope descriptions
diff --git a/changelog.d/3870.skip b/changelog.d/3870.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/3872.remove b/changelog.d/3872.remove
deleted file mode 100644
index 54cbb660e..000000000
--- a/changelog.d/3872.remove
+++ /dev/null
@@ -1 +0,0 @@
-remove BBS/SSH feature, replaced by an external bridge.
\ No newline at end of file
diff --git a/changelog.d/3873.fix b/changelog.d/3873.fix
deleted file mode 100644
index 4699f7b58..000000000
--- a/changelog.d/3873.fix
+++ /dev/null
@@ -1 +0,0 @@
-UploadedMedia: Add missing disposition_type to Content-Disposition
\ No newline at end of file
diff --git a/changelog.d/3874.remove b/changelog.d/3874.remove
deleted file mode 100644
index a81f744bf..000000000
--- a/changelog.d/3874.remove
+++ /dev/null
@@ -1 +0,0 @@
-Remove a few unused indexes.
diff --git a/changelog.d/3876.skip b/changelog.d/3876.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/3877.skip b/changelog.d/3877.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/3878.skip b/changelog.d/3878.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/3879.fix b/changelog.d/3879.fix
deleted file mode 100644
index 7c58cc3c2..000000000
--- a/changelog.d/3879.fix
+++ /dev/null
@@ -1 +0,0 @@
-fix not being able to fetch flash file from remote instance
\ No newline at end of file
diff --git a/changelog.d/3880.remove b/changelog.d/3880.remove
deleted file mode 100644
index 113c76c85..000000000
--- a/changelog.d/3880.remove
+++ /dev/null
@@ -1 +0,0 @@
-Cleanup OStatus-era user upgrades and ap_enabled indicator
\ No newline at end of file
diff --git a/changelog.d/3882.add b/changelog.d/3882.add
deleted file mode 100644
index 4712de1dc..000000000
--- a/changelog.d/3882.add
+++ /dev/null
@@ -1 +0,0 @@
-Allow lang attribute in status text
diff --git a/changelog.d/3883.fix b/changelog.d/3883.fix
deleted file mode 100644
index 6824f2013..000000000
--- a/changelog.d/3883.fix
+++ /dev/null
@@ -1 +0,0 @@
-Fix abnormal behaviour when refetching a poll
diff --git a/changelog.d/3884.fix b/changelog.d/3884.fix
deleted file mode 100644
index f8dbb2bbf..000000000
--- a/changelog.d/3884.fix
+++ /dev/null
@@ -1 +0,0 @@
-Allow non-HTTP(s) URIs in "url" fields for compatibility with "FEP-fffd: Proxy Objects"
\ No newline at end of file
diff --git a/changelog.d/3885.fix b/changelog.d/3885.fix
deleted file mode 100644
index c5fbb0ed4..000000000
--- a/changelog.d/3885.fix
+++ /dev/null
@@ -1 +0,0 @@
-Fix opengraph and twitter card meta tags
diff --git a/changelog.d/3888.fix b/changelog.d/3888.fix
deleted file mode 100644
index 886aa7b39..000000000
--- a/changelog.d/3888.fix
+++ /dev/null
@@ -1 +0,0 @@
-ForceMentionsInContent: fix double mentions for Mastodon/Misskey posts
\ No newline at end of file
diff --git a/changelog.d/3891.fix b/changelog.d/3891.fix
deleted file mode 100644
index f1fb62d82..000000000
--- a/changelog.d/3891.fix
+++ /dev/null
@@ -1 +0,0 @@
-OEmbed HTML tags are now filtered
diff --git a/changelog.d/3893.skip b/changelog.d/3893.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/3897.add b/changelog.d/3897.add
deleted file mode 100644
index 5c4402f45..000000000
--- a/changelog.d/3897.add
+++ /dev/null
@@ -1 +0,0 @@
-OnlyMedia Upload Filter
diff --git a/changelog.d/3899.skip b/changelog.d/3899.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/3901.security b/changelog.d/3901.security
deleted file mode 100644
index a3d8bd01f..000000000
--- a/changelog.d/3901.security
+++ /dev/null
@@ -1 +0,0 @@
-Preload: Make generated JSON html-safe. It already was html safe because it only consists of config data that is base64 encoded, but this will keep it safe it that ever changes.
diff --git a/changelog.d/3902.skip b/changelog.d/3902.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/3909.skip b/changelog.d/3909.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/akkoma-xml-remote-entities.security b/changelog.d/akkoma-xml-remote-entities.security
deleted file mode 100644
index 5e6725e5b..000000000
--- a/changelog.d/akkoma-xml-remote-entities.security
+++ /dev/null
@@ -1 +0,0 @@
-Fix XML External Entity (XXE) loading vulnerability allowing to fetch arbitary files from the server's filesystem
diff --git a/changelog.d/amd64-runner.skip b/changelog.d/amd64-runner.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/attachment-type-check.fix b/changelog.d/attachment-type-check.fix
deleted file mode 100644
index 9e14b75f1..000000000
--- a/changelog.d/attachment-type-check.fix
+++ /dev/null
@@ -1 +0,0 @@
-Restrict attachments to only uploaded files only
diff --git a/changelog.d/changelog-improve.skip b/changelog.d/changelog-improve.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/check-attachment-attribution.security b/changelog.d/check-attachment-attribution.security
deleted file mode 100644
index e0e46525b..000000000
--- a/changelog.d/check-attachment-attribution.security
+++ /dev/null
@@ -1 +0,0 @@
-CommonAPI: Prevent users from accessing media of other users by creating a status with reused attachment ID
diff --git a/changelog.d/delete-status-of-banned-user.fix b/changelog.d/delete-status-of-banned-user.fix
deleted file mode 100644
index 1fa6a29d8..000000000
--- a/changelog.d/delete-status-of-banned-user.fix
+++ /dev/null
@@ -1 +0,0 @@
-Fix error 404 when deleting status of a banned user
diff --git a/changelog.d/deprecate-scrobbles.remove b/changelog.d/deprecate-scrobbles.remove
deleted file mode 100644
index c453a9784..000000000
--- a/changelog.d/deprecate-scrobbles.remove
+++ /dev/null
@@ -1 +0,0 @@
-Deprecate Pleroma's audio scrobbling
diff --git a/changelog.d/disable-xml-entity-resolution.security b/changelog.d/disable-xml-entity-resolution.security
deleted file mode 100644
index db8e12f67..000000000
--- a/changelog.d/disable-xml-entity-resolution.security
+++ /dev/null
@@ -1 +0,0 @@
-Disable XML entity resolution completely to fix a dos vulnerability
diff --git a/changelog.d/distro-docs-elixir-1.11.skip b/changelog.d/distro-docs-elixir-1.11.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/dockerfile-config-perms.fix b/changelog.d/dockerfile-config-perms.fix
deleted file mode 100644
index 49ea5becb..000000000
--- a/changelog.d/dockerfile-config-perms.fix
+++ /dev/null
@@ -1 +0,0 @@
-- Fix config ownership in dockerfile to pass restriction test
diff --git a/changelog.d/emoji-pack-sanitization.security b/changelog.d/emoji-pack-sanitization.security
deleted file mode 100644
index f3218abd4..000000000
--- a/changelog.d/emoji-pack-sanitization.security
+++ /dev/null
@@ -1 +0,0 @@
-Emoji pack loader sanitizes pack names
diff --git a/changelog.d/emoji-policy.add b/changelog.d/emoji-policy.add
deleted file mode 100644
index 45510c4f6..000000000
--- a/changelog.d/emoji-policy.add
+++ /dev/null
@@ -1 +0,0 @@
-Implement MRF policy to reject or delist according to emojis
diff --git a/changelog.d/featured-collection-shouldnt-break-user-fetch.fix b/changelog.d/featured-collection-shouldnt-break-user-fetch.fix
deleted file mode 100644
index e8ce288cc..000000000
--- a/changelog.d/featured-collection-shouldnt-break-user-fetch.fix
+++ /dev/null
@@ -1 +0,0 @@
-Fix user fetch completely broken if featured collection is not in a supported form
diff --git a/changelog.d/fix-object-test.fix b/changelog.d/fix-object-test.fix
deleted file mode 100644
index 5eea719f0..000000000
--- a/changelog.d/fix-object-test.fix
+++ /dev/null
@@ -1 +0,0 @@
-Correctly handle the situation when a poll has both "anyOf" and "oneOf" but one of them being empty
diff --git a/changelog.d/gentoo_otp.skip b/changelog.d/gentoo_otp.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/gentoo_otp_hotfix.skip b/changelog.d/gentoo_otp_hotfix.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/gentoo_otp_intro.skip b/changelog.d/gentoo_otp_intro.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/handle-report-from-deactivated-user.fix b/changelog.d/handle-report-from-deactivated-user.fix
deleted file mode 100644
index 6692d1aa8..000000000
--- a/changelog.d/handle-report-from-deactivated-user.fix
+++ /dev/null
@@ -1 +0,0 @@
-Fix handling report from a deactivated user
diff --git a/changelog.d/lint.skip b/changelog.d/lint.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/media-altdomain.skip b/changelog.d/media-altdomain.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/no_new_privs.add b/changelog.d/no_new_privs.add
deleted file mode 100644
index b67396a4b..000000000
--- a/changelog.d/no_new_privs.add
+++ /dev/null
@@ -1 +0,0 @@
-(hardening) Add no_new_privs=yes to OpenRC service files
diff --git a/changelog.d/otp_perms.security b/changelog.d/otp_perms.security
deleted file mode 100644
index a3da1c677..000000000
--- a/changelog.d/otp_perms.security
+++ /dev/null
@@ -1 +0,0 @@
-- Reduced permissions of config files and directories, distros requiring greater permissions like group-read need to pre-create the directories
\ No newline at end of file
diff --git a/changelog.d/pipeline-triggers.skip b/changelog.d/pipeline-triggers.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/prevent-bypassing-authorized-fetch-mode.fix b/changelog.d/prevent-bypassing-authorized-fetch-mode.fix
deleted file mode 100644
index 12f7260d7..000000000
--- a/changelog.d/prevent-bypassing-authorized-fetch-mode.fix
+++ /dev/null
@@ -1 +0,0 @@
-Prevent using the .json format to bypass authorized fetch mode
\ No newline at end of file
diff --git a/changelog.d/punycode-mention.fix b/changelog.d/punycode-mention.fix
deleted file mode 100644
index f013c2dac..000000000
--- a/changelog.d/punycode-mention.fix
+++ /dev/null
@@ -1 +0,0 @@
-Fix mentioning punycode domains when using Markdown
diff --git a/changelog.d/quote.add b/changelog.d/quote.add
deleted file mode 100644
index 1c368ae75..000000000
--- a/changelog.d/quote.add
+++ /dev/null
@@ -1 +0,0 @@
-Implement quotes
diff --git a/changelog.d/testfix-system-config-use.skip b/changelog.d/testfix-system-config-use.skip
deleted file mode 100644
index e69de29bb..000000000
diff --git a/changelog.d/unified-streaming.add b/changelog.d/unified-streaming.add
deleted file mode 100644
index 84821fcc8..000000000
--- a/changelog.d/unified-streaming.add
+++ /dev/null
@@ -1 +0,0 @@
-Add unified streaming endpoint
diff --git a/changelog.d/update-credentials-limit-error.fix b/changelog.d/update-credentials-limit-error.fix
deleted file mode 100644
index 7682f958e..000000000
--- a/changelog.d/update-credentials-limit-error.fix
+++ /dev/null
@@ -1 +0,0 @@
-Show more informative errors when profile exceeds char limits
diff --git a/mix.exs b/mix.exs
index b071e7c7b..082b39e55 100644
--- a/mix.exs
+++ b/mix.exs
@@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do
def project do
[
app: :pleroma,
- version: version("2.5.54"),
+ version: version("2.6.0"),
elixir: "~> 1.11",
elixirc_paths: elixirc_paths(Mix.env()),
compilers: [:phoenix] ++ Mix.compilers(),
From 6a13c2d180177692b976b179f0bd86f8f93e2803 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Wed, 25 Oct 2023 20:44:47 -0400
Subject: [PATCH 221/823] Add collect-changelog script
---
.gitlab/merge_request_templates/Release.md | 2 +-
tools/collect-changelog | 27 ++++++++++++++++++++++
2 files changed, 28 insertions(+), 1 deletion(-)
create mode 100755 tools/collect-changelog
diff --git a/.gitlab/merge_request_templates/Release.md b/.gitlab/merge_request_templates/Release.md
index 9638d6d11..e57556e6c 100644
--- a/.gitlab/merge_request_templates/Release.md
+++ b/.gitlab/merge_request_templates/Release.md
@@ -1,6 +1,6 @@
### Release checklist
* [ ] Bump version in `mix.exs`
-* [ ] Compile a changelog
+* [ ] Compile a changelog with the `tools/collect-changelog` script
* [ ] Create an MR with an announcement to pleroma.social
#### post-merge
* [ ] Tag the release on the merge commit
diff --git a/tools/collect-changelog b/tools/collect-changelog
new file mode 100755
index 000000000..1e12d5640
--- /dev/null
+++ b/tools/collect-changelog
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+collectType() {
+ local suffix="$1"
+ local header="$2"
+ local printed=0
+ for file in changelog.d/*."$suffix"; do
+ if [ '!' -f "$file" ]; then
+ continue
+ fi
+ if [ "$printed" = 0 ]; then
+ echo
+ echo "### $header"
+ printed=1
+ fi
+ # Normalize any trailing newlines/spaces, etc.
+ echo "- $(cat "$file")"
+ done
+}
+
+collectType security Security
+collectType change Changed
+collectType add Added
+collectType fix Fixed
+collectType remove Removed
+
+rm changelog.d/*
From 6b8c5e12dfe759ac1286e81e72ad7f8727e01386 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Thu, 26 Oct 2023 23:30:38 +0200
Subject: [PATCH 222/823] Add contact account to InstanceView
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
changelog.d/instance-contact-account.add | 1 +
config/description.exs | 6 ++++++
.../web/mastodon_api/views/instance_view.ex | 17 +++++++++++++++++
.../controllers/instance_controller_test.exs | 12 ++++++++++++
4 files changed, 36 insertions(+)
create mode 100644 changelog.d/instance-contact-account.add
diff --git a/changelog.d/instance-contact-account.add b/changelog.d/instance-contact-account.add
new file mode 100644
index 000000000..e119446d2
--- /dev/null
+++ b/changelog.d/instance-contact-account.add
@@ -0,0 +1 @@
+Add contact account to InstanceView
\ No newline at end of file
diff --git a/config/description.exs b/config/description.exs
index d18649ae8..239ba7cb0 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -566,6 +566,12 @@ config :pleroma, :config_description, [
"Cool instance"
]
},
+ %{
+ key: :contact_username,
+ type: :string,
+ description: "Instance owner username",
+ suggestions: ["admin"]
+ },
%{
key: :limit,
type: :integer,
diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index 1b01d7371..06df63b0f 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -30,6 +30,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
languages: Keyword.get(instance, :languages, ["en"]),
registrations: Keyword.get(instance, :registrations_open),
approval_required: Keyword.get(instance, :account_approval_required),
+ contact_account: contact_account(Keyword.get(instance, :contact_username)),
# Extra (not present in Mastodon):
max_toot_chars: Keyword.get(instance, :limit),
max_media_attachments: Keyword.get(instance, :max_media_attachments),
@@ -141,4 +142,20 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
value_length: Config.get([:instance, :account_field_value_length])
}
end
+
+ defp contact_account(nil), do: nil
+
+ defp contact_account("@" <> username) do
+ contact_account(username)
+ end
+
+ defp contact_account(username) do
+ user = Pleroma.User.get_cached_by_nickname(username)
+
+ if user do
+ Pleroma.Web.MastodonAPI.AccountView.render("show.json", %{user: user, for: nil})
+ else
+ nil
+ end
+ end
end
diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs
index a556ef6a8..cfa86468f 100644
--- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs
@@ -106,4 +106,16 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do
|> get("/api/v1/instance")
|> json_response_and_validate_schema(200)
end
+
+ test "get instance contact information", %{conn: conn} do
+ user = insert(:user, %{local: true})
+
+ clear_config([:instance, :contact_username], user.nickname)
+
+ conn = get(conn, "/api/v1/instance")
+
+ assert result = json_response_and_validate_schema(conn, 200)
+
+ assert result["contact_account"]["id"] == user.id
+ end
end
From c6cedbb8106a16527e48ac8ae03907e1d66c5a1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Sat, 28 Oct 2023 00:07:18 +0200
Subject: [PATCH 223/823] InstanceV2: skip auth
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
lib/pleroma/web/mastodon_api/controllers/instance_controller.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex
index 3757a850a..3e664903a 100644
--- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex
@@ -7,7 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do
plug(Pleroma.Web.ApiSpec.CastAndValidate)
- plug(:skip_auth when action in [:show, :peers])
+ plug(:skip_auth when action in [:show, :show2, :peers])
defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.InstanceOperation
From 93370b870a8d16b5dada5ef7c7e3ef2de378395e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Sun, 7 Aug 2022 23:56:52 +0200
Subject: [PATCH 224/823] Expose nonAnonymous field from Smithereen polls
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
.../object_validators/question_validator.ex | 1 +
lib/pleroma/web/api_spec/schemas/poll.ex | 11 ++++++++++-
lib/pleroma/web/mastodon_api/views/poll_view.ex | 5 ++++-
3 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
index 621085e6c..7f9d4d648 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -29,6 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
field(:closed, ObjectValidators.DateTime)
field(:voters, {:array, ObjectValidators.ObjectID}, default: [])
+ field(:nonAnonymous, :boolean)
embeds_many(:anyOf, QuestionOptionsValidator)
embeds_many(:oneOf, QuestionOptionsValidator)
end
diff --git a/lib/pleroma/web/api_spec/schemas/poll.ex b/lib/pleroma/web/api_spec/schemas/poll.ex
index 91570582b..cb2ffdc68 100644
--- a/lib/pleroma/web/api_spec/schemas/poll.ex
+++ b/lib/pleroma/web/api_spec/schemas/poll.ex
@@ -56,6 +56,12 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do
}
},
description: "Possible answers for the poll."
+ },
+ pleroma: %Schema{
+ type: :object,
+ properties: %{
+ non_anonymous: %Schema{type: :boolean, description: "Is the voters collection public?"}
+ }
}
},
example: %{
@@ -79,7 +85,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do
votes_count: 4
}
],
- emojis: []
+ emojis: [],
+ pleroma: %{
+ non_anonymous: false
+ }
}
})
end
diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex
index 34e23873e..1e3c9f36d 100644
--- a/lib/pleroma/web/mastodon_api/views/poll_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex
@@ -21,7 +21,10 @@ defmodule Pleroma.Web.MastodonAPI.PollView do
votes_count: votes_count,
voters_count: voters_count(object),
options: options,
- emojis: Pleroma.Web.MastodonAPI.StatusView.build_emojis(object.data["emoji"])
+ emojis: Pleroma.Web.MastodonAPI.StatusView.build_emojis(object.data["emoji"]),
+ pleroma: %{
+ non_anonymous: object.data["nonAnonymous"] || false
+ }
}
if params[:for] do
From e5bd1ee80154b31457567c3d31de6c63e0146352 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?=
Date: Mon, 8 Aug 2022 10:45:43 +0200
Subject: [PATCH 225/823] Add entry to @context, tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: marcin mikołajczak
---
changelog.d/public-polls.add | 1 +
priv/static/schemas/litepub-0.1.jsonld | 4 +++-
.../smithereen_non_anonymous_poll.json | 1 +
test/fixtures/tesla_mock/smithereen_user.json | 1 +
.../web/mastodon_api/views/poll_view_test.exs | 10 +++++++++-
test/support/http_request_mock.ex | 18 ++++++++++++++++++
6 files changed, 33 insertions(+), 2 deletions(-)
create mode 100644 changelog.d/public-polls.add
create mode 100644 test/fixtures/tesla_mock/smithereen_non_anonymous_poll.json
create mode 100644 test/fixtures/tesla_mock/smithereen_user.json
diff --git a/changelog.d/public-polls.add b/changelog.d/public-polls.add
new file mode 100644
index 000000000..0dae0c38e
--- /dev/null
+++ b/changelog.d/public-polls.add
@@ -0,0 +1 @@
+Expose nonAnonymous field from Smithereen polls
\ No newline at end of file
diff --git a/priv/static/schemas/litepub-0.1.jsonld b/priv/static/schemas/litepub-0.1.jsonld
index b499a96f5..d156cba55 100644
--- a/priv/static/schemas/litepub-0.1.jsonld
+++ b/priv/static/schemas/litepub-0.1.jsonld
@@ -40,7 +40,9 @@
"@type": "@id"
},
"vcard": "http://www.w3.org/2006/vcard/ns#",
- "formerRepresentations": "litepub:formerRepresentations"
+ "formerRepresentations": "litepub:formerRepresentations",
+ "sm": "http://smithereen.software/ns#",
+ "nonAnonymous": "sm:nonAnonymous"
}
]
}
diff --git a/test/fixtures/tesla_mock/smithereen_non_anonymous_poll.json b/test/fixtures/tesla_mock/smithereen_non_anonymous_poll.json
new file mode 100644
index 000000000..2b343ea64
--- /dev/null
+++ b/test/fixtures/tesla_mock/smithereen_non_anonymous_poll.json
@@ -0,0 +1 @@
+{"type":"Question","id":"https://friends.grishka.me/posts/54642","attributedTo":"https://friends.grishka.me/users/1","content":"здесь тоже можно что-то написать отдельно от опроса
","published":"2021-09-04T00:22:16Z","url":"https://friends.grishka.me/posts/54642","to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://friends.grishka.me/users/1/followers"],"replies":{"type":"Collection","id":"https://friends.grishka.me/posts/54642/replies","first":{"type":"CollectionPage","items":[],"partOf":"https://friends.grishka.me/posts/54642/replies","next":"https://friends.grishka.me/posts/54642/replies?page=1"}},"sensitive":false,"likes":"https://friends.grishka.me/posts/54642/likes","name":"тестовый опрос","oneOf":[{"type":"Note","id":"https://friends.grishka.me/posts/54642#options/76","name":"тестовый ответ 1","replies":{"type":"Collection","id":"https://friends.grishka.me/activitypub/objects/polls/24/options/76/votes","totalItems":4,"items":[]}},{"type":"Note","id":"https://friends.grishka.me/posts/54642#options/77","name":"тестовый ответ 2","replies":{"type":"Collection","id":"https://friends.grishka.me/activitypub/objects/polls/24/options/77/votes","totalItems":4,"items":[]}},{"type":"Note","id":"https://friends.grishka.me/posts/54642#options/78","name":"тестовый ответ 3","replies":{"type":"Collection","id":"https://friends.grishka.me/activitypub/objects/polls/24/options/78/votes","totalItems":6,"items":[]}}],"votersCount":14,"nonAnonymous":true,"@context":["https://www.w3.org/ns/activitystreams",{"sensitive":"as:sensitive","toot":"http://joinmastodon.org/ns#","sm":"http://smithereen.software/ns#","votersCount":"toot:votersCount","nonAnonymous":"sm:nonAnonymous"}]}
\ No newline at end of file
diff --git a/test/fixtures/tesla_mock/smithereen_user.json b/test/fixtures/tesla_mock/smithereen_user.json
new file mode 100644
index 000000000..6468fc519
--- /dev/null
+++ b/test/fixtures/tesla_mock/smithereen_user.json
@@ -0,0 +1 @@
+{"type":"Person","id":"https://friends.grishka.me/users/1","name":"Григорий Клюшников","icon":{"type":"Image","image":{"type":"Image","url":"https://friends.grishka.me/i/6QLsOws97AWp5N_osd74C1IC1ijnFopyCBD9MSEeXNQ/q:93/bG9jYWw6Ly8vcy91cGxvYWRzL2F2YXRhcnMvNTYzODRhODEwODk5ZTRjMzI4YmY4YmQwM2Q2MWM3NmMud2VicA.jpg","mediaType":"image/jpeg","width":1280,"height":960},"width":573,"height":572,"cropRegion":[0.26422762870788574,0.3766937553882599,0.7113820910453796,0.9728997349739075],"url":"https://friends.grishka.me/i/ql_49PQcETAWgY_nC-Qj63H_Oa6FyOAEoWFkUSSkUvQ/c:573:572:nowe:338:362/q:93/bG9jYWw6Ly8vcy91cGxvYWRzL2F2YXRhcnMvNTYzODRhODEwODk5ZTRjMzI4YmY4YmQwM2Q2MWM3NmMud2VicA.jpg","mediaType":"image/jpeg"},"summary":"Делаю эту хрень, пытаюсь вырвать социальные сети из жадных лап корпораций
\n
\n
\n
\n
\n
\n
\n
\nThis server does NOT support direct messages. Please write me on Telegram or Matrix .
","url":"https://friends.grishka.me/grishka","preferredUsername":"grishka","inbox":"https://friends.grishka.me/users/1/inbox","outbox":"https://friends.grishka.me/users/1/outbox","followers":"https://friends.grishka.me/users/1/followers","following":"https://friends.grishka.me/users/1/following","endpoints":{"sharedInbox":"https://friends.grishka.me/activitypub/sharedInbox","collectionSimpleQuery":"https://friends.grishka.me/users/1/collectionQuery"},"publicKey":{"id":"https://friends.grishka.me/users/1#main-key","owner":"https://friends.grishka.me/users/1","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAjlakm+i/d9ER/hIeR7KfiFW+SdLZj2SkKIeM8cmR+YFJuh9ghFqXrkFEjcaqUnAFqe5gYDNSQACnDLA8y4DnzjfGNIohKAnRoa9x6GORmfKQvcnjaTZ53S1NvUiPPyc0Pv/vfCtY/Ab0CEXe5BLqL38oZn817Jf7pBrPRTYH7m012kvwAUTT6k0Y8lPITBEG7nzYbbuGcrN9Y/RDdwE08jmBXlZ45bahRH3VNXVpQE17dCzJB+7k+iJ1R7YCoI+DuMlBYGXGE2KVk46NZTuLnOjFV9SyXfWX4/SrJM4oxev+SX2N75tQgmNZmVVHeqg2ZcbC0WCfNjJOi2HHS9MujwIDAQAB\n-----END PUBLIC KEY-----\n"},"wall":"https://friends.grishka.me/users/1/wall","firstName":"Григорий","lastName":"Клюшников","middleName":"Александрович","vcard:bday":"1993-01-22","gender":"http://schema.org#Male","supportsFriendRequests":true,"friends":"https://friends.grishka.me/users/1/friends","groups":"https://friends.grishka.me/users/1/groups","capabilities":{"supportsFriendRequests":true},"@context":["https://www.w3.org/ns/activitystreams",{"sm":"http://smithereen.software/ns#","cropRegion":{"@id":"sm:cropRegion","@container":"@list"},"wall":{"@id":"sm:wall","@type":"@id"},"collectionSimpleQuery":"sm:collectionSimpleQuery","sc":"http://schema.org#","firstName":"sc:givenName","lastName":"sc:familyName","middleName":"sc:additionalName","gender":{"@id":"sc:gender","@type":"sc:GenderType"},"maidenName":"sm:maidenName","friends":{"@id":"sm:friends","@type":"@id"},"groups":{"@id":"sm:groups","@type":"@id"},"vcard":"http://www.w3.org/2006/vcard/ns#","capabilities":"litepub:capabilities","supportsFriendRequests":"sm:supportsFriendRequests","litepub":"http://litepub.social/ns#"},"https://w3id.org/security/v1"]}
\ No newline at end of file
diff --git a/test/pleroma/web/mastodon_api/views/poll_view_test.exs b/test/pleroma/web/mastodon_api/views/poll_view_test.exs
index a73d862fd..3aa73c224 100644
--- a/test/pleroma/web/mastodon_api/views/poll_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/poll_view_test.exs
@@ -43,7 +43,8 @@ defmodule Pleroma.Web.MastodonAPI.PollViewTest do
%{title: "why are you even asking?", votes_count: 0}
],
votes_count: 0,
- voters_count: 0
+ voters_count: 0,
+ pleroma: %{non_anonymous: false}
}
result = PollView.render("show.json", %{object: object})
@@ -165,4 +166,11 @@ defmodule Pleroma.Web.MastodonAPI.PollViewTest do
]
} = PollView.render("show.json", %{object: object})
end
+
+ test "that poll is non anonymous" do
+ object = Object.normalize("https://friends.grishka.me/posts/54642", fetch: true)
+ result = PollView.render("show.json", %{object: object})
+
+ assert result[:pleroma][:non_anonymous] == true
+ end
end
diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex
index 78a367024..2707804bc 100644
--- a/test/support/http_request_mock.ex
+++ b/test/support/http_request_mock.ex
@@ -1464,6 +1464,24 @@ defmodule HttpRequestMock do
}}
end
+ def get("https://friends.grishka.me/posts/54642", _, _, _) do
+ {:ok,
+ %Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/tesla_mock/smithereen_non_anonymous_poll.json"),
+ headers: activitypub_object_headers()
+ }}
+ end
+
+ def get("https://friends.grishka.me/users/1", _, _, _) do
+ {:ok,
+ %Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/tesla_mock/smithereen_user.json"),
+ headers: activitypub_object_headers()
+ }}
+ end
+
def get(url, query, body, headers) do
{:error,
"Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"}
From ab894d98f4be5aadc033b71dacc499b85b579bc1 Mon Sep 17 00:00:00 2001
From: tusooa
Date: Sun, 29 Oct 2023 12:59:03 -0400
Subject: [PATCH 226/823] Bundle 2.6.0 frontend
---
priv/static/index.html | 2 +-
priv/static/static/css/5948.06d2a0d84620cba6a4fb.css | 2 --
.../static/css/5948.06d2a0d84620cba6a4fb.css.map | 1 -
priv/static/static/css/7586.0d43f70bc6240422f179.css | 2 ++
.../static/css/7586.0d43f70bc6240422f179.css.map | 1 +
priv/static/static/css/7962.76663e78ad5ea0bb0b90.css | 11 +++++++++++
.../static/css/7962.76663e78ad5ea0bb0b90.css.map | 1 +
priv/static/static/css/9114.8def3b2b7fe70b3b3712.css | 11 -----------
.../static/css/9114.8def3b2b7fe70b3b3712.css.map | 1 -
priv/static/static/css/9801.cfe503d4c949ae0c3813.css | 2 ++
.../static/css/9801.cfe503d4c949ae0c3813.css.map | 1 +
priv/static/static/css/app.48e52505beba5b9ab69b.css | 2 --
.../static/css/app.48e52505beba5b9ab69b.css.map | 1 -
priv/static/static/css/app.c18a2c80794a1b699a61.css | 2 ++
.../static/css/app.c18a2c80794a1b699a61.css.map | 1 +
priv/static/static/js/159.3a9274574f1e33801c4a.js.map | 1 -
...4574f1e33801c4a.js => 159.903e90c9de8ef6c67077.js} | 2 +-
priv/static/static/js/159.903e90c9de8ef6c67077.js.map | 1 +
priv/static/static/js/2724.e4840c73281069ba54ab.js | 3 ---
.../static/static/js/2724.e4840c73281069ba54ab.js.map | 1 -
priv/static/static/js/3733.7060d1e6bca813125a0c.js | 3 +++
...E.txt => 3733.7060d1e6bca813125a0c.js.LICENSE.txt} | 0
.../static/static/js/3733.7060d1e6bca813125a0c.js.map | 1 +
...79b200a6c89c4958.js => 48.b5ecdbc517423af07ca4.js} | 6 +++---
...NSE.txt => 48.b5ecdbc517423af07ca4.js.LICENSE.txt} | 4 ++--
priv/static/static/js/48.b5ecdbc517423af07ca4.js.map | 1 +
priv/static/static/js/48.d7e479b200a6c89c4958.js.map | 1 -
priv/static/static/js/5948.2b7b4e97487f2539eb44.js | 2 --
.../static/static/js/5948.2b7b4e97487f2539eb44.js.map | 1 -
...a80a7373e4e5f8.js => 6464.eb9c90a1c948cde554e9.js} | 2 +-
.../static/static/js/6464.eb9c90a1c948cde554e9.js.map | 1 +
.../static/static/js/6464.fea96fa80a7373e4e5f8.js.map | 1 -
priv/static/static/js/7586.981b2305a0019f6042a5.js | 2 ++
.../static/static/js/7586.981b2305a0019f6042a5.js.map | 1 +
priv/static/static/js/7962.e25d40b042f8ee7389c3.js | 2 ++
.../static/static/js/7962.e25d40b042f8ee7389c3.js.map | 1 +
priv/static/static/js/9060.24271e167e0471a1a732.js | 2 ++
.../static/static/js/9060.24271e167e0471a1a732.js.map | 1 +
priv/static/static/js/9114.e761a1c6846fea99aaf1.js | 2 --
.../static/static/js/9114.e761a1c6846fea99aaf1.js.map | 1 -
priv/static/static/js/9801.99ace6b5dc657bf1a65b.js | 2 ++
.../static/static/js/9801.99ace6b5dc657bf1a65b.js.map | 1 +
priv/static/static/js/app.7c4b412b26221a7c8572.js | 2 ++
priv/static/static/js/app.7c4b412b26221a7c8572.js.map | 1 +
priv/static/static/js/app.8d2126d35dba9482db51.js | 2 --
priv/static/static/js/app.8d2126d35dba9482db51.js.map | 1 -
.../static/js/i18n/ar-json.4916f840147303aa65fe.js | 2 ++
.../js/i18n/ar-json.4916f840147303aa65fe.js.map | 1 +
.../static/js/i18n/ar-json.d09609af3224232857d6.js | 2 --
.../js/i18n/ar-json.d09609af3224232857d6.js.map | 1 -
.../static/js/i18n/eo-json.6c62eef99e850912498b.js | 2 ++
.../js/i18n/eo-json.6c62eef99e850912498b.js.map | 1 +
.../static/js/i18n/eo-json.d81690d5be30b23e516b.js | 2 --
.../js/i18n/eo-json.d81690d5be30b23e516b.js.map | 1 -
.../static/js/i18n/id-json.3e42564ce7a3a847ecb0.js | 2 --
.../js/i18n/id-json.3e42564ce7a3a847ecb0.js.map | 1 -
.../static/js/i18n/id-json.e5c9ee768155f88128b9.js | 2 ++
.../js/i18n/id-json.e5c9ee768155f88128b9.js.map | 1 +
.../static/js/i18n/ko-json.4bd28b26a7390a09afc2.js | 2 --
.../js/i18n/ko-json.4bd28b26a7390a09afc2.js.map | 1 -
.../static/js/i18n/ko-json.9029d09084bb22d8b705.js | 2 ++
.../js/i18n/ko-json.9029d09084bb22d8b705.js.map | 1 +
.../js/i18n/nan-TW-json.7f2789d8a461e86d1734.js | 2 ++
.../js/i18n/nan-TW-json.7f2789d8a461e86d1734.js.map | 1 +
...197374a5dac.js => zh-json.5831b903c3e6d281f122.js} | 4 ++--
.../js/i18n/zh-json.5831b903c3e6d281f122.js.map | 1 +
.../js/i18n/zh-json.63e4c9fe0197374a5dac.js.map | 1 -
.../js/i18n/zh_Hant-json.bfa569654a5cd74767ce.js.map | 1 -
...4767ce.js => zh_Hant-json.f7e1d0f4b873c60d6396.js} | 4 ++--
.../js/i18n/zh_Hant-json.f7e1d0f4b873c60d6396.js.map | 1 +
priv/static/sw-pleroma.js | 4 ++--
priv/static/sw-pleroma.js.map | 2 +-
72 files changed, 75 insertions(+), 63 deletions(-)
delete mode 100644 priv/static/static/css/5948.06d2a0d84620cba6a4fb.css
delete mode 100644 priv/static/static/css/5948.06d2a0d84620cba6a4fb.css.map
create mode 100644 priv/static/static/css/7586.0d43f70bc6240422f179.css
create mode 100644 priv/static/static/css/7586.0d43f70bc6240422f179.css.map
create mode 100644 priv/static/static/css/7962.76663e78ad5ea0bb0b90.css
create mode 100644 priv/static/static/css/7962.76663e78ad5ea0bb0b90.css.map
delete mode 100644 priv/static/static/css/9114.8def3b2b7fe70b3b3712.css
delete mode 100644 priv/static/static/css/9114.8def3b2b7fe70b3b3712.css.map
create mode 100644 priv/static/static/css/9801.cfe503d4c949ae0c3813.css
create mode 100644 priv/static/static/css/9801.cfe503d4c949ae0c3813.css.map
delete mode 100644 priv/static/static/css/app.48e52505beba5b9ab69b.css
delete mode 100644 priv/static/static/css/app.48e52505beba5b9ab69b.css.map
create mode 100644 priv/static/static/css/app.c18a2c80794a1b699a61.css
create mode 100644 priv/static/static/css/app.c18a2c80794a1b699a61.css.map
delete mode 100644 priv/static/static/js/159.3a9274574f1e33801c4a.js.map
rename priv/static/static/js/{159.3a9274574f1e33801c4a.js => 159.903e90c9de8ef6c67077.js} (96%)
create mode 100644 priv/static/static/js/159.903e90c9de8ef6c67077.js.map
delete mode 100644 priv/static/static/js/2724.e4840c73281069ba54ab.js
delete mode 100644 priv/static/static/js/2724.e4840c73281069ba54ab.js.map
create mode 100644 priv/static/static/js/3733.7060d1e6bca813125a0c.js
rename priv/static/static/js/{2724.e4840c73281069ba54ab.js.LICENSE.txt => 3733.7060d1e6bca813125a0c.js.LICENSE.txt} (100%)
create mode 100644 priv/static/static/js/3733.7060d1e6bca813125a0c.js.map
rename priv/static/static/js/{48.d7e479b200a6c89c4958.js => 48.b5ecdbc517423af07ca4.js} (59%)
rename priv/static/static/js/{48.d7e479b200a6c89c4958.js.LICENSE.txt => 48.b5ecdbc517423af07ca4.js.LICENSE.txt} (77%)
create mode 100644 priv/static/static/js/48.b5ecdbc517423af07ca4.js.map
delete mode 100644 priv/static/static/js/48.d7e479b200a6c89c4958.js.map
delete mode 100644 priv/static/static/js/5948.2b7b4e97487f2539eb44.js
delete mode 100644 priv/static/static/js/5948.2b7b4e97487f2539eb44.js.map
rename priv/static/static/js/{6464.fea96fa80a7373e4e5f8.js => 6464.eb9c90a1c948cde554e9.js} (98%)
create mode 100644 priv/static/static/js/6464.eb9c90a1c948cde554e9.js.map
delete mode 100644 priv/static/static/js/6464.fea96fa80a7373e4e5f8.js.map
create mode 100644 priv/static/static/js/7586.981b2305a0019f6042a5.js
create mode 100644 priv/static/static/js/7586.981b2305a0019f6042a5.js.map
create mode 100644 priv/static/static/js/7962.e25d40b042f8ee7389c3.js
create mode 100644 priv/static/static/js/7962.e25d40b042f8ee7389c3.js.map
create mode 100644 priv/static/static/js/9060.24271e167e0471a1a732.js
create mode 100644 priv/static/static/js/9060.24271e167e0471a1a732.js.map
delete mode 100644 priv/static/static/js/9114.e761a1c6846fea99aaf1.js
delete mode 100644 priv/static/static/js/9114.e761a1c6846fea99aaf1.js.map
create mode 100644 priv/static/static/js/9801.99ace6b5dc657bf1a65b.js
create mode 100644 priv/static/static/js/9801.99ace6b5dc657bf1a65b.js.map
create mode 100644 priv/static/static/js/app.7c4b412b26221a7c8572.js
create mode 100644 priv/static/static/js/app.7c4b412b26221a7c8572.js.map
delete mode 100644 priv/static/static/js/app.8d2126d35dba9482db51.js
delete mode 100644 priv/static/static/js/app.8d2126d35dba9482db51.js.map
create mode 100644 priv/static/static/js/i18n/ar-json.4916f840147303aa65fe.js
create mode 100644 priv/static/static/js/i18n/ar-json.4916f840147303aa65fe.js.map
delete mode 100644 priv/static/static/js/i18n/ar-json.d09609af3224232857d6.js
delete mode 100644 priv/static/static/js/i18n/ar-json.d09609af3224232857d6.js.map
create mode 100644 priv/static/static/js/i18n/eo-json.6c62eef99e850912498b.js
create mode 100644 priv/static/static/js/i18n/eo-json.6c62eef99e850912498b.js.map
delete mode 100644 priv/static/static/js/i18n/eo-json.d81690d5be30b23e516b.js
delete mode 100644 priv/static/static/js/i18n/eo-json.d81690d5be30b23e516b.js.map
delete mode 100644 priv/static/static/js/i18n/id-json.3e42564ce7a3a847ecb0.js
delete mode 100644 priv/static/static/js/i18n/id-json.3e42564ce7a3a847ecb0.js.map
create mode 100644 priv/static/static/js/i18n/id-json.e5c9ee768155f88128b9.js
create mode 100644 priv/static/static/js/i18n/id-json.e5c9ee768155f88128b9.js.map
delete mode 100644 priv/static/static/js/i18n/ko-json.4bd28b26a7390a09afc2.js
delete mode 100644 priv/static/static/js/i18n/ko-json.4bd28b26a7390a09afc2.js.map
create mode 100644 priv/static/static/js/i18n/ko-json.9029d09084bb22d8b705.js
create mode 100644 priv/static/static/js/i18n/ko-json.9029d09084bb22d8b705.js.map
create mode 100644 priv/static/static/js/i18n/nan-TW-json.7f2789d8a461e86d1734.js
create mode 100644 priv/static/static/js/i18n/nan-TW-json.7f2789d8a461e86d1734.js.map
rename priv/static/static/js/i18n/{zh-json.63e4c9fe0197374a5dac.js => zh-json.5831b903c3e6d281f122.js} (58%)
create mode 100644 priv/static/static/js/i18n/zh-json.5831b903c3e6d281f122.js.map
delete mode 100644 priv/static/static/js/i18n/zh-json.63e4c9fe0197374a5dac.js.map
delete mode 100644 priv/static/static/js/i18n/zh_Hant-json.bfa569654a5cd74767ce.js.map
rename priv/static/static/js/i18n/{zh_Hant-json.bfa569654a5cd74767ce.js => zh_Hant-json.f7e1d0f4b873c60d6396.js} (61%)
create mode 100644 priv/static/static/js/i18n/zh_Hant-json.f7e1d0f4b873c60d6396.js.map
diff --git a/priv/static/index.html b/priv/static/index.html
index 7dd5d0b78..52ff685c0 100644
--- a/priv/static/index.html
+++ b/priv/static/index.html
@@ -1 +1 @@
-To use Pleroma, please enable JavaScript.