From 01c45ddc9ead715131b3c583caa14fcf20845354 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 11 May 2019 11:26:46 +0200
Subject: [PATCH 01/56] Search: Use RUM index.

---
 .../mastodon_api/mastodon_api_controller.ex   |  6 ++--
 ...510135645_add_fts_index_to_objects_two.exs | 33 +++++++++++++++++++
 2 files changed, 36 insertions(+), 3 deletions(-)
 create mode 100644 priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs

diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 956736780..32677df95 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1019,12 +1019,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
         where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
         where:
           fragment(
-            "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
-            o.data,
+            "? @@ plainto_tsquery('english', ?)",
+            o.fts_content,
             ^query
           ),
         limit: 20,
-        order_by: [desc: :id]
+        order_by: [fragment("? <=> now()::date", o.inserted_at)]
       )
 
     Repo.all(q) ++ fetched
diff --git a/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs
new file mode 100644
index 000000000..14b964847
--- /dev/null
+++ b/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs
@@ -0,0 +1,33 @@
+defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do
+  use Ecto.Migration
+
+  def up do
+    drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+    alter table(:objects) do
+      add(:fts_content, :tsvector)
+    end
+
+    execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$
+    begin
+      new.fts_content := to_tsvector('english', new.data->>'content');
+      return new;
+    end
+    $$ LANGUAGE plpgsql")
+    execute("create index objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');")
+
+    execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects
+    FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()")
+
+    execute("UPDATE objects SET updated_at = NOW()")
+  end
+
+  def down do
+    execute "drop index objects_fts"
+    execute "drop trigger tsvectorupdate on objects"
+    execute "drop function objects_fts_update()"
+    alter table(:objects) do
+      remove(:fts_content, :tsvector)
+    end
+    create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+  end
+end

From f1e67bdc312ba16a37916024244d6cb9d4417c9e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 15 May 2019 15:28:01 +0200
Subject: [PATCH 02/56] Search: Add optional rum indexing / searching.

---
 config/config.exs                             |  2 +
 docs/config.md                                | 15 +++++++
 .../mastodon_api/mastodon_api_controller.ex   | 43 ++++++++++++++-----
 mix.exs                                       |  5 ++-
 mix.lock                                      | 10 ++---
 ...510135645_add_fts_index_to_objects_two.exs |  3 ++
 6 files changed, 62 insertions(+), 16 deletions(-)
 rename priv/repo/{migrations => optional_migrations/rum_indexing}/20190510135645_add_fts_index_to_objects_two.exs (91%)

diff --git a/config/config.exs b/config/config.exs
index 1e64b79a7..42e4cb4ce 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -476,6 +476,8 @@ config :pleroma, :oauth2,
   token_expires_in: 600,
   issue_new_refresh_token: true
 
+config :pleroma, :database, rum_enabled: false
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"
diff --git a/docs/config.md b/docs/config.md
index 43ea24d80..99cee25cd 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -537,3 +537,18 @@ Configure OAuth 2 provider capabilities:
 * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]`
 * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]`
 * `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays).
+
+## Database options
+
+### RUM indexing for full text search
+* `rum_enabled`: If RUM indexes should be used. Defaults to `false`.
+
+RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum.
+
+Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes.
+
+To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run:
+
+`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`
+
+This will probably take a long time.
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 32677df95..fc0100b1e 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1000,6 +1000,30 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
+  def status_search_query_with_gin(q, query) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
+          o.data,
+          ^query
+        ),
+      order_by: [desc: :id]
+    )
+  end
+
+  def status_search_query_with_rum(q, query) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "? @@ plainto_tsquery('english', ?)",
+          o.fts_content,
+          ^query
+        ),
+      order_by: [fragment("? <=> now()::date", o.inserted_at)]
+    )
+  end
+
   def status_search(user, query) do
     fetched =
       if Regex.match?(~r/https?:/, query) do
@@ -1013,20 +1037,19 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       end || []
 
     q =
-      from(
-        [a, o] in Activity.with_preloaded_object(Activity),
+      from([a, o] in Activity.with_preloaded_object(Activity),
         where: fragment("?->>'type' = 'Create'", a.data),
         where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
-        where:
-          fragment(
-            "? @@ plainto_tsquery('english', ?)",
-            o.fts_content,
-            ^query
-          ),
-        limit: 20,
-        order_by: [fragment("? <=> now()::date", o.inserted_at)]
+        limit: 20
       )
 
+    q =
+      if Pleroma.Config.get([:database, :rum_enabled]) do
+        status_search_query_with_rum(q, query)
+      else
+        status_search_query_with_gin(q, query)
+      end
+
     Repo.all(q) ++ fetched
   end
 
diff --git a/mix.exs b/mix.exs
index fae21f18d..f7955c6ca 100644
--- a/mix.exs
+++ b/mix.exs
@@ -65,7 +65,10 @@ defmodule Pleroma.Mixfile do
       {:plug_cowboy, "~> 2.0"},
       {:phoenix_pubsub, "~> 1.1"},
       {:phoenix_ecto, "~> 4.0"},
-      {:ecto_sql, "~>3.0.5"},
+      {:ecto_sql,
+       git: "https://github.com/elixir-ecto/ecto_sql",
+       ref: "e839a9a327b632d73533ac8105ba360bc831cf83",
+       override: true},
       {:postgrex, ">= 0.13.5"},
       {:gettext, "~> 0.15"},
       {:comeonin, "~> 4.1.1"},
diff --git a/mix.lock b/mix.lock
index 624c0fb35..d223803ab 100644
--- a/mix.lock
+++ b/mix.lock
@@ -16,12 +16,12 @@
   "cowlib": {:hex, :cowlib, "2.7.0", "3ef16e77562f9855a2605900cedb15c1462d76fb1be6a32fc3ae91973ee543d2", [:rebar3], [], "hexpm"},
   "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"},
   "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]},
-  "db_connection": {:hex, :db_connection, "2.0.5", "ddb2ba6761a08b2bb9ca0e7d260e8f4dd39067426d835c24491a321b7f92a4da", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"},
+  "db_connection": {:hex, :db_connection, "2.0.6", "bde2f85d047969c5b5800cb8f4b3ed6316c8cb11487afedac4aa5f93fd39abfa", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"},
   "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
   "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
-  "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
-  "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
+  "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
+  "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "e839a9a327b632d73533ac8105ba360bc831cf83", [ref: "e839a9a327b632d73533ac8105ba360bc831cf83"]},
   "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"},
   "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"},
   "ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"},
@@ -61,7 +61,7 @@
   "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"},
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"},
   "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"},
-  "postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
+  "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
   "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm"},
   "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"},
   "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"},
@@ -74,7 +74,7 @@
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
   "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"},
   "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},
-  "telemetry": {:hex, :telemetry, "0.3.0", "099a7f3ce31e4780f971b4630a3c22ec66d22208bc090fe33a2a3a6a67754a73", [:rebar3], [], "hexpm"},
+  "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},
   "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
   "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
   "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
diff --git a/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
similarity index 91%
rename from priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs
rename to priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
index 14b964847..09e6cbfb1 100644
--- a/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs
+++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
@@ -2,6 +2,7 @@ defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do
   use Ecto.Migration
 
   def up do
+    execute("create extension if not exists rum")
     drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
     alter table(:objects) do
       add(:fts_content, :tsvector)
@@ -19,6 +20,7 @@ defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do
     FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()")
 
     execute("UPDATE objects SET updated_at = NOW()")
+    execute("vacuum analyze")
   end
 
   def down do
@@ -29,5 +31,6 @@ defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do
       remove(:fts_content, :tsvector)
     end
     create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+    execute("vacuum analyze")
   end
 end

From a3fc7294da43a7e2d17626200e8d3bf5c05808d3 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 12:47:38 +0200
Subject: [PATCH 03/56] CI: Add rum variant testing.

---
 .gitlab-ci.yml | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f9745122a..3ea275127 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,6 +15,7 @@ cache:
 stages:
   - build
   - test
+  - test-rum
   - deploy
 
 before_script:
@@ -45,7 +46,7 @@ docs-build:
 unit-testing:
   stage: test
   services:
-  - name: postgres:9.6.2
+  - name: lainsoykaf/postgres-with-rum
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
     - mix deps.get
@@ -54,6 +55,20 @@ unit-testing:
     - mix test --trace --preload-modules
     - mix coveralls
 
+unit-testing-rum:
+  stage: test-rum
+  services:
+  - name: lainsoykaf/postgres-with-rum
+    command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+  script:
+    - "echo 'config :pleroma, :database, rum_enabled: true' >> config/test.exs"
+    - mix deps.get
+    - mix ecto.create
+    - mix ecto.migrate
+    - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
+    - mix test --trace --preload-modules
+    - mix coveralls
+
 lint:
   stage: test
   script:
@@ -65,7 +80,6 @@ analysis:
     - mix deps.get
     - mix credo --strict --only=warnings,todo,fixme,consistency,readability
 
-
 docs-deploy:
   stage: deploy
   image: alpine:3.9

From 022e6e4b44229be70f0ec6720a66610a0e2a403a Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 14:59:01 +0200
Subject: [PATCH 04/56] RUM: Remove vacuum analyze from migration

Can't be run in a trnasaction.
---
 .../20190510135645_add_fts_index_to_objects_two.exs             | 2 --
 1 file changed, 2 deletions(-)

diff --git a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
index 09e6cbfb1..b6a24441a 100644
--- a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
+++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
@@ -20,7 +20,6 @@ defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do
     FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()")
 
     execute("UPDATE objects SET updated_at = NOW()")
-    execute("vacuum analyze")
   end
 
   def down do
@@ -31,6 +30,5 @@ defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do
       remove(:fts_content, :tsvector)
     end
     create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
-    execute("vacuum analyze")
   end
 end

From 8784a7d1b40394cb1b4d5ce2d04b8cd057a57ceb Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 15:06:51 +0200
Subject: [PATCH 05/56] RUM: Set rum status by the environment.

---
 config/test.exs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/config/test.exs b/config/test.exs
index 40db66170..e1785d10f 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -63,6 +63,10 @@ config :pleroma, :app_account_creation, max_requests: 5
 
 config :pleroma, :http_security, report_uri: "https://endpoint.com"
 
+rum_enabled = System.get_env("RUM_ENABLED") == "true"
+config :pleroma, :database, rum_enabled: rum_enabled
+IO.puts("RUM enabled: #{rum_enabled}")
+
 try do
   import_config "test.secret.exs"
 rescue

From ef63cf70883f4c7e3814ebac907f000da835ea10 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 15:21:29 +0200
Subject: [PATCH 06/56] CI: Use the correct image with the correct hostname.

---
 .gitlab-ci.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3ea275127..8b5131dc3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,7 +15,6 @@ cache:
 stages:
   - build
   - test
-  - test-rum
   - deploy
 
 before_script:
@@ -47,6 +46,7 @@ unit-testing:
   stage: test
   services:
   - name: lainsoykaf/postgres-with-rum
+    alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
     - mix deps.get
@@ -56,18 +56,19 @@ unit-testing:
     - mix coveralls
 
 unit-testing-rum:
-  stage: test-rum
+  stage: test
   services:
   - name: lainsoykaf/postgres-with-rum
+    alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+  variables:
+    RUM_ENABLED: "true"
   script:
-    - "echo 'config :pleroma, :database, rum_enabled: true' >> config/test.exs"
     - mix deps.get
     - mix ecto.create
     - mix ecto.migrate
     - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
     - mix test --trace --preload-modules
-    - mix coveralls
 
 lint:
   stage: test

From f959bf7aa6b878ee5b669c4caabd5cdc4cc2dc9e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 18:21:11 +0200
Subject: [PATCH 07/56] MongooseIM: Add basic integration endpoints.

---
 .../web/mongooseim/mongoose_im_controller.ex  | 41 +++++++++++++
 lib/pleroma/web/router.ex                     |  5 ++
 .../mongoose_im_controller_test.exs           | 59 +++++++++++++++++++
 3 files changed, 105 insertions(+)
 create mode 100644 lib/pleroma/web/mongooseim/mongoose_im_controller.ex
 create mode 100644 test/web/mongooseim/mongoose_im_controller_test.exs

diff --git a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex
new file mode 100644
index 000000000..f8c634653
--- /dev/null
+++ b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex
@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MongooseIM.MongooseIMController do
+  use Pleroma.Web, :controller
+  alias Comeonin.Pbkdf2
+  alias Pleroma.User
+  alias Pleroma.Repo
+
+  def user_exists(conn, %{"user" => username}) do
+    with %User{} <- Repo.get_by(User, nickname: username, local: true) do
+      conn
+      |> json(true)
+    else
+      _ ->
+        conn
+        |> put_status(:not_found)
+        |> json(false)
+    end
+  end
+
+  def check_password(conn, %{"user" => username, "pass" => password}) do
+    with %User{password_hash: password_hash} <-
+           Repo.get_by(User, nickname: username, local: true),
+         true <- Pbkdf2.checkpw(password, password_hash) do
+      conn
+      |> json(true)
+    else
+      false ->
+        conn
+        |> put_status(403)
+        |> json(false)
+
+      _ ->
+        conn
+        |> put_status(:not_found)
+        |> json(false)
+    end
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 6a4e4a1d4..552778c92 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -704,6 +704,11 @@ defmodule Pleroma.Web.Router do
     end
   end
 
+  scope "/", Pleroma.Web.MongooseIM do
+    get("/user_exists", MongooseIMController, :user_exists)
+    get("/check_password", MongooseIMController, :check_password)
+  end
+
   scope "/", Fallback do
     get("/registration/:token", RedirectController, :registration_page)
     get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta)
diff --git a/test/web/mongooseim/mongoose_im_controller_test.exs b/test/web/mongooseim/mongoose_im_controller_test.exs
new file mode 100644
index 000000000..eb83999bb
--- /dev/null
+++ b/test/web/mongooseim/mongoose_im_controller_test.exs
@@ -0,0 +1,59 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MongooseIMController do
+  use Pleroma.Web.ConnCase
+  import Pleroma.Factory
+
+  test "/user_exists", %{conn: conn} do
+    _user = insert(:user, nickname: "lain")
+    _remote_user = insert(:user, nickname: "alice", local: false)
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :user_exists), user: "lain")
+      |> json_response(200)
+
+    assert res == true
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :user_exists), user: "alice")
+      |> json_response(404)
+
+    assert res == false
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :user_exists), user: "bob")
+      |> json_response(404)
+
+    assert res == false
+  end
+
+  test "/check_password", %{conn: conn} do
+    user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt("cool"))
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :check_password), user: user.nickname, pass: "cool")
+      |> json_response(200)
+
+    assert res == true
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :check_password), user: user.nickname, pass: "uncool")
+      |> json_response(403)
+
+    assert res == false
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :check_password), user: "nobody", pass: "cool")
+      |> json_response(404)
+
+    assert res == false
+  end
+end

From 075eecec907b0a623a90eed44a0378a6812d8037 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 18:32:30 +0200
Subject: [PATCH 08/56] Linting.

---
 lib/pleroma/web/mongooseim/mongoose_im_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex
index f8c634653..489d5d3a5 100644
--- a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex
+++ b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex
@@ -5,8 +5,8 @@
 defmodule Pleroma.Web.MongooseIM.MongooseIMController do
   use Pleroma.Web, :controller
   alias Comeonin.Pbkdf2
-  alias Pleroma.User
   alias Pleroma.Repo
+  alias Pleroma.User
 
   def user_exists(conn, %{"user" => username}) do
     with %User{} <- Repo.get_by(User, nickname: username, local: true) do

From 78588dbd80580fbef53819dd87ee8fcc26cb09e9 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 17 May 2019 18:49:10 +0000
Subject: [PATCH 09/56] mrf: simple policy: mark all posts instead of posts
 with media as sensitive if they match media_nsfw

---
 lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index 50426e920..9627c3400 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -48,10 +48,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
          %{host: actor_host} = _actor_info,
          %{
            "type" => "Create",
-           "object" => %{"attachment" => child_attachment} = child_object
+           "object" => child_object
          } = object
-       )
-       when length(child_attachment) > 0 do
+       ) do
     object =
       if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_nsfw]), actor_host) do
         tags = (child_object["tag"] || []) ++ ["nsfw"]

From 0da1233e8e093ff7c69994f9e81d58611be60507 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 17 May 2019 18:49:43 +0000
Subject: [PATCH 10/56] rich media: suppress link previews if post is marked as
 sensitive

---
 lib/pleroma/web/rich_media/helpers.ex |  1 +
 test/web/rich_media/helpers_test.exs  | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 0162a5be9..9bc8f2559 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -24,6 +24,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do
     with true <- Pleroma.Config.get([:rich_media, :enabled]),
          %Object{} = object <- Object.normalize(activity),
+         false <- object.data["sensitive"] || false,
          {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),
          :ok <- validate_page_url(page_url),
          {:ok, rich_media} <- Parser.parse(page_url) do
diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs
index 60d93768f..6e23392ca 100644
--- a/test/web/rich_media/helpers_test.exs
+++ b/test/web/rich_media/helpers_test.exs
@@ -1,6 +1,7 @@
 defmodule Pleroma.Web.RichMedia.HelpersTest do
   use Pleroma.DataCase
 
+  alias Pleroma.Object
   alias Pleroma.Web.CommonAPI
 
   import Pleroma.Factory
@@ -59,4 +60,24 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do
 
     Pleroma.Config.put([:rich_media, :enabled], false)
   end
+
+  test "refuses to crawl URLs from posts marked sensitive" do
+    user = insert(:user)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        "status" => "http://example.com/ogp",
+        "spoiler_text" => "."
+      })
+
+    %Object{} = object = Object.normalize(activity)
+
+    assert object.data["sensitive"]
+
+    Pleroma.Config.put([:rich_media, :enabled], true)
+
+    assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+
+    Pleroma.Config.put([:rich_media, :enabled], false)
+  end
 end

From d3b8cd342ff6d1fe87bcfe6305424faf49666252 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 17 May 2019 19:03:19 +0000
Subject: [PATCH 11/56] http: request builder: send user-agent when making
 requests

---
 lib/pleroma/http/request_builder.ex | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex
index 5f2cff2c0..522728da1 100644
--- a/lib/pleroma/http/request_builder.ex
+++ b/lib/pleroma/http/request_builder.ex
@@ -45,8 +45,9 @@ defmodule Pleroma.HTTP.RequestBuilder do
   Add headers to the request
   """
   @spec headers(map(), list(tuple)) :: map()
-  def headers(request, h) do
-    Map.put_new(request, :headers, h)
+  def headers(request, header_list) do
+    header_list = header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
+    Map.put_new(request, :headers, header_list)
   end
 
   @doc """

From 290f5b2cfe91dd2acba56f79ef137f15c68a0db0 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 17 May 2019 20:28:58 +0000
Subject: [PATCH 12/56] config: make sending the user agent configurable,
 disable sending the user agent in tests

---
 config/config.exs                   | 1 +
 config/test.exs                     | 2 ++
 lib/pleroma/http/request_builder.ex | 8 +++++++-
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index e82f08e07..9a10b0ff7 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -192,6 +192,7 @@ config :tesla, adapter: Tesla.Adapter.Hackney
 # Configures http settings, upstream proxy etc.
 config :pleroma, :http,
   proxy_url: nil,
+  send_user_agent: true,
   adapter: [
     ssl_options: [
       # We don't support TLS v1.3 yet
diff --git a/config/test.exs b/config/test.exs
index e1785d10f..6100989c4 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -63,6 +63,8 @@ config :pleroma, :app_account_creation, max_requests: 5
 
 config :pleroma, :http_security, report_uri: "https://endpoint.com"
 
+config :pleroma, :http, send_user_agent: false
+
 rum_enabled = System.get_env("RUM_ENABLED") == "true"
 config :pleroma, :database, rum_enabled: rum_enabled
 IO.puts("RUM enabled: #{rum_enabled}")
diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex
index 522728da1..e23457999 100644
--- a/lib/pleroma/http/request_builder.ex
+++ b/lib/pleroma/http/request_builder.ex
@@ -46,7 +46,13 @@ defmodule Pleroma.HTTP.RequestBuilder do
   """
   @spec headers(map(), list(tuple)) :: map()
   def headers(request, header_list) do
-    header_list = header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
+    header_list =
+      if Pleroma.Config.get([:http, :send_user_agent]) do
+        header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
+      else
+        header_list
+      end
+
     Map.put_new(request, :headers, header_list)
   end
 

From c234ce546a769747f436c19fee99bed2a7a58f3b Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 17 May 2019 20:31:39 +0000
Subject: [PATCH 13/56] add CHANGELOG entry

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 12c439135..5de9ae292 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -102,6 +102,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Correct `reblogged`, `favourited`, and `bookmarked` values in the reblog status JSON
 - Mastodon API: Exposing default scope of the user to anyone
 - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`]
+- User-Agent is now sent correctly for all HTTP requests.
 
 ## Removed
 - Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations`

From dc081595385084fe6b382e4b38c17cb51cf0a611 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 17 May 2019 20:42:51 +0000
Subject: [PATCH 14/56] also suppress link previews from posts marked #nsfw

---
 lib/pleroma/web/common_api/common_api.ex |  4 +++-
 lib/pleroma/web/common_api/utils.ex      |  4 +++-
 test/web/rich_media/helpers_test.exs     | 21 ++++++++++++++++++++-
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index a599ffee5..5a312d673 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -157,6 +157,7 @@ defmodule Pleroma.Web.CommonAPI do
          {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility),
          context <- make_context(in_reply_to),
          cw <- data["spoiler_text"] || "",
+         sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}),
          full_payload <- String.trim(status <> cw),
          length when length in 1..limit <- String.length(full_payload),
          object <-
@@ -169,7 +170,8 @@ defmodule Pleroma.Web.CommonAPI do
              in_reply_to,
              tags,
              cw,
-             cc
+             cc,
+             sensitive
            ),
          object <-
            Map.put(
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index bee2fd159..8d6160976 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -223,7 +223,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do
         in_reply_to,
         tags,
         cw \\ nil,
-        cc \\ []
+        cc \\ [],
+        sensitive \\ false
       ) do
     object = %{
       "type" => "Note",
@@ -231,6 +232,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
       "cc" => cc,
       "content" => content_html,
       "summary" => cw,
+      "sensitive" => sensitive,
       "context" => context,
       "attachment" => attachments,
       "actor" => actor,
diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs
index 6e23392ca..53b0596f5 100644
--- a/test/web/rich_media/helpers_test.exs
+++ b/test/web/rich_media/helpers_test.exs
@@ -67,7 +67,26 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do
     {:ok, activity} =
       CommonAPI.post(user, %{
         "status" => "http://example.com/ogp",
-        "spoiler_text" => "."
+        "sensitive" => true
+      })
+
+    %Object{} = object = Object.normalize(activity)
+
+    assert object.data["sensitive"]
+
+    Pleroma.Config.put([:rich_media, :enabled], true)
+
+    assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+
+    Pleroma.Config.put([:rich_media, :enabled], false)
+  end
+
+  test "refuses to crawl URLs from posts tagged NSFW" do
+    user = insert(:user)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        "status" => "http://example.com/ogp #nsfw"
       })
 
     %Object{} = object = Object.normalize(activity)

From c4a55e167afcfd25cf4c1efb46886f2defe72c22 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 17 May 2019 20:43:31 +0000
Subject: [PATCH 15/56] add Changelog entry

---
 CHANGELOG.md                        | 1 +
 lib/pleroma/web/common_api/utils.ex | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 12c439135..a73bf47d2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -71,6 +71,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Deps: Updated Ecto to 3.0.7
 - Don't ship finmoji by default, they can be installed as an emoji pack
 - Hide deactivated users and their statuses
+- Posts which are marked sensitive or tagged nsfw no longer have link previews.
 
 ### Fixed
 - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended.
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 8d6160976..d93c0d46e 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -232,7 +232,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
       "cc" => cc,
       "content" => content_html,
       "summary" => cw,
-      "sensitive" => sensitive,
+      "sensitive" => !Enum.member?(["false", "False", "0", false], sensitive),
       "context" => context,
       "attachment" => attachments,
       "actor" => actor,

From 2375e9a95ba9042958ff7e8f75830df4ab53fed2 Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Mon, 20 May 2019 06:02:50 +0800
Subject: [PATCH 16/56] Add report filtering to MRF.SimplePolicy

---
 config/config.exs                             |  1 +
 .../web/activity_pub/mrf/simple_policy.ex     | 13 ++++++++-
 .../activity_pub/mrf/simple_policy_test.exs   | 28 +++++++++++++++++++
 3 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 9a10b0ff7..bab47a8a2 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -298,6 +298,7 @@ config :pleroma, :mrf_simple,
   media_removal: [],
   media_nsfw: [],
   federated_timeline_removal: [],
+  report_removal: [],
   reject: [],
   accept: []
 
diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index 9627c3400..7190652d2 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -94,6 +94,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
     {:ok, object}
   end
 
+  defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do
+    if actor_host in Pleroma.Config.get([:mrf_simple, :report_removal]) do
+      {:reject, nil}
+    else
+      {:ok, object}
+    end
+  end
+
+  defp check_report_removal(_actor_info, object), do: {:ok, object}
+
   @impl true
   def filter(object) do
     actor_info = URI.parse(object["actor"])
@@ -102,7 +112,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
          {:ok, object} <- check_reject(actor_info, object),
          {:ok, object} <- check_media_removal(actor_info, object),
          {:ok, object} <- check_media_nsfw(actor_info, object),
-         {:ok, object} <- check_ftl_removal(actor_info, object) do
+         {:ok, object} <- check_ftl_removal(actor_info, object),
+         {:ok, object} <- check_report_removal(actor_info, object) do
       {:ok, object}
     else
       _e -> {:reject, nil}
diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs
index 1e0511975..74af7dcde 100644
--- a/test/web/activity_pub/mrf/simple_policy_test.exs
+++ b/test/web/activity_pub/mrf/simple_policy_test.exs
@@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do
       media_removal: [],
       media_nsfw: [],
       federated_timeline_removal: [],
+      report_removal: [],
       reject: [],
       accept: []
     )
@@ -85,6 +86,33 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do
     }
   end
 
+  describe "when :report_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :report_removal], [])
+      report_message = build_report_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(report_message) == {:ok, report_message}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :report_removal], ["remote.instance"])
+      report_message = build_report_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(report_message) == {:reject, nil}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+  end
+
+  defp build_report_message do
+    %{
+      "actor" => "https://remote.instance/users/bob",
+      "type" => "Flag"
+    }
+  end
+
   describe "when :federated_timeline_removal" do
     test "is empty" do
       Config.put([:mrf_simple, :federated_timeline_removal], [])

From 54e9cb5c2db580bc12441f3651fa87a7b976137d Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 12:39:23 +0100
Subject: [PATCH 17/56] Add API endpoints for a custom user mascot

---
 docs/api/pleroma_api.md                       | 39 +++++++++++++++++++
 lib/pleroma/user/info.ex                      | 21 ++++++++++
 .../mastodon_api/mastodon_api_controller.ex   | 36 ++++++++++++++++-
 lib/pleroma/web/router.ex                     |  3 ++
 4 files changed, 98 insertions(+), 1 deletion(-)

diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md
index dd0b6ca73..4d99a2d2b 100644
--- a/docs/api/pleroma_api.md
+++ b/docs/api/pleroma_api.md
@@ -252,6 +252,45 @@ See [Admin-API](Admin-API.md)
 ]
 ```
 
+## `/api/v1/pleroma/mascot`
+### Gets user mascot image
+* Method `GET`
+* Authentication: required
+
+* Response: JSON. Returns a mastodon media attachment entity.
+* Example response:
+```json
+{
+    "id": "abcdefg",
+    "url": "https://pleroma.example.org/media/abcdefg.png",
+    "type": "image",
+    "pleroma": {
+        "mime_type": "image/png"
+    }
+}
+```
+
+### Updates user mascot image
+* Method `PUT`
+* Authentication: required
+* Params:
+    * `image`: Multipart image
+* Response: JSON. Returns a mastodon media attachment entity
+  when successful, otherwise returns HTTP 415 `{"error": "error_msg"}`
+* Example response:
+```json
+{
+    "id": "abcdefg",
+    "url": "https://pleroma.example.org/media/abcdefg.png",
+    "type": "image",
+    "pleroma": {
+        "mime_type": "image/png"
+    }
+}
+```
+* Note: Behaves exactly the same as `POST /api/v1/upload`.
+  Can only accept images - any attempt to upload non-image files will be met with `HTTP 415 Unsupported Media Type`.
+
 ## `/api/pleroma/notification_settings`
 ### Updates user notification settings
 * Method `PUT`
diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index 5f0cefc00..ffcd06e3e 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -43,6 +43,19 @@ defmodule Pleroma.User.Info do
     field(:hide_favorites, :boolean, default: true)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:flavour, :string, default: nil)
+
+    field(:mascot, :map,
+      default: %{
+        id: "pleromatan",
+        url: "/images/pleroma-fox-tan-smol.png",
+        type: "image",
+        preview_url: "/images/pleroma-fox-tan-smol.png",
+        pleroma: %{
+          mime_type: "image/png"
+        }
+      }
+    )
+
     field(:emoji, {:array, :map}, default: [])
 
     field(:notification_settings, :map,
@@ -248,6 +261,14 @@ defmodule Pleroma.User.Info do
     |> validate_required([:flavour])
   end
 
+  def mascot_update(info, url) do
+    params = %{mascot: url}
+
+    info
+    |> cast(params, [:mascot])
+    |> validate_required([:mascot])
+  end
+
   def set_source_data(info, source_data) do
     params = %{source_data: source_data}
 
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 1051861ff..67f363859 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -707,6 +707,40 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
+  def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do
+    with {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)),
+         %{} = attachment_data <- Map.put(object.data, "id", object.id),
+         %{type: type} = rendered <-
+           StatusView.render("attachment.json", %{attachment: attachment_data}) do
+      # Reject if not an image
+      if type == "image" do
+        # Sure!
+        # Save to the user's info
+        info_changeset = User.Info.mascot_update(user.info, rendered)
+
+        user_changeset =
+          user
+          |> Ecto.Changeset.change()
+          |> Ecto.Changeset.put_embed(:info, info_changeset)
+
+        {:ok, _user} = User.update_and_set_cache(user_changeset)
+
+        conn
+        |> json(rendered)
+      else
+        conn
+        |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"}))
+      end
+    end
+  end
+
+  def get_mascot(%{assigns: %{user: user}} = conn, _params) do
+    %{info: %{mascot: mascot}} = user
+
+    conn
+    |> json(mascot)
+  end
+
   def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
          %Object{data: %{"likes" => likes}} <- Object.normalize(object) do
@@ -1329,7 +1363,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
             display_sensitive_media: false,
             reduce_motion: false,
             max_toot_chars: limit,
-            mascot: "/images/pleroma-fox-tan-smol.png"
+            mascot: Map.get(user.info.mascot, "url", "/images/pleroma-fox-tan-smol.png")
           },
           rights: %{
             delete_others_notice: present?(user.info.is_moderator),
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 6a4e4a1d4..4c29b24eb 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -352,6 +352,9 @@ defmodule Pleroma.Web.Router do
 
       post("/pleroma/flavour/:flavour", MastodonAPIController, :set_flavour)
 
+      get("/pleroma/mascot", MastodonAPIController, :get_mascot)
+      put("/pleroma/mascot", MastodonAPIController, :set_mascot)
+
       post("/reports", MastodonAPIController, :reports)
     end
 

From e81f0fc6d45249dd70656c58af926c21c70c482f Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 12:58:06 +0100
Subject: [PATCH 18/56] Add mascot get/set tests

---
 .../mastodon_api/mastodon_api_controller.ex   |   1 +
 test/fixtures/sound.mp3                       | Bin 0 -> 521 bytes
 .../mastodon_api_controller_test.exs          |  65 ++++++++++++++++++
 3 files changed, 66 insertions(+)
 create mode 100644 test/fixtures/sound.mp3

diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 67f363859..d7f095a1f 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -729,6 +729,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
         |> json(rendered)
       else
         conn
+        |> put_resp_content_type("application/json")
         |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"}))
       end
     end
diff --git a/test/fixtures/sound.mp3 b/test/fixtures/sound.mp3
new file mode 100644
index 0000000000000000000000000000000000000000..9f0f661a3bc34b6366304e9fd1d4fe8513411083
GIT binary patch
literal 521
zcmezWd%_V0bP$o5mkt!;2VzDB1}091|Fj1{y8?V1eO-<93=IrecEX$_s-VK;;K;!E
q0OXGES_-dW5+jBF|6AY)1M>j}#w9=>D=;vaG%zr*zym6jY5)LcXO3V1

literal 0
HcmV?d00001

diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
index cbff141c8..87e1c105d 100644
--- a/test/web/mastodon_api/mastodon_api_controller_test.exs
+++ b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -1455,6 +1455,71 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
     assert object.data["actor"] == User.ap_id(user)
   end
 
+  test "mascot upload", %{conn: conn} do
+    user = insert(:user)
+
+    non_image_file = %Plug.Upload{
+      content_type: "audio/mpeg",
+      path: Path.absname("test/fixtures/sound.mp3"),
+      filename: "sound.mp3"
+    }
+
+    conn =
+      conn
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => non_image_file})
+
+    assert json_response(conn, 415)
+
+    file = %Plug.Upload{
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image.jpg"),
+      filename: "an_image.jpg"
+    }
+
+    conn =
+      build_conn()
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => file})
+
+    assert %{"id" => _, "type" => image} = json_response(conn, 200)
+  end
+
+  test "mascot retrieving", %{conn: conn} do
+    user = insert(:user)
+    # When user hasn't set a mascot, we should just get pleroma tan back
+    conn =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/pleroma/mascot")
+
+    assert %{"url" => url} = json_response(conn, 200)
+    assert url =~ "pleroma-fox-tan-smol"
+
+    # When a user sets their mascot, we should get that back
+    file = %Plug.Upload{
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image.jpg"),
+      filename: "an_image.jpg"
+    }
+
+    conn = 
+      build_conn()
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => file})
+    assert json_response(conn, 200)
+
+    user = User.get_cached_by_id(user.id)
+
+    conn =
+      build_conn()
+      |> assign(:user, user)
+      |> get("/api/v1/pleroma/mascot")
+
+    assert %{"url" => url, "type" => "image"} = json_response(conn, 200)
+    assert url =~ "an_image"
+  end
+
   test "hashtag timeline", %{conn: conn} do
     following = insert(:user)
 

From dc916ba15f0fd77afa015084849f082065ed6f74 Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 12:58:17 +0100
Subject: [PATCH 19/56] Format mascot tests

---
 test/web/mastodon_api/mastodon_api_controller_test.exs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
index 87e1c105d..1d9f5a816 100644
--- a/test/web/mastodon_api/mastodon_api_controller_test.exs
+++ b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -1503,10 +1503,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
       filename: "an_image.jpg"
     }
 
-    conn = 
+    conn =
       build_conn()
       |> assign(:user, user)
       |> put("/api/v1/pleroma/mascot", %{"file" => file})
+
     assert json_response(conn, 200)
 
     user = User.get_cached_by_id(user.id)

From 3d0d9e7a5680bcb1b79e5f4aab0e8c514fc9e5ff Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 13:10:04 +0100
Subject: [PATCH 20/56] Use string map for default mascot

---
 lib/pleroma/user/info.ex | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index ffcd06e3e..e76d04d7f 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -46,12 +46,12 @@ defmodule Pleroma.User.Info do
 
     field(:mascot, :map,
       default: %{
-        id: "pleromatan",
-        url: "/images/pleroma-fox-tan-smol.png",
-        type: "image",
-        preview_url: "/images/pleroma-fox-tan-smol.png",
-        pleroma: %{
-          mime_type: "image/png"
+        "id" => "pleromatan",
+        "url" => "/images/pleroma-fox-tan-smol.png",
+        "type" => "image",
+        "preview_url" => "/images/pleroma-fox-tan-smol.png",
+        "pleroma" => %{
+          "mime_type" => "image/png"
         }
       }
     )

From d835810610e5e7660a56d305bc8509e38c642942 Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 14:19:42 +0100
Subject: [PATCH 21/56] Add changelog entry for mascot config

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 87101db30..07e6bce31 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `report_uri` option
 - Pleroma API: User subscriptions
 - Pleroma API: Healthcheck endpoint
+- Pleroma API: `/api/v1/pleroma/mascot` per-user frontend mascot configuration endpoints
 - Admin API: Endpoints for listing/revoking invite tokens
 - Admin API: Endpoints for making users follow/unfollow each other
 - Admin API: added filters (role, tags, email, name) for users endpoint

From daeae8e2e7c506b72c66dea6ac790408f948ec16 Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 16:12:55 +0100
Subject: [PATCH 22/56] Move default mascot configuration to `config/`

---
 config/config.exs                             | 13 ++++++++++++
 docs/config.md                                | 10 ++++++++++
 lib/pleroma/user.ex                           | 20 +++++++++++++++++++
 lib/pleroma/user/info.ex                      | 14 +------------
 .../mastodon_api/mastodon_api_controller.ex   |  4 ++--
 5 files changed, 46 insertions(+), 15 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index bab47a8a2..72908266d 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -276,6 +276,19 @@ config :pleroma, :frontend_configurations,
     showInstanceSpecificPanel: true
   }
 
+config :pleroma, :assets,
+  mascots: [
+    pleroma_fox_tan: %{
+      url: "/images/pleroma-fox-tan-smol.png",
+      mime_type: "image/png"
+    },
+    pleroma_fox_tan_shy: %{
+      url: "/images/pleroma-fox-tan-shy.png",
+      mime_type: "image/png"
+    }
+  ],
+  default_mascot: :pleroma_fox_tan
+
 config :pleroma, :activitypub,
   accept_blocks: true,
   unfollow_blocked: true,
diff --git a/docs/config.md b/docs/config.md
index 450d73fda..197326bbd 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -203,6 +203,16 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i
 * `hide_post_stats`: Hide notices statistics(repeats, favorites, …)
 * `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …)
 
+## :assets
+
+This section configures assets to be used with various frontends. Currently the only option
+relates to mascots on the mastodon frontend
+
+* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a
+  `mime_type` key.
+* `default_mascot`: An element from `mascots` - This will be used as the default mascot
+  on MastoFE (default: `:pleroma_fox_tan`)
+
 ## :mrf_simple
 * `media_removal`: List of instances to remove medias from
 * `media_nsfw`: List of instances to put medias as NSFW(sensitive) from
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 28da310ee..05fe58f7c 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1402,4 +1402,24 @@ defmodule Pleroma.User do
     |> put_embed(:info, info_changeset)
     |> update_and_set_cache()
   end
+
+  def get_mascot(%{info: %{mascot: %{} = mascot}}) when not is_nil(mascot) do
+    mascot
+  end
+
+  def get_mascot(%{info: %{mascot: mascot}}) when is_nil(mascot) do
+    # use instance-default
+    config = Pleroma.Config.get([:assets, :mascots])
+    default_mascot = Pleroma.Config.get([:assets, :default_mascot])
+    mascot = Keyword.get(config, default_mascot)
+
+    %{
+      "id" => "default-mascot",
+      "url" => mascot[:url],
+      "preview_url" => mascot[:url],
+      "pleroma" => %{
+        "mime_type" => mascot[:mime_type]
+      }
+    }
+  end
 end
diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index e76d04d7f..6397e2737 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -43,19 +43,7 @@ defmodule Pleroma.User.Info do
     field(:hide_favorites, :boolean, default: true)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:flavour, :string, default: nil)
-
-    field(:mascot, :map,
-      default: %{
-        "id" => "pleromatan",
-        "url" => "/images/pleroma-fox-tan-smol.png",
-        "type" => "image",
-        "preview_url" => "/images/pleroma-fox-tan-smol.png",
-        "pleroma" => %{
-          "mime_type" => "image/png"
-        }
-      }
-    )
-
+    field(:mascot, :map, default: nil)
     field(:emoji, {:array, :map}, default: [])
 
     field(:notification_settings, :map,
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index d7f095a1f..1ec0f30a1 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -736,7 +736,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def get_mascot(%{assigns: %{user: user}} = conn, _params) do
-    %{info: %{mascot: mascot}} = user
+    mascot = User.get_mascot(user)
 
     conn
     |> json(mascot)
@@ -1364,7 +1364,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
             display_sensitive_media: false,
             reduce_motion: false,
             max_toot_chars: limit,
-            mascot: Map.get(user.info.mascot, "url", "/images/pleroma-fox-tan-smol.png")
+            mascot: User.get_mascot(user)["url"]
           },
           rights: %{
             delete_others_notice: present?(user.info.is_moderator),

From eb02edcad9cb0d65fc216408960aec63713e5d2b Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Tue, 21 May 2019 00:35:46 +0800
Subject: [PATCH 23/56] Add virtual :thread_muted? field

that may be set when fetching activities
---
 lib/pleroma/activity.ex                       | 12 +++++++++
 lib/pleroma/web/activity_pub/activity_pub.ex  |  8 ++++++
 .../web/mastodon_api/views/status_view.ex     |  8 +++++-
 .../web/twitter_api/views/activity_view.ex    |  8 +++++-
 test/activity_test.exs                        | 26 +++++++++++++++++++
 test/user_test.exs                            |  2 +-
 6 files changed, 61 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 4e54b15ba..99589590c 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Activity do
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
+  alias Pleroma.ThreadMute
   alias Pleroma.User
 
   import Ecto.Changeset
@@ -37,6 +38,7 @@ defmodule Pleroma.Activity do
     field(:local, :boolean, default: true)
     field(:actor, :string)
     field(:recipients, {:array, :string}, default: [])
+    field(:thread_muted?, :boolean, virtual: true)
     # This is a fake relation, do not use outside of with_preloaded_bookmark/get_bookmark
     has_one(:bookmark, Bookmark)
     has_many(:notifications, Notification, on_delete: :delete_all)
@@ -90,6 +92,16 @@ defmodule Pleroma.Activity do
 
   def with_preloaded_bookmark(query, _), do: query
 
+  def with_set_thread_muted_field(query, %User{} = user) do
+    from([a] in query,
+      left_join: tm in ThreadMute,
+      on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data),
+      select: %Activity{a | thread_muted?: not is_nil(tm.id)}
+    )
+  end
+
+  def with_set_thread_muted_field(query, _), do: query
+
   def get_by_ap_id(ap_id) do
     Repo.one(
       from(
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 5c3156978..3d9679ec0 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -834,6 +834,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> Activity.with_preloaded_bookmark(opts["user"])
   end
 
+  defp maybe_set_thread_muted_field(query, %{"skip_preload" => true}), do: query
+
+  defp maybe_set_thread_muted_field(query, opts) do
+    query
+    |> Activity.with_set_thread_muted_field(opts["user"])
+  end
+
   defp maybe_order(query, %{order: :desc}) do
     query
     |> order_by(desc: :id)
@@ -852,6 +859,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     base_query
     |> maybe_preload_objects(opts)
     |> maybe_preload_bookmarks(opts)
+    |> maybe_set_thread_muted_field(opts)
     |> maybe_order(opts)
     |> restrict_recipients(recipients, opts["user"])
     |> restrict_tag(opts)
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index c93d915e5..e55f9b96e 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -157,6 +157,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
     bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil
 
+    thread_muted? =
+      case activity.thread_muted? do
+        thread_muted? when is_boolean(thread_muted?) -> thread_muted?
+        nil -> CommonAPI.thread_muted?(user, activity)
+      end
+
     attachment_data = object.data["attachment"] || []
     attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment)
 
@@ -228,7 +234,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       reblogged: reblogged?(activity, opts[:for]),
       favourited: present?(favorited),
       bookmarked: present?(bookmarked),
-      muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user),
+      muted: thread_muted? || User.mutes?(opts[:for], user),
       pinned: pinned?(activity, user),
       sensitive: sensitive,
       spoiler_text: summary_html,
diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex
index 44bcafe0e..e84af84dc 100644
--- a/lib/pleroma/web/twitter_api/views/activity_view.ex
+++ b/lib/pleroma/web/twitter_api/views/activity_view.ex
@@ -284,6 +284,12 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
         Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
       )
 
+    thread_muted? =
+      case activity.thread_muted? do
+        thread_muted? when is_boolean(thread_muted?) -> thread_muted?
+        nil -> CommonAPI.thread_muted?(user, activity)
+      end
+
     %{
       "id" => activity.id,
       "uri" => object.data["id"],
@@ -314,7 +320,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
       "summary" => summary,
       "summary_html" => summary |> Formatter.emojify(object.data["emoji"]),
       "card" => card,
-      "muted" => CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user)
+      "muted" => thread_muted? || User.mutes?(opts[:for], user)
     }
   end
 
diff --git a/test/activity_test.exs b/test/activity_test.exs
index 7e91d534b..15c95502a 100644
--- a/test/activity_test.exs
+++ b/test/activity_test.exs
@@ -6,6 +6,7 @@ defmodule Pleroma.ActivityTest do
   use Pleroma.DataCase
   alias Pleroma.Activity
   alias Pleroma.Bookmark
+  alias Pleroma.ThreadMute
   import Pleroma.Factory
 
   test "returns an activity by it's AP id" do
@@ -47,6 +48,31 @@ defmodule Pleroma.ActivityTest do
     assert queried_activity.bookmark == bookmark3
   end
 
+  test "setting thread_muted?" do
+    activity = insert(:note_activity)
+    user = insert(:user)
+    annoyed_user = insert(:user)
+    {:ok, _} = ThreadMute.add_mute(annoyed_user.id, activity.data["context"])
+
+    activity_with_unset_thread_muted_field =
+      Ecto.Query.from(Activity)
+      |> Repo.one()
+
+    activity_for_user =
+      Ecto.Query.from(Activity)
+      |> Activity.with_set_thread_muted_field(user)
+      |> Repo.one()
+
+    activity_for_annoyed_user =
+      Ecto.Query.from(Activity)
+      |> Activity.with_set_thread_muted_field(annoyed_user)
+      |> Repo.one()
+
+    assert activity_with_unset_thread_muted_field.thread_muted? == nil
+    assert activity_for_user.thread_muted? == false
+    assert activity_for_annoyed_user.thread_muted? == true
+  end
+
   describe "getting a bookmark" do
     test "when association is loaded" do
       user = insert(:user)
diff --git a/test/user_test.exs b/test/user_test.exs
index 10e463ff8..cb6afbe07 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -902,7 +902,7 @@ defmodule Pleroma.UserTest do
 
       assert [activity] == ActivityPub.fetch_public_activities(%{}) |> Repo.preload(:bookmark)
 
-      assert [activity] ==
+      assert [%{activity | thread_muted?: CommonAPI.thread_muted?(user2, activity)}] ==
                ActivityPub.fetch_activities([user2.ap_id | user2.following], %{"user" => user2})
 
       {:ok, _user} = User.deactivate(user)

From 5aa107d9126c484f0ae06eddec6995b535623b32 Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Tue, 21 May 2019 00:59:12 +0800
Subject: [PATCH 24/56] Document MRF.Simple :report_removal

---
 docs/config/mrf.md | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/config/mrf.md b/docs/config/mrf.md
index 2cc16cef0..45be18fc5 100644
--- a/docs/config/mrf.md
+++ b/docs/config/mrf.md
@@ -5,11 +5,12 @@ Possible uses include:
 
 * marking incoming messages with media from a given account or instance as sensitive
 * rejecting messages from a specific instance
+* rejecting reports (flags) from a specific instance
 * removing/unlisting messages from the public timelines
 * removing media from messages
 * sending only public messages to a specific instance
 
-The MRF provides user-configurable policies.  The default policy is `NoOpPolicy`, which disables the MRF functionality.  Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module.  
+The MRF provides user-configurable policies.  The default policy is `NoOpPolicy`, which disables the MRF functionality.  Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module.
 It is possible to use multiple, active MRF policies at the same time.
 
 ## Quarantine Instances
@@ -41,12 +42,13 @@ Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_si
 * `media_nsfw`: Servers in this group will have the #nsfw tag and sensitive setting injected into incoming messages which contain media.
 * `reject`: Servers in this group will have their messages rejected.
 * `federated_timeline_removal`: Servers in this group will have their messages unlisted from the public timelines by flipping the `to` and `cc` fields.
+* `report_removal`: Servers in this group will have their reports (flags) rejected.
 
 Servers should be configured as lists.
 
 ### Example
 
-This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com` and remove messages from `spam.university` from the federated timeline:
+This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com`, remove messages from `spam.university` from the federated timeline and block reports (flags) from `whiny.whiner`:
 
 ```
 config :pleroma, :instance,
@@ -56,7 +58,8 @@ config :pleroma, :mrf_simple,
   media_removal: ["illegalporn.biz"],
   media_nsfw: ["porn.biz", "porn.business"],
   reject: ["spam.com"],
-  federated_timeline_removal: ["spam.university"]
+  federated_timeline_removal: ["spam.university"],
+  report_removal: ["whiny.whiner"]
 
 ```
 

From 75c7bb9289066bfaebe7d96aaa474d34ec4d5a5f Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 20 May 2019 17:18:59 -0500
Subject: [PATCH 25/56] Additional reserved usernames

---
 config/config.exs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/config/config.exs b/config/config.exs
index 61e2648a9..387c1a5a7 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -369,6 +369,7 @@ config :pleroma, Pleroma.User,
     "activities",
     "api",
     "auth",
+    "check_password",
     "dev",
     "friend-requests",
     "inbox",
@@ -389,6 +390,7 @@ config :pleroma, Pleroma.User,
     "status",
     "tag",
     "user-search",
+    "user_exists",
     "users",
     "web"
   ]

From f96e9b28bb5ee241a3f0ca6a622b925ca560c141 Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Tue, 21 May 2019 07:30:18 +0800
Subject: [PATCH 26/56] Fix prometheus-ecto error when not configured

---
 lib/pleroma/application.ex | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index eeb415084..dab45a0b2 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -131,19 +131,22 @@ defmodule Pleroma.Application do
   defp setup_instrumenters do
     require Prometheus.Registry
 
-    :ok =
-      :telemetry.attach(
-        "prometheus-ecto",
-        [:pleroma, :repo, :query],
-        &Pleroma.Repo.Instrumenter.handle_event/4,
-        %{}
-      )
+    if Application.get_env(:prometheus, Pleroma.Repo.Instrumenter) do
+      :ok =
+        :telemetry.attach(
+          "prometheus-ecto",
+          [:pleroma, :repo, :query],
+          &Pleroma.Repo.Instrumenter.handle_event/4,
+          %{}
+        )
+
+      Pleroma.Repo.Instrumenter.setup()
+    end
 
     Prometheus.Registry.register_collector(:prometheus_process_collector)
     Pleroma.Web.Endpoint.MetricsExporter.setup()
     Pleroma.Web.Endpoint.PipelineInstrumenter.setup()
     Pleroma.Web.Endpoint.Instrumenter.setup()
-    Pleroma.Repo.Instrumenter.setup()
   end
 
   def enabled_hackney_pools do

From c972d0bb14ee5a65f053b8c9629d93fc9b94ca78 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Tue, 21 May 2019 04:58:26 +0000
Subject: [PATCH 27/56] http: bump connection timeout to 10 seconds

---
 CHANGELOG.md                   | 1 +
 lib/pleroma/http/connection.ex | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 07e6bce31..256df91b7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -73,6 +73,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Don't ship finmoji by default, they can be installed as an emoji pack
 - Hide deactivated users and their statuses
 - Posts which are marked sensitive or tagged nsfw no longer have link previews.
+- HTTP connection timeout is now set to 10 seconds.
 
 ### Fixed
 - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended.
diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex
index c0173465a..558005c19 100644
--- a/lib/pleroma/http/connection.ex
+++ b/lib/pleroma/http/connection.ex
@@ -8,7 +8,7 @@ defmodule Pleroma.HTTP.Connection do
   """
 
   @hackney_options [
-    connect_timeout: 2_000,
+    connect_timeout: 10_000,
     recv_timeout: 20_000,
     follow_redirect: true,
     pool: :federation

From d378b342ba0d7f54be3b47f031c602a578191dfd Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 21 May 2019 18:57:36 +0200
Subject: [PATCH 28/56] MongooseIM: Add documentation.

---
 CHANGELOG.md                    |  1 +
 docs/config/howto_mongooseim.md | 10 ++++++++++
 2 files changed, 11 insertions(+)
 create mode 100644 docs/config/howto_mongooseim.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 12c439135..b50ca895c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
 ### Added
+- [MongooseIM](https://github.com/esl/MongooseIM) http authentication support.
 - LDAP authentication
 - External OAuth provider authentication
 - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc.
diff --git a/docs/config/howto_mongooseim.md b/docs/config/howto_mongooseim.md
new file mode 100644
index 000000000..a33e590a1
--- /dev/null
+++ b/docs/config/howto_mongooseim.md
@@ -0,0 +1,10 @@
+# Configuring MongooseIM (XMPP Server) to use Pleroma for authentication
+
+If you want to give your Pleroma users an XMPP (chat) account, you can configure [MongooseIM](https://github.com/esl/MongooseIM) to use your Pleroma server for user authentication, automatically giving every local user an XMPP account.
+
+In general, you just have to follow the configuration described at [https://mongooseim.readthedocs.io/en/latest/authentication-backends/HTTP-authentication-module/](https://mongooseim.readthedocs.io/en/latest/authentication-backends/HTTP-authentication-module/) and do these changes to your mongooseim.cfg.
+
+1. Set the auth_method to `{auth_method, http}`.
+2. Add the http auth pool like this: `{http, global, auth, [{workers, 50}], [{server, "https://yourpleromainstance.com"}]}`
+
+Restart your MongooseIM server, your users should now be able to connect with their Pleroma credentials.

From c2b0b82e6a6d40f945feacc001ad984a17e23336 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Tue, 21 May 2019 00:41:40 +0000
Subject: [PATCH 29/56] object: add Object.prune()

---
 lib/pleroma/object.ex | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index 740d687a3..cc6fc9c5d 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -130,6 +130,13 @@ defmodule Pleroma.Object do
     end
   end
 
+  def prune(%Object{data: %{"id" => id}} = object) do
+    with {:ok, object} <- Repo.delete(object),
+         {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
+      {:ok, object}
+    end
+  end
+
   def set_cache(%Object{data: %{"id" => ap_id}} = object) do
     Cachex.put(:object_cache, "object:#{ap_id}", object)
     {:ok, object}

From 73df9d690d5c1a9c11f0f04b8d877c0677022591 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Tue, 21 May 2019 00:41:58 +0000
Subject: [PATCH 30/56] object: fetcher: add support for reinjecting pruned
 objects

---
 lib/pleroma/object/fetcher.ex | 22 ++++++++++++++++++++--
 test/object/fetcher_test.exs  | 19 +++++++++++++++++++
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 8d4bcc95e..bb9388d4f 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -8,6 +8,19 @@ defmodule Pleroma.Object.Fetcher do
 
   @httpoison Application.get_env(:pleroma, :httpoison)
 
+  defp reinject_object(data) do
+    Logger.debug("Reinjecting object #{data["id"]}")
+
+    with data <- Transmogrifier.fix_object(data),
+         {:ok, object} <- Object.create(data) do
+      {:ok, object}
+    else
+      e ->
+        Logger.error("Error while processing object: #{inspect(e)}")
+        {:error, e}
+    end
+  end
+
   # TODO:
   # This will create a Create activity, which we need internally at the moment.
   def fetch_object_from_id(id) do
@@ -26,12 +39,17 @@ defmodule Pleroma.Object.Fetcher do
              "object" => data
            },
            :ok <- Containment.contain_origin(id, params),
-           {:ok, activity} <- Transmogrifier.handle_incoming(params) do
-        {:ok, Object.normalize(activity, false)}
+           {:ok, activity} <- Transmogrifier.handle_incoming(params),
+           {:object, _data, %Object{} = object} <-
+             {:object, data, Object.normalize(activity, false)} do
+        {:ok, object}
       else
         {:error, {:reject, nil}} ->
           {:reject, nil}
 
+        {:object, data, nil} ->
+          reinject_object(data)
+
         object = %Object{} ->
           {:ok, object}
 
diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs
index 72f616782..d604fd5f5 100644
--- a/test/object/fetcher_test.exs
+++ b/test/object/fetcher_test.exs
@@ -87,4 +87,23 @@ defmodule Pleroma.Object.FetcherTest do
         )
     end
   end
+
+  describe "pruning" do
+    test "it can refetch pruned objects" do
+      object_id = "http://mastodon.example.org/@admin/99541947525187367"
+
+      {:ok, object} = Fetcher.fetch_object_from_id(object_id)
+
+      assert object
+
+      {:ok, _object} = Object.prune(object)
+
+      refute Object.get_by_ap_id(object_id)
+
+      {:ok, %Object{} = object_two} = Fetcher.fetch_object_from_id(object_id)
+
+      assert object.data["id"] == object_two.data["id"]
+      assert object.id != object_two.id
+    end
+  end
 end

From 16b260fb19cca02463766c2e36a41bfcc823af9b Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Tue, 21 May 2019 01:21:28 +0000
Subject: [PATCH 31/56] add mix task to prune the object database using a
 configured retention period

---
 config/config.exs                 |  3 ++-
 docs/config.md                    |  1 +
 lib/mix/tasks/pleroma/database.ex | 40 +++++++++++++++++++++++++++++++
 3 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 72908266d..466a6e9b7 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -239,7 +239,8 @@ config :pleroma, :instance,
   welcome_message: nil,
   max_report_comment_size: 1000,
   safe_dm_mentions: false,
-  healthcheck: false
+  healthcheck: false,
+  remote_post_retention_days: 90
 
 config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800
 
diff --git a/docs/config.md b/docs/config.md
index 197326bbd..a050068f4 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -104,6 +104,7 @@ config :pleroma, Pleroma.Emails.Mailer,
 * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`)
 * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`)
 * `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``.
+* `remote_post_retention_days`: the default amount of days to retain remote posts when pruning the database
 
 ## :app_account_creation
 REST API for creating an account settings
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index f650b447d..fdb216037 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -23,6 +23,10 @@ defmodule Mix.Tasks.Pleroma.Database do
     Options:
     - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
 
+  ## Prune old objects from the database
+
+      mix pleroma.database prune_objects
+
   ## Create a conversation for all existing DMs. Can be safely re-run.
 
       mix pleroma.database bump_all_conversations
@@ -72,4 +76,40 @@ defmodule Mix.Tasks.Pleroma.Database do
     Enum.each(users, &User.remove_duplicated_following/1)
     Enum.each(users, &User.update_follower_count/1)
   end
+
+  def run(["prune_objects" | args]) do
+    {options, [], []} =
+      OptionParser.parse(
+        args,
+        strict: [
+          vacuum: :boolean
+        ]
+      )
+
+    Common.start_pleroma()
+
+    deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
+
+    Logger.info("Pruning objects older than #{deadline} days")
+
+    time_deadline =
+      NaiveDateTime.utc_now()
+      |> NaiveDateTime.add(-(deadline * 86_400))
+
+    Repo.query!(
+      "DELETE FROM objects WHERE inserted_at < $1 AND split_part(data->>'actor', '/', 3) != $2",
+      [time_deadline, Pleroma.Web.Endpoint.host()],
+      timeout: :infinity
+    )
+
+    if Keyword.get(options, :vacuum) do
+      Logger.info("Runnning VACUUM FULL")
+
+      Repo.query!(
+        "vacuum full;",
+        [],
+        timeout: :infinity
+      )
+    end
+  end
 end

From f446f94290a6cdae531859c2efa55f55dbaeb7e8 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Tue, 21 May 2019 01:22:27 +0000
Subject: [PATCH 32/56] add changelog entry for object pruning

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 256df91b7..a21c4bff2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Metadata: RelMe provider
 - OAuth: added support for refresh tokens
 - Emoji packs and emoji pack manager
+- Object pruning (`mix pleroma.database prune_objects`)
 
 ### Changed
 - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer

From a13d449b241b6e5fa14fb741694e63cd21569b2c Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Tue, 21 May 2019 09:39:32 +0800
Subject: [PATCH 33/56] Add tests for fallback routes

---
 test/web/fallback_test.exs | 46 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 test/web/fallback_test.exs

diff --git a/test/web/fallback_test.exs b/test/web/fallback_test.exs
new file mode 100644
index 000000000..514923a20
--- /dev/null
+++ b/test/web/fallback_test.exs
@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.FallbackTest do
+  use Pleroma.Web.ConnCase
+  import Pleroma.Factory
+
+  test "GET /registration/:token", %{conn: conn} do
+    assert conn
+           |> get("/registration/foo")
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+  end
+
+  test "GET /:maybe_nickname_or_id", %{conn: conn} do
+    user = insert(:user)
+
+    assert conn
+           |> get("/foo")
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+
+    refute conn
+           |> get("/" <> user.nickname)
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+  end
+
+  test "GET /*path", %{conn: conn} do
+    assert conn
+           |> get("/foo")
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+
+    assert conn
+           |> get("/foo/bar")
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+  end
+
+  test "OPTIONS /*path", %{conn: conn} do
+    assert conn
+           |> options("/foo")
+           |> response(204) == ""
+
+    assert conn
+           |> options("/foo/bar")
+           |> response(204) == ""
+  end
+end

From 3ab9255eda21a5f8a25047375af9608e0c0c7592 Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Tue, 21 May 2019 09:40:29 +0800
Subject: [PATCH 34/56] Respond with a 404 Not implemented JSON error message

when requested API is not implemented
---
 CHANGELOG.md               | 1 +
 lib/pleroma/web/router.ex  | 7 +++++++
 test/web/fallback_test.exs | 6 ++++++
 3 files changed, 14 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 256df91b7..2ed380102 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -74,6 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Hide deactivated users and their statuses
 - Posts which are marked sensitive or tagged nsfw no longer have link previews.
 - HTTP connection timeout is now set to 10 seconds.
+- Respond with a 404 Not implemented JSON error message when requested API is not implemented
 
 ### Fixed
 - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended.
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 4c29b24eb..49e28cc2d 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -710,6 +710,7 @@ defmodule Pleroma.Web.Router do
   scope "/", Fallback do
     get("/registration/:token", RedirectController, :registration_page)
     get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta)
+    get("/api*path", RedirectController, :api_not_implemented)
     get("/*path", RedirectController, :redirector)
 
     options("/*path", RedirectController, :empty)
@@ -721,6 +722,12 @@ defmodule Fallback.RedirectController do
   alias Pleroma.User
   alias Pleroma.Web.Metadata
 
+  def api_not_implemented(conn, _params) do
+    conn
+    |> put_status(404)
+    |> json(%{error: "Not implemented"})
+  end
+
   def redirector(conn, _params, code \\ 200) do
     conn
     |> put_resp_content_type("text/html")
diff --git a/test/web/fallback_test.exs b/test/web/fallback_test.exs
index 514923a20..cc78b3ae1 100644
--- a/test/web/fallback_test.exs
+++ b/test/web/fallback_test.exs
@@ -24,6 +24,12 @@ defmodule Pleroma.Web.FallbackTest do
            |> html_response(200) =~ "<!--server-generated-meta-->"
   end
 
+  test "GET /api*path", %{conn: conn} do
+    assert conn
+           |> get("/api/foo")
+           |> json_response(404) == %{"error" => "Not implemented"}
+  end
+
   test "GET /*path", %{conn: conn} do
     assert conn
            |> get("/foo")

From f76268135c014c20a482d30a7c9596ec2e7d6a69 Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Wed, 22 May 2019 07:11:09 +0800
Subject: [PATCH 35/56] Fix failing test

---
 test/web/admin_api/admin_api_controller_test.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs
index ca12c7215..c15c67e31 100644
--- a/test/web/admin_api/admin_api_controller_test.exs
+++ b/test/web/admin_api/admin_api_controller_test.exs
@@ -397,14 +397,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
     end
   end
 
-  test "/api/pleroma/admin/invite_token" do
+  test "/api/pleroma/admin/users/invite_token" do
     admin = insert(:user, info: %{is_admin: true})
 
     conn =
       build_conn()
       |> assign(:user, admin)
       |> put_req_header("accept", "application/json")
-      |> get("/api/pleroma/admin/invite_token")
+      |> get("/api/pleroma/admin/users/invite_token")
 
     assert conn.status == 200
   end

From a023ca004cbd90e330cab35e4dfda16346d08668 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 03:12:48 +0000
Subject: [PATCH 36/56] prune objects task: use Repo.delete_all()

---
 lib/mix/tasks/pleroma/database.ex | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index fdb216037..f9bafb277 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -5,6 +5,7 @@
 defmodule Mix.Tasks.Pleroma.Database do
   alias Mix.Tasks.Pleroma.Common
   alias Pleroma.Conversation
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
   require Logger
@@ -78,6 +79,8 @@ defmodule Mix.Tasks.Pleroma.Database do
   end
 
   def run(["prune_objects" | args]) do
+    import Ecto.Query
+
     {options, [], []} =
       OptionParser.parse(
         args,
@@ -96,11 +99,15 @@ defmodule Mix.Tasks.Pleroma.Database do
       NaiveDateTime.utc_now()
       |> NaiveDateTime.add(-(deadline * 86_400))
 
-    Repo.query!(
-      "DELETE FROM objects WHERE inserted_at < $1 AND split_part(data->>'actor', '/', 3) != $2",
-      [time_deadline, Pleroma.Web.Endpoint.host()],
-      timeout: :infinity
+    public = "https://www.w3.org/ns/activitystreams#Public"
+
+    from(o in Object,
+      where: fragment("?->'to' \\? ? OR ?->'cc' \\? ?", o.data, ^public, o.data, ^public),
+      where: o.inserted_at < ^time_deadline,
+      where:
+        fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
     )
+    |> Repo.delete_all()
 
     if Keyword.get(options, :vacuum) do
       Logger.info("Runnning VACUUM FULL")

From 045803346d70c1f9c6ea770485904fd7cc52969a Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 03:58:15 +0000
Subject: [PATCH 37/56] move key generation functions into Pleroma.Keys module

---
 lib/pleroma/keys.ex                           | 44 +++++++++++++++++++
 lib/pleroma/signature.ex                      |  7 ++-
 lib/pleroma/user.ex                           | 21 +++++++++
 .../activity_pub/activity_pub_controller.ex   | 14 +++---
 .../web/activity_pub/views/user_view.ex       | 11 +++--
 lib/pleroma/web/federator/federator.ex        |  6 +--
 lib/pleroma/web/salmon/salmon.ex              | 44 ++-----------------
 lib/pleroma/web/web_finger/web_finger.ex      | 26 +----------
 test/keys_test.exs                            | 20 +++++++++
 test/user_test.exs                            | 15 +++++++
 test/web/activity_pub/activity_pub_test.exs   |  2 +-
 .../web/activity_pub/views/user_view_test.exs | 13 +++---
 test/web/salmon/salmon_test.exs               | 19 ++------
 test/web/web_finger/web_finger_test.exs       | 15 -------
 14 files changed, 133 insertions(+), 124 deletions(-)
 create mode 100644 lib/pleroma/keys.ex
 create mode 100644 test/keys_test.exs

diff --git a/lib/pleroma/keys.ex b/lib/pleroma/keys.ex
new file mode 100644
index 000000000..b7bc7a4da
--- /dev/null
+++ b/lib/pleroma/keys.ex
@@ -0,0 +1,44 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Keys do
+  # Native generation of RSA keys is only available since OTP 20+ and in default build conditions
+  # We try at compile time to generate natively an RSA key otherwise we fallback on the old way.
+  try do
+    _ = :public_key.generate_key({:rsa, 2048, 65_537})
+
+    def generate_rsa_pem do
+      key = :public_key.generate_key({:rsa, 2048, 65_537})
+      entry = :public_key.pem_entry_encode(:RSAPrivateKey, key)
+      pem = :public_key.pem_encode([entry]) |> String.trim_trailing()
+      {:ok, pem}
+    end
+  rescue
+    _ ->
+      def generate_rsa_pem do
+        port = Port.open({:spawn, "openssl genrsa"}, [:binary])
+
+        {:ok, pem} =
+          receive do
+            {^port, {:data, pem}} -> {:ok, pem}
+          end
+
+        Port.close(port)
+
+        if Regex.match?(~r/RSA PRIVATE KEY/, pem) do
+          {:ok, pem}
+        else
+          :error
+        end
+      end
+  end
+
+  def keys_from_pem(pem) do
+    [private_key_code] = :public_key.pem_decode(pem)
+    private_key = :public_key.pem_entry_decode(private_key_code)
+    {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key
+    public_key = {:RSAPublicKey, modulus, exponent}
+    {:ok, private_key, public_key}
+  end
+end
diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex
index b7ecf00a0..1a4d54c62 100644
--- a/lib/pleroma/signature.ex
+++ b/lib/pleroma/signature.ex
@@ -5,11 +5,10 @@
 defmodule Pleroma.Signature do
   @behaviour HTTPSignatures.Adapter
 
+  alias Pleroma.Keys
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Web.Salmon
-  alias Pleroma.Web.WebFinger
 
   def fetch_public_key(conn) do
     with actor_id <- Utils.get_ap_id(conn.params["actor"]),
@@ -33,8 +32,8 @@ defmodule Pleroma.Signature do
   end
 
   def sign(%User{} = user, headers) do
-    with {:ok, %{info: %{keys: keys}}} <- WebFinger.ensure_keys_present(user),
-         {:ok, private_key, _} <- Salmon.keys_from_pem(keys) do
+    with {:ok, %{info: %{keys: keys}}} <- User.ensure_keys_present(user),
+         {:ok, private_key, _} <- Keys.keys_from_pem(keys) do
       HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers)
     end
   end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 05fe58f7c..653dec95f 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.User do
 
   alias Comeonin.Pbkdf2
   alias Pleroma.Activity
+  alias Pleroma.Keys
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Registration
@@ -1422,4 +1423,24 @@ defmodule Pleroma.User do
       }
     }
   end
+
+  def ensure_keys_present(user) do
+    info = user.info
+
+    if info.keys do
+      {:ok, user}
+    else
+      {:ok, pem} = Keys.generate_rsa_pem()
+
+      info_cng =
+        info
+        |> User.Info.set_keys(pem)
+
+      cng =
+        Ecto.Changeset.change(user)
+        |> Ecto.Changeset.put_embed(:info, info_cng)
+
+      update_and_set_cache(cng)
+    end
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index c967ab7a9..ad2ca1e54 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -39,7 +39,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   def user(conn, %{"nickname" => nickname}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_header("content-type", "application/activity+json")
       |> json(UserView.render("user.json", %{user: user}))
@@ -106,7 +106,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   def following(conn, %{"nickname" => nickname, "page" => page}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
       {page, _} = Integer.parse(page)
 
       conn
@@ -117,7 +117,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   def following(conn, %{"nickname" => nickname}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_header("content-type", "application/activity+json")
       |> json(UserView.render("following.json", %{user: user}))
@@ -126,7 +126,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   def followers(conn, %{"nickname" => nickname, "page" => page}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
       {page, _} = Integer.parse(page)
 
       conn
@@ -137,7 +137,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   def followers(conn, %{"nickname" => nickname}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_header("content-type", "application/activity+json")
       |> json(UserView.render("followers.json", %{user: user}))
@@ -146,7 +146,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   def outbox(conn, %{"nickname" => nickname} = params) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_header("content-type", "application/activity+json")
       |> json(UserView.render("outbox.json", %{user: user, max_id: params["max_id"]}))
@@ -195,7 +195,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   def relay(conn, _params) do
     with %User{} = user <- Relay.get_actor(),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_header("content-type", "application/activity+json")
       |> json(UserView.render("user.json", %{user: user}))
diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex
index 1254fdf6c..327e0e05b 100644
--- a/lib/pleroma/web/activity_pub/views/user_view.ex
+++ b/lib/pleroma/web/activity_pub/views/user_view.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.UserView do
   use Pleroma.Web, :view
 
+  alias Pleroma.Keys
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -12,8 +13,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.Router.Helpers
-  alias Pleroma.Web.Salmon
-  alias Pleroma.Web.WebFinger
 
   import Ecto.Query
 
@@ -34,8 +33,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do
 
   # the instance itself is not a Person, but instead an Application
   def render("user.json", %{user: %{nickname: nil} = user}) do
-    {:ok, user} = WebFinger.ensure_keys_present(user)
-    {:ok, _, public_key} = Salmon.keys_from_pem(user.info.keys)
+    {:ok, user} = User.ensure_keys_present(user)
+    {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys)
     public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
     public_key = :public_key.pem_encode([public_key])
 
@@ -62,8 +61,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do
   end
 
   def render("user.json", %{user: user}) do
-    {:ok, user} = WebFinger.ensure_keys_present(user)
-    {:ok, _, public_key} = Salmon.keys_from_pem(user.info.keys)
+    {:ok, user} = User.ensure_keys_present(user)
+    {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys)
     public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
     public_key = :public_key.pem_encode([public_key])
 
diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index 169fdf4dc..6b0b75284 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -11,7 +11,6 @@ defmodule Pleroma.Web.Federator do
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Federator.Publisher
   alias Pleroma.Web.Federator.RetryQueue
-  alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Websub
 
   require Logger
@@ -77,9 +76,8 @@ defmodule Pleroma.Web.Federator do
   def perform(:publish, activity) do
     Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end)
 
-    with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do
-      {:ok, actor} = WebFinger.ensure_keys_present(actor)
-
+    with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]),
+         {:ok, actor} <- User.ensure_keys_present(actor) do
       Publisher.publish(actor, activity)
     end
   end
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index 42709ab47..fa30f73cd 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.Salmon do
   use Bitwise
 
   alias Pleroma.Activity
+  alias Pleroma.Keys
   alias Pleroma.Instances
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Visibility
@@ -89,45 +90,6 @@ defmodule Pleroma.Web.Salmon do
     "RSA.#{modulus_enc}.#{exponent_enc}"
   end
 
-  # Native generation of RSA keys is only available since OTP 20+ and in default build conditions
-  # We try at compile time to generate natively an RSA key otherwise we fallback on the old way.
-  try do
-    _ = :public_key.generate_key({:rsa, 2048, 65_537})
-
-    def generate_rsa_pem do
-      key = :public_key.generate_key({:rsa, 2048, 65_537})
-      entry = :public_key.pem_entry_encode(:RSAPrivateKey, key)
-      pem = :public_key.pem_encode([entry]) |> String.trim_trailing()
-      {:ok, pem}
-    end
-  rescue
-    _ ->
-      def generate_rsa_pem do
-        port = Port.open({:spawn, "openssl genrsa"}, [:binary])
-
-        {:ok, pem} =
-          receive do
-            {^port, {:data, pem}} -> {:ok, pem}
-          end
-
-        Port.close(port)
-
-        if Regex.match?(~r/RSA PRIVATE KEY/, pem) do
-          {:ok, pem}
-        else
-          :error
-        end
-      end
-  end
-
-  def keys_from_pem(pem) do
-    [private_key_code] = :public_key.pem_decode(pem)
-    private_key = :public_key.pem_entry_decode(private_key_code)
-    {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key
-    public_key = {:RSAPublicKey, modulus, exponent}
-    {:ok, private_key, public_key}
-  end
-
   def encode(private_key, doc) do
     type = "application/atom+xml"
     encoding = "base64url"
@@ -227,7 +189,7 @@ defmodule Pleroma.Web.Salmon do
         |> :xmerl.export_simple(:xmerl_xml)
         |> to_string
 
-      {:ok, private, _} = keys_from_pem(keys)
+      {:ok, private, _} = Keys.keys_from_pem(keys)
       {:ok, feed} = encode(private, feed)
 
       remote_users = remote_users(activity)
@@ -253,7 +215,7 @@ defmodule Pleroma.Web.Salmon do
   def publish(%{id: id}, _), do: Logger.debug(fn -> "Keys missing for user #{id}" end)
 
   def gather_webfinger_links(%User{} = user) do
-    {:ok, _private, public} = keys_from_pem(user.info.keys)
+    {:ok, _private, public} = Keys.keys_from_pem(user.info.keys)
     magic_key = encode_key(public)
 
     [
diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex
index 1239b962a..c5b7d4acb 100644
--- a/lib/pleroma/web/web_finger/web_finger.ex
+++ b/lib/pleroma/web/web_finger/web_finger.ex
@@ -8,7 +8,6 @@ defmodule Pleroma.Web.WebFinger do
   alias Pleroma.User
   alias Pleroma.Web
   alias Pleroma.Web.Federator.Publisher
-  alias Pleroma.Web.Salmon
   alias Pleroma.Web.XML
   alias Pleroma.XmlBuilder
   require Jason
@@ -61,7 +60,7 @@ defmodule Pleroma.Web.WebFinger do
   end
 
   def represent_user(user, "JSON") do
-    {:ok, user} = ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)
 
     %{
       "subject" => "acct:#{user.nickname}@#{Pleroma.Web.Endpoint.host()}",
@@ -71,7 +70,7 @@ defmodule Pleroma.Web.WebFinger do
   end
 
   def represent_user(user, "XML") do
-    {:ok, user} = ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)
 
     links =
       gather_links(user)
@@ -88,27 +87,6 @@ defmodule Pleroma.Web.WebFinger do
     |> XmlBuilder.to_doc()
   end
 
-  # This seems a better fit in Salmon
-  def ensure_keys_present(user) do
-    info = user.info
-
-    if info.keys do
-      {:ok, user}
-    else
-      {:ok, pem} = Salmon.generate_rsa_pem()
-
-      info_cng =
-        info
-        |> User.Info.set_keys(pem)
-
-      cng =
-        Ecto.Changeset.change(user)
-        |> Ecto.Changeset.put_embed(:info, info_cng)
-
-      User.update_and_set_cache(cng)
-    end
-  end
-
   defp get_magic_key(magic_key) do
     "data:application/magic-public-key," <> magic_key = magic_key
     {:ok, magic_key}
diff --git a/test/keys_test.exs b/test/keys_test.exs
new file mode 100644
index 000000000..776fdea6f
--- /dev/null
+++ b/test/keys_test.exs
@@ -0,0 +1,20 @@
+defmodule Pleroma.KeysTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Keys
+
+  test "generates an RSA private key pem" do
+    {:ok, key} = Keys.generate_rsa_pem()
+
+    assert is_binary(key)
+    assert Regex.match?(~r/RSA/, key)
+  end
+
+  test "returns a public and private key from a pem" do
+    pem = File.read!("test/fixtures/private_key.pem")
+    {:ok, private, public} = Keys.keys_from_pem(pem)
+
+    assert elem(private, 0) == :RSAPrivateKey
+    assert elem(public, 0) == :RSAPublicKey
+  end
+end
diff --git a/test/user_test.exs b/test/user_test.exs
index cb6afbe07..019f2b56d 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -1251,4 +1251,19 @@ defmodule Pleroma.UserTest do
       refute user.info.confirmation_token
     end
   end
+
+  describe "ensure_keys_present" do
+    test "it creates keys for a user and stores them in info" do
+      user = insert(:user)
+      refute is_binary(user.info.keys)
+      {:ok, user} = User.ensure_keys_present(user)
+      assert is_binary(user.info.keys)
+    end
+
+    test "it doesn't create keys if there already are some" do
+      user = insert(:user, %{info: %{keys: "xxx"}})
+      {:ok, user} = User.ensure_keys_present(user)
+      assert user.info.keys == "xxx"
+    end
+  end
 end
diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index c18e0ab5f..f743f380b 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -1005,7 +1005,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
   describe "update" do
     test "it creates an update activity with the new user data" do
       user = insert(:user)
-      {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+      {:ok, user} = User.ensure_keys_present(user)
       user_data = Pleroma.Web.ActivityPub.UserView.render("user.json", %{user: user})
 
       {:ok, update} =
diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs
index 9fb9455d2..e6483db8b 100644
--- a/test/web/activity_pub/views/user_view_test.exs
+++ b/test/web/activity_pub/views/user_view_test.exs
@@ -2,11 +2,12 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do
   use Pleroma.DataCase
   import Pleroma.Factory
 
+  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.UserView
 
   test "Renders a user, including the public key" do
     user = insert(:user)
-    {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)
 
     result = UserView.render("user.json", %{user: user})
 
@@ -18,7 +19,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do
 
   test "Does not add an avatar image if the user hasn't set one" do
     user = insert(:user)
-    {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)
 
     result = UserView.render("user.json", %{user: user})
     refute result["icon"]
@@ -32,7 +33,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do
         }
       )
 
-    {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)
 
     result = UserView.render("user.json", %{user: user})
     assert result["icon"]["url"] == "https://someurl"
@@ -42,7 +43,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do
   describe "endpoints" do
     test "local users have a usable endpoints structure" do
       user = insert(:user)
-      {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+      {:ok, user} = User.ensure_keys_present(user)
 
       result = UserView.render("user.json", %{user: user})
 
@@ -58,7 +59,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do
 
     test "remote users have an empty endpoints structure" do
       user = insert(:user, local: false)
-      {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+      {:ok, user} = User.ensure_keys_present(user)
 
       result = UserView.render("user.json", %{user: user})
 
@@ -68,7 +69,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do
 
     test "instance users do not expose oAuth endpoints" do
       user = insert(:user, nickname: nil, local: true)
-      {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+      {:ok, user} = User.ensure_keys_present(user)
 
       result = UserView.render("user.json", %{user: user})
 
diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs
index 232082779..e86e76fe9 100644
--- a/test/web/salmon/salmon_test.exs
+++ b/test/web/salmon/salmon_test.exs
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.Salmon.SalmonTest do
   use Pleroma.DataCase
   alias Pleroma.Activity
+  alias Pleroma.Keys
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.Federator.Publisher
@@ -34,12 +35,6 @@ defmodule Pleroma.Web.Salmon.SalmonTest do
     assert Salmon.decode_and_validate(@wrong_magickey, salmon) == :error
   end
 
-  test "generates an RSA private key pem" do
-    {:ok, key} = Salmon.generate_rsa_pem()
-    assert is_binary(key)
-    assert Regex.match?(~r/RSA/, key)
-  end
-
   test "it encodes a magic key from a public key" do
     key = Salmon.decode_key(@magickey)
     magic_key = Salmon.encode_key(key)
@@ -51,18 +46,10 @@ defmodule Pleroma.Web.Salmon.SalmonTest do
     _key = Salmon.decode_key(@magickey_friendica)
   end
 
-  test "returns a public and private key from a pem" do
-    pem = File.read!("test/fixtures/private_key.pem")
-    {:ok, private, public} = Salmon.keys_from_pem(pem)
-
-    assert elem(private, 0) == :RSAPrivateKey
-    assert elem(public, 0) == :RSAPublicKey
-  end
-
   test "encodes an xml payload with a private key" do
     doc = File.read!("test/fixtures/incoming_note_activity.xml")
     pem = File.read!("test/fixtures/private_key.pem")
-    {:ok, private, public} = Salmon.keys_from_pem(pem)
+    {:ok, private, public} = Keys.keys_from_pem(pem)
 
     # Let's try a roundtrip.
     {:ok, salmon} = Salmon.encode(private, doc)
@@ -105,7 +92,7 @@ defmodule Pleroma.Web.Salmon.SalmonTest do
 
     {:ok, activity} = Repo.insert(%Activity{data: activity_data, recipients: activity_data["to"]})
     user = User.get_cached_by_ap_id(activity.data["actor"])
-    {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)
 
     Salmon.publish(user, activity)
 
diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs
index 6b20d8d56..335c95b18 100644
--- a/test/web/web_finger/web_finger_test.exs
+++ b/test/web/web_finger/web_finger_test.exs
@@ -105,19 +105,4 @@ defmodule Pleroma.Web.WebFingerTest do
       assert template == "http://status.alpicola.com/main/xrd?uri={uri}"
     end
   end
-
-  describe "ensure_keys_present" do
-    test "it creates keys for a user and stores them in info" do
-      user = insert(:user)
-      refute is_binary(user.info.keys)
-      {:ok, user} = WebFinger.ensure_keys_present(user)
-      assert is_binary(user.info.keys)
-    end
-
-    test "it doesn't create keys if there already are some" do
-      user = insert(:user, %{info: %{keys: "xxx"}})
-      {:ok, user} = WebFinger.ensure_keys_present(user)
-      assert user.info.keys == "xxx"
-    end
-  end
 end

From 913484817076bf5ca4bdbe7c3c1ff34f7debd3e5 Mon Sep 17 00:00:00 2001
From: Sergey Suprunenko <suprunenko.s@gmail.com>
Date: Wed, 22 May 2019 04:04:20 +0000
Subject: [PATCH 38/56] Do not truncate DM when it contains newlines and
 safe_dm_mentions is set to true

---
 lib/pleroma/formatter.ex | 2 +-
 test/formatter_test.exs  | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index 3d7c36d21..3e3b9fe97 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -8,7 +8,7 @@ defmodule Pleroma.Formatter do
   alias Pleroma.User
   alias Pleroma.Web.MediaProxy
 
-  @safe_mention_regex ~r/^(\s*(?<mentions>@.+?\s+)+)(?<rest>.*)/
+  @safe_mention_regex ~r/^(\s*(?<mentions>@.+?\s+)+)(?<rest>.*)/s
   @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
 
diff --git a/test/formatter_test.exs b/test/formatter_test.exs
index 5e7011160..47b91b121 100644
--- a/test/formatter_test.exs
+++ b/test/formatter_test.exs
@@ -206,6 +206,15 @@ defmodule Pleroma.FormatterTest do
       assert mentions == []
       assert expected_text == text
     end
+
+    test "given the 'safe_mention' option, it will keep text after newlines" do
+      user = insert(:user)
+      text = " @#{user.nickname}\n hey dude\n\nhow are you doing?"
+
+      {expected_text, _, _} = Formatter.linkify(text, safe_mention: true)
+
+      assert expected_text =~ "how are you doing?"
+    end
   end
 
   describe ".parse_tags" do

From 0e2c215a006c7ca5756e80a357ff6395a4325946 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Wed, 22 May 2019 07:22:19 +0200
Subject: [PATCH 39/56] MastoAPI AccountView: fill source.note with plaintext
 version of note

Closes: https://git.pleroma.social/pleroma/pleroma/issues/926
---
 lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +-
 test/web/mastodon_api/account_view_test.exs        | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 134c07b7e..b82d3319b 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -112,7 +112,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       fields: fields,
       bot: bot,
       source: %{
-        note: "",
+        note: HTML.strip_tags((user.bio || "") |> String.replace("<br>", "\n")),
         sensitive: false,
         pleroma: %{}
       },
diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs
index a24f2a050..aaf2261bb 100644
--- a/test/web/mastodon_api/account_view_test.exs
+++ b/test/web/mastodon_api/account_view_test.exs
@@ -55,7 +55,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
       fields: [],
       bot: false,
       source: %{
-        note: "",
+        note: "valid html",
         sensitive: false,
         pleroma: %{}
       },
@@ -120,7 +120,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
       fields: [],
       bot: true,
       source: %{
-        note: "",
+        note: user.bio,
         sensitive: false,
         pleroma: %{}
       },
@@ -209,7 +209,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
       fields: [],
       bot: true,
       source: %{
-        note: "",
+        note: user.bio,
         sensitive: false,
         pleroma: %{}
       },

From 1344c85e2fe636fd6b9d033eb7add1c3a9701c7f Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 05:58:51 +0000
Subject: [PATCH 40/56] salmon: fix credo

---
 lib/pleroma/web/salmon/salmon.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index fa30f73cd..f25d92fad 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -10,8 +10,8 @@ defmodule Pleroma.Web.Salmon do
   use Bitwise
 
   alias Pleroma.Activity
-  alias Pleroma.Keys
   alias Pleroma.Instances
+  alias Pleroma.Keys
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.Federator.Publisher

From f9e0d09ec0082a096dcd4980bc5ffebe8e3139ae Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 May 2019 10:17:32 +0200
Subject: [PATCH 41/56] Changelog: Add SSH mode.

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3ff70e6e5..3d1e7640d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
 ### Added
+- Optional SSH access mode.
 - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support.
 - LDAP authentication
 - External OAuth provider authentication

From b6cf62ddeab04db6bd2695c5537c81e0fb1aecaf Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 May 2019 10:28:50 +0200
Subject: [PATCH 42/56] Mix: Don't start esshd application if we don't need it.

---
 mix.exs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/mix.exs b/mix.exs
index 95c052c34..bc5b6204f 100644
--- a/mix.exs
+++ b/mix.exs
@@ -40,9 +40,16 @@ defmodule Pleroma.Mixfile do
   #
   # Type `mix help compile.app` for more information.
   def application do
+    extra_applications = [:logger, :runtime_tools, :comeonin, :quack]
+    extra_applications = if Application.get_env(:esshd, :enabled, false) do
+      [:esshd | extra_applications]
+    else
+      extra_applications
+    end
+
     [
       mod: {Pleroma.Application, []},
-      extra_applications: [:logger, :runtime_tools, :comeonin, :esshd, :quack],
+      extra_applications: extra_applications,
       included_applications: [:ex_syslogger]
     ]
   end

From db9a82d168cfc452611a44d92df2b81a5e6d1e69 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 May 2019 10:40:15 +0200
Subject: [PATCH 43/56] Linting.

---
 mix.exs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/mix.exs b/mix.exs
index bc5b6204f..25ec46a46 100644
--- a/mix.exs
+++ b/mix.exs
@@ -41,11 +41,13 @@ defmodule Pleroma.Mixfile do
   # Type `mix help compile.app` for more information.
   def application do
     extra_applications = [:logger, :runtime_tools, :comeonin, :quack]
-    extra_applications = if Application.get_env(:esshd, :enabled, false) do
-      [:esshd | extra_applications]
-    else
-      extra_applications
-    end
+
+    extra_applications =
+      if Application.get_env(:esshd, :enabled, false) do
+        [:esshd | extra_applications]
+      else
+        extra_applications
+      end
 
     [
       mod: {Pleroma.Application, []},

From b22145cbc40b57cf83f6389f063a76a03625ff16 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 May 2019 10:44:26 +0200
Subject: [PATCH 44/56] Documentation: Specify PEM format for SSH keys.

Otherwise openssh-client 7.9 will generate a different format that can't be used
by esshd.
---
 docs/config.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/config.md b/docs/config.md
index 197326bbd..63ca61d1e 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -477,7 +477,7 @@ config :esshd,
   password_authenticator: "Pleroma.BBS.Authenticator"
 ```
 
-Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT`
+Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -m PEM -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT`
 
 ## :auth
 

From 3b12e1ba7c99382c678ce17629352135f44dcb9f Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 May 2019 11:01:10 +0200
Subject: [PATCH 45/56] Changelog: Add tip for debian users.

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3d1e7640d..b88edd072 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
 ### Added
-- Optional SSH access mode.
+- Optional SSH access mode. (Needs `erlang-ssh` package on Debian).
 - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support.
 - LDAP authentication
 - External OAuth provider authentication

From f4cfcead8868481c19ebd93b0fcd2b942dc0e477 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 May 2019 11:44:17 +0200
Subject: [PATCH 46/56] Mix: Bring ecto-sql back to mainline.

---
 mix.exs  | 5 +----
 mix.lock | 2 +-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/mix.exs b/mix.exs
index 95c052c34..9149b241f 100644
--- a/mix.exs
+++ b/mix.exs
@@ -66,10 +66,7 @@ defmodule Pleroma.Mixfile do
       {:plug_cowboy, "~> 2.0"},
       {:phoenix_pubsub, "~> 1.1"},
       {:phoenix_ecto, "~> 4.0"},
-      {:ecto_sql,
-       git: "https://github.com/elixir-ecto/ecto_sql",
-       ref: "14cb065a74c488d737d973f7a91bc036c6245f78",
-       override: true},
+      {:ecto_sql, "~> 3.1"},
       {:postgrex, ">= 0.13.5"},
       {:gettext, "~> 0.15"},
       {:comeonin, "~> 4.1.1"},
diff --git a/mix.lock b/mix.lock
index bacc09787..857bfca79 100644
--- a/mix.lock
+++ b/mix.lock
@@ -21,7 +21,7 @@
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
   "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
   "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
-  "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "14cb065a74c488d737d973f7a91bc036c6245f78", [ref: "14cb065a74c488d737d973f7a91bc036c6245f78"]},
+  "ecto_sql": {:hex, :ecto_sql, "3.1.3", "2c536139190492d9de33c5fefac7323c5eaaa82e1b9bf93482a14649042f7cd9", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
   "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"},
   "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"},
   "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"},

From f323031927ecaf155e661b17cc9b96333fb9e4ad Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 May 2019 12:57:20 +0200
Subject: [PATCH 47/56] Mix: Only start sshd when needed, second try.

---
 mix.exs | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/mix.exs b/mix.exs
index 25ec46a46..b2c075c85 100644
--- a/mix.exs
+++ b/mix.exs
@@ -40,18 +40,9 @@ defmodule Pleroma.Mixfile do
   #
   # Type `mix help compile.app` for more information.
   def application do
-    extra_applications = [:logger, :runtime_tools, :comeonin, :quack]
-
-    extra_applications =
-      if Application.get_env(:esshd, :enabled, false) do
-        [:esshd | extra_applications]
-      else
-        extra_applications
-      end
-
     [
       mod: {Pleroma.Application, []},
-      extra_applications: extra_applications,
+      extra_applications: [:logger, :runtime_tools, :comeonin, :quack],
       included_applications: [:ex_syslogger]
     ]
   end
@@ -129,7 +120,7 @@ defmodule Pleroma.Mixfile do
       {:recon, github: "ferd/recon", tag: "2.4.0"},
       {:quack, "~> 0.1.1"},
       {:benchee, "~> 1.0"},
-      {:esshd, "~> 0.1.0"},
+      {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)},
       {:ex_rated, "~> 1.2"},
       {:plug_static_index_html, "~> 1.0.0"},
       {:excoveralls, "~> 0.11.1", only: :test}

From 78ac8ee56139ed98625c54ce627eb37047a361f0 Mon Sep 17 00:00:00 2001
From: lambda <lain@soykaf.club>
Date: Wed, 22 May 2019 11:07:51 +0000
Subject: [PATCH 48/56] Update CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b88edd072..bb2306fc4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
 ### Added
-- Optional SSH access mode. (Needs `erlang-ssh` package on Debian).
+- Optional SSH access mode. (Needs `erlang-ssh` package on some distributions).
 - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support.
 - LDAP authentication
 - External OAuth provider authentication

From 620908a2db86942a00bc0ba9c71c037061e26967 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 22 May 2019 15:44:50 +0000
Subject: [PATCH 49/56] [#699] add worker to clean expired oauth tokens

---
 CHANGELOG.md                                |  1 +
 config/config.exs                           |  4 +-
 docs/config.md                              |  2 +
 lib/pleroma/application.ex                  |  1 +
 lib/pleroma/web/oauth/token.ex              | 36 ++++++--------
 lib/pleroma/web/oauth/token/clean_worker.ex | 41 +++++++++++++++
 lib/pleroma/web/oauth/token/query.ex        | 55 +++++++++++++++++++++
 test/web/oauth/token_test.exs               | 13 +++++
 8 files changed, 132 insertions(+), 21 deletions(-)
 create mode 100644 lib/pleroma/web/oauth/token/clean_worker.ex
 create mode 100644 lib/pleroma/web/oauth/token/query.ex

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b5c42d1fd..02d64a850 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - OAuth: added support for refresh tokens
 - Emoji packs and emoji pack manager
 - Object pruning (`mix pleroma.database prune_objects`)
+- OAuth: added job to clean expired access tokens
 
 ### Changed
 - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer
diff --git a/config/config.exs b/config/config.exs
index a05f8b1d2..33b7e713d 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -481,7 +481,9 @@ config :pleroma, Pleroma.ScheduledActivity,
 
 config :pleroma, :oauth2,
   token_expires_in: 600,
-  issue_new_refresh_token: true
+  issue_new_refresh_token: true,
+  clean_expired_tokens: false,
+  clean_expired_tokens_interval: 86_400_000
 
 config :pleroma, :database, rum_enabled: false
 
diff --git a/docs/config.md b/docs/config.md
index a050068f4..264b65499 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -550,6 +550,8 @@ Configure OAuth 2 provider capabilities:
 
 * `token_expires_in` - The lifetime in seconds of the access token.
 * `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token.
+* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`.
+* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours).
 
 ## :emoji
 * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]`
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index dab45a0b2..76df3945e 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -110,6 +110,7 @@ defmodule Pleroma.Application do
         hackney_pool_children() ++
         [
           worker(Pleroma.Web.Federator.RetryQueue, []),
+          worker(Pleroma.Web.OAuth.Token.CleanWorker, []),
           worker(Pleroma.Stats, []),
           worker(Task, [&Pleroma.Web.Push.init/0], restart: :temporary, id: :web_push_init),
           worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary, id: :federator_init)
diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex
index 66c95c2e9..f412f7eb2 100644
--- a/lib/pleroma/web/oauth/token.ex
+++ b/lib/pleroma/web/oauth/token.ex
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.OAuth.Token do
   use Ecto.Schema
 
-  import Ecto.Query
   import Ecto.Changeset
 
   alias Pleroma.Repo
@@ -13,6 +12,7 @@ defmodule Pleroma.Web.OAuth.Token do
   alias Pleroma.Web.OAuth.App
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.OAuth.Token.Query
 
   @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
   @type t :: %__MODULE__{}
@@ -31,17 +31,17 @@ defmodule Pleroma.Web.OAuth.Token do
   @doc "Gets token for app by access token"
   @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
   def get_by_token(%App{id: app_id} = _app, token) do
-    from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token)
+    Query.get_by_app(app_id)
+    |> Query.get_by_token(token)
     |> Repo.find_resource()
   end
 
   @doc "Gets token for app by refresh token"
   @spec get_by_refresh_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
   def get_by_refresh_token(%App{id: app_id} = _app, token) do
-    from(t in __MODULE__,
-      where: t.app_id == ^app_id and t.refresh_token == ^token,
-      preload: [:user]
-    )
+    Query.get_by_app(app_id)
+    |> Query.get_by_refresh_token(token)
+    |> Query.preload([:user])
     |> Repo.find_resource()
   end
 
@@ -97,29 +97,25 @@ defmodule Pleroma.Web.OAuth.Token do
   end
 
   def delete_user_tokens(%User{id: user_id}) do
-    from(
-      t in Token,
-      where: t.user_id == ^user_id
-    )
+    Query.get_by_user(user_id)
     |> Repo.delete_all()
   end
 
   def delete_user_token(%User{id: user_id}, token_id) do
-    from(
-      t in Token,
-      where: t.user_id == ^user_id,
-      where: t.id == ^token_id
-    )
+    Query.get_by_user(user_id)
+    |> Query.get_by_id(token_id)
+    |> Repo.delete_all()
+  end
+
+  def delete_expired_tokens do
+    Query.get_expired_tokens()
     |> Repo.delete_all()
   end
 
   def get_user_tokens(%User{id: user_id}) do
-    from(
-      t in Token,
-      where: t.user_id == ^user_id
-    )
+    Query.get_by_user(user_id)
+    |> Query.preload([:app])
     |> Repo.all()
-    |> Repo.preload(:app)
   end
 
   def is_expired?(%__MODULE__{valid_until: valid_until}) do
diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex
new file mode 100644
index 000000000..dca852449
--- /dev/null
+++ b/lib/pleroma/web/oauth/token/clean_worker.ex
@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.OAuth.Token.CleanWorker do
+  @moduledoc """
+  The module represents functions to clean an expired oauth tokens.
+  """
+
+  # 10 seconds
+  @start_interval 10_000
+  @interval Pleroma.Config.get(
+              # 24 hours
+              [:oauth2, :clean_expired_tokens_interval],
+              86_400_000
+            )
+  @queue :background
+
+  alias Pleroma.Web.OAuth.Token
+
+  def start_link, do: GenServer.start_link(__MODULE__, nil)
+
+  def init(_) do
+    if Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) do
+      Process.send_after(self(), :perform, @start_interval)
+      {:ok, nil}
+    else
+      :ignore
+    end
+  end
+
+  @doc false
+  def handle_info(:perform, state) do
+    Process.send_after(self(), :perform, @interval)
+    PleromaJobQueue.enqueue(@queue, __MODULE__, [:clean])
+    {:noreply, state}
+  end
+
+  # Job Worker Callbacks
+  def perform(:clean), do: Token.delete_expired_tokens()
+end
diff --git a/lib/pleroma/web/oauth/token/query.ex b/lib/pleroma/web/oauth/token/query.ex
new file mode 100644
index 000000000..d92e1f071
--- /dev/null
+++ b/lib/pleroma/web/oauth/token/query.ex
@@ -0,0 +1,55 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.OAuth.Token.Query do
+  @moduledoc """
+  Contains queries for OAuth Token.
+  """
+
+  import Ecto.Query, only: [from: 2]
+
+  @type query :: Ecto.Queryable.t() | Token.t()
+
+  alias Pleroma.Web.OAuth.Token
+
+  @spec get_by_refresh_token(query, String.t()) :: query
+  def get_by_refresh_token(query \\ Token, refresh_token) do
+    from(q in query, where: q.refresh_token == ^refresh_token)
+  end
+
+  @spec get_by_token(query, String.t()) :: query
+  def get_by_token(query \\ Token, token) do
+    from(q in query, where: q.token == ^token)
+  end
+
+  @spec get_by_app(query, String.t()) :: query
+  def get_by_app(query \\ Token, app_id) do
+    from(q in query, where: q.app_id == ^app_id)
+  end
+
+  @spec get_by_id(query, String.t()) :: query
+  def get_by_id(query \\ Token, id) do
+    from(q in query, where: q.id == ^id)
+  end
+
+  @spec get_expired_tokens(query, DateTime.t() | nil) :: query
+  def get_expired_tokens(query \\ Token, date \\ nil) do
+    expired_date = date || Timex.now()
+    from(q in query, where: fragment("?", q.valid_until) < ^expired_date)
+  end
+
+  @spec get_by_user(query, String.t()) :: query
+  def get_by_user(query \\ Token, user_id) do
+    from(q in query, where: q.user_id == ^user_id)
+  end
+
+  @spec preload(query, any) :: query
+  def preload(query \\ Token, assoc_preload \\ [])
+
+  def preload(query, assoc_preload) when is_list(assoc_preload) do
+    from(q in query, preload: ^assoc_preload)
+  end
+
+  def preload(query, _assoc_preload), do: query
+end
diff --git a/test/web/oauth/token_test.exs b/test/web/oauth/token_test.exs
index ad2a49f09..3c07309b7 100644
--- a/test/web/oauth/token_test.exs
+++ b/test/web/oauth/token_test.exs
@@ -69,4 +69,17 @@ defmodule Pleroma.Web.OAuth.TokenTest do
 
     assert tokens == 2
   end
+
+  test "deletes expired tokens" do
+    insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3))
+    insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3))
+    t3 = insert(:oauth_token)
+    t4 = insert(:oauth_token, valid_until: Timex.shift(Timex.now(), minutes: 10))
+    {tokens, _} = Token.delete_expired_tokens()
+    assert tokens == 2
+    available_tokens = Pleroma.Repo.all(Token)
+
+    token_ids = available_tokens |> Enum.map(& &1.id)
+    assert token_ids == [t3.id, t4.id]
+  end
 end

From 54e10a3e55fe46b71ef7f330605baf8bcccd5a44 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 22 May 2019 20:10:52 +0300
Subject: [PATCH 50/56] Disable timeouts for object pruning query

---
 lib/mix/tasks/pleroma/database.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index f9bafb277..4d480ac3f 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -107,7 +107,7 @@ defmodule Mix.Tasks.Pleroma.Database do
       where:
         fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
     )
-    |> Repo.delete_all()
+    |> Repo.delete_all(timeout: :infinity)
 
     if Keyword.get(options, :vacuum) do
       Logger.info("Runnning VACUUM FULL")

From 75b6c4b00433560fb5ee502f13e8261b4b8a246a Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 04:15:59 +0000
Subject: [PATCH 51/56] mrf: defang policy modules for filtering user profile
 objects

---
 lib/pleroma/web/activity_pub/mrf/simple_policy.ex  | 6 ++++--
 lib/pleroma/web/activity_pub/mrf/user_allowlist.ex | 6 ++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index 7190652d2..ffaa4b7db 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -105,8 +105,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
   defp check_report_removal(_actor_info, object), do: {:ok, object}
 
   @impl true
-  def filter(object) do
-    actor_info = URI.parse(object["actor"])
+  def filter(%{"actor" => actor} = object) do
+    actor_info = URI.parse(actor)
 
     with {:ok, object} <- check_accept(actor_info, object),
          {:ok, object} <- check_reject(actor_info, object),
@@ -119,4 +119,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
       _e -> {:reject, nil}
     end
   end
+
+  def filter(object), do: {:ok, object}
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex
index f5078d818..47663414a 100644
--- a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex
+++ b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex
@@ -19,10 +19,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do
   end
 
   @impl true
-  def filter(object) do
-    actor_info = URI.parse(object["actor"])
+  def filter(%{"actor" => actor} = object) do
+    actor_info = URI.parse(actor)
     allow_list = Config.get([:mrf_user_allowlist, String.to_atom(actor_info.host)], [])
 
     filter_by_list(object, allow_list)
   end
+
+  def filter(object), do: {:ok, object}
 end

From 60f882b09f5f837546d59f8eef56b905e065ec60 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 04:33:10 +0000
Subject: [PATCH 52/56] activitypub: run user objects through MRF filters

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 3d9679ec0..aa0229db7 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -909,7 +909,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def user_data_from_user_object(data) do
+  defp object_to_user_data(data) do
     avatar =
       data["icon"]["url"] &&
         %{
@@ -956,9 +956,19 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     {:ok, user_data}
   end
 
+  def user_data_from_user_object(data) do
+    with {:ok, data} <- MRF.filter(data),
+         {:ok, data} <- object_to_user_data(data) do
+      {:ok, data}
+    else
+      e -> {:error, e}
+    end
+  end
+
   def fetch_and_prepare_user_from_ap_id(ap_id) do
-    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do
-      user_data_from_user_object(data)
+    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
+         {:ok, data} <- user_data_from_user_object(data) do
+      {:ok, data}
     else
       e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
     end

From baf72d6c580e5c05ef5fea8a57c57150a5d38589 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 04:55:16 +0000
Subject: [PATCH 53/56] mrf: simple policy: add the ability to strip avatars
 and banners from user profiles

---
 config/config.exs                             |  4 ++-
 .../web/activity_pub/mrf/simple_policy.ex     | 32 +++++++++++++++++++
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 33b7e713d..e90821d66 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -314,7 +314,9 @@ config :pleroma, :mrf_simple,
   federated_timeline_removal: [],
   report_removal: [],
   reject: [],
-  accept: []
+  accept: [],
+  avatar_removal: [],
+  banner_removal: []
 
 config :pleroma, :mrf_keyword,
   reject: [],
diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index ffaa4b7db..890d70a7a 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -104,6 +104,26 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
 
   defp check_report_removal(_actor_info, object), do: {:ok, object}
 
+  defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do
+    if actor_host in Pleroma.Config.get([:mrf_simple, :avatar_removal]) do
+      {:ok, Map.delete(object, "icon")}
+    else
+      {:ok, object}
+    end
+  end
+
+  defp check_avatar_removal(_actor_info, object), do: {:ok, object}
+
+  defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do
+    if actor_host in Pleroma.Config.get([:mrf_simple, :banner_removal]) do
+      {:ok, Map.delete(object, "image")}
+    else
+      {:ok, object}
+    end
+  end
+
+  defp check_banner_removal(_actor_info, object), do: {:ok, object}
+
   @impl true
   def filter(%{"actor" => actor} = object) do
     actor_info = URI.parse(actor)
@@ -120,5 +140,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
     end
   end
 
+  def filter(%{"id" => actor, "type" => obj_type} = object)
+      when obj_type in ["Application", "Group", "Organization", "Person", "Service"] do
+    actor_info = URI.parse(actor)
+
+    with {:ok, object} <- check_avatar_removal(actor_info, object),
+         {:ok, object} <- check_banner_removal(actor_info, object) do
+      {:ok, object}
+    else
+      _e -> {:reject, nil}
+    end
+  end
+
   def filter(object), do: {:ok, object}
 end

From 8086c7aed6cdc3b2ac1c09c6c40344e47be08ed9 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 05:08:37 +0000
Subject: [PATCH 54/56] tests: add tests for banner and avatar removal

---
 .../activity_pub/mrf/simple_policy_test.exs   | 73 ++++++++++++++++++-
 1 file changed, 72 insertions(+), 1 deletion(-)

diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs
index 74af7dcde..3d1f26e60 100644
--- a/test/web/activity_pub/mrf/simple_policy_test.exs
+++ b/test/web/activity_pub/mrf/simple_policy_test.exs
@@ -17,7 +17,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do
       federated_timeline_removal: [],
       report_removal: [],
       reject: [],
-      accept: []
+      accept: [],
+      avatar_removal: [],
+      banner_removal: []
     )
 
     on_exit(fn ->
@@ -206,6 +208,60 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do
     end
   end
 
+  describe "when :avatar_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :avatar_removal], [])
+
+      remote_user = build_remote_user()
+
+      assert SimplePolicy.filter(remote_user) == {:ok, remote_user}
+    end
+
+    test "is not empty but it doesn't have a matching host" do
+      Config.put([:mrf_simple, :avatar_removal], ["non.matching.remote"])
+
+      remote_user = build_remote_user()
+
+      assert SimplePolicy.filter(remote_user) == {:ok, remote_user}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :avatar_removal], ["remote.instance"])
+
+      remote_user = build_remote_user()
+      {:ok, filtered} = SimplePolicy.filter(remote_user)
+
+      refute filtered["icon"]
+    end
+  end
+
+  describe "when :banner_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :banner_removal], [])
+
+      remote_user = build_remote_user()
+
+      assert SimplePolicy.filter(remote_user) == {:ok, remote_user}
+    end
+
+    test "is not empty but it doesn't have a matching host" do
+      Config.put([:mrf_simple, :banner_removal], ["non.matching.remote"])
+
+      remote_user = build_remote_user()
+
+      assert SimplePolicy.filter(remote_user) == {:ok, remote_user}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :banner_removal], ["remote.instance"])
+
+      remote_user = build_remote_user()
+      {:ok, filtered} = SimplePolicy.filter(remote_user)
+
+      refute filtered["image"]
+    end
+  end
+
   defp build_local_message do
     %{
       "actor" => "#{Pleroma.Web.base_url()}/users/alice",
@@ -217,4 +273,19 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do
   defp build_remote_message do
     %{"actor" => "https://remote.instance/users/bob"}
   end
+
+  defp build_remote_user do
+    %{
+      "id" => "https://remote.instance/users/bob",
+      "icon" => %{
+        "url" => "http://example.com/image.jpg",
+        "type" => "Image"
+      },
+      "image" => %{
+        "url" => "http://example.com/image.jpg",
+        "type" => "Image"
+      },
+      "type" => "Person"
+    }
+  end
 end

From 7d9b33b3cebeed451210f754a8c34cc14a9e969b Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 05:55:09 +0000
Subject: [PATCH 55/56] update documentation for the new MRF features [no-ci]

---
 CHANGELOG.md   | 2 ++
 docs/config.md | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 02d64a850..2a2b11ddf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -42,6 +42,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Emoji packs and emoji pack manager
 - Object pruning (`mix pleroma.database prune_objects`)
 - OAuth: added job to clean expired access tokens
+- MRF: Support for rejecting reports from specific instances (`mrf_simple`)
+- MRF: Support for stripping avatars and banner images from specific instances (`mrf_simple`)
 
 ### Changed
 - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer
diff --git a/docs/config.md b/docs/config.md
index 264b65499..1d1d24c32 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -220,6 +220,9 @@ relates to mascots on the mastodon frontend
 * `federated_timeline_removal`: List of instances to remove from Federated (aka The Whole Known Network) Timeline
 * `reject`: List of instances to reject any activities from
 * `accept`: List of instances to accept any activities from
+* `report_removal`: List of instances to reject reports from
+* `avatar_removal`: List of instances to strip avatars from
+* `banner_removal`: List of instances to strip banners from
 
 ## :mrf_rejectnonpublic
 * `allow_followersonly`: whether to allow followers-only posts

From 356c047759735bcd984ebd44059a21a3cf22af0e Mon Sep 17 00:00:00 2001
From: Alfie Pates <alfie@alfiepates.me>
Date: Thu, 23 May 2019 22:33:27 +0100
Subject: [PATCH 56/56] explicitly set reverse proxy upstream to IPv4

  since Pleroma.Web.Endpoint binds on IPv4 only and `localhost.` resolves to
  [::0] on some systems

  fixes #930.
---
 installation/caddyfile-pleroma.example | 4 +++-
 installation/pleroma-apache.conf       | 6 ++++--
 installation/pleroma.nginx             | 4 +++-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/installation/caddyfile-pleroma.example b/installation/caddyfile-pleroma.example
index fcf76718e..7985d9c67 100644
--- a/installation/caddyfile-pleroma.example
+++ b/installation/caddyfile-pleroma.example
@@ -10,7 +10,9 @@ example.tld  {
 
   gzip
 
-  proxy / localhost:4000 {
+  # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+  # and `localhost.` resolves to [::0] on some systems: see issue #930
+  proxy / 127.0.0.1:4000 {
     websocket
     transparent
   }
diff --git a/installation/pleroma-apache.conf b/installation/pleroma-apache.conf
index 2beb7c4cc..b5640ac3d 100644
--- a/installation/pleroma-apache.conf
+++ b/installation/pleroma-apache.conf
@@ -58,8 +58,10 @@ CustomLog ${APACHE_LOG_DIR}/access.log combined
     RewriteRule /(.*) ws://localhost:4000/$1 [P,L]
 
     ProxyRequests off
-    ProxyPass / http://localhost:4000/
-    ProxyPassReverse / http://localhost:4000/
+    # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+    # and `localhost.` resolves to [::0] on some systems: see issue #930
+    ProxyPass / http://127.0.0.1:4000/
+    ProxyPassReverse / http://127.0.0.1:4000/
 
     RequestHeader set Host ${servername}
     ProxyPreserveHost On
diff --git a/installation/pleroma.nginx b/installation/pleroma.nginx
index cc75d78b2..7425da33f 100644
--- a/installation/pleroma.nginx
+++ b/installation/pleroma.nginx
@@ -69,7 +69,9 @@ server {
         proxy_set_header Connection "upgrade";
         proxy_set_header Host $http_host;
 
-        proxy_pass http://localhost:4000;
+	# this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+	# and `localhost.` resolves to [::0] on some systems: see issue #930
+        proxy_pass http://127.0.0.1:4000;
 
         client_max_body_size 16m;
     }