Merge remote-tracking branch 'upstream/develop' into aliases

.gitattributes

@@ -1,2 +1,10 @@
 *.ex diff=elixir
 *.exs diff=elixir
+
+priv/static/instance/static.css diff=css
+
+# Most of js/css files included in the repo are minified bundles,
+#   and we don't want to search/diff those as text files.
+*.js binary
+*.js.map binary
+*.css binary

.gitignore

@@ -3,6 +3,7 @@
 /db
 /deps
 /*.ez
+/test/instance
 /test/uploads
 /.elixir_ls
 /test/fixtures/DSCN0010_tmp.jpg
@@ -27,6 +28,8 @@ erl_crash.dump
 # variables.
 /config/*.secret.exs
 /config/generated_config.exs
+/config/*.env
+
 
 # Database setup file, some may forget to delete it
 /config/setup_db.psql

.gitlab-ci.yml

@@ -57,7 +57,7 @@ unit-testing:
     policy: pull
 
   services:
-  - name: postgres:9.6
+  - name: postgres:13
     alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
@@ -198,7 +198,7 @@ amd64:
   variables: &release-variables
     MIX_ENV: prod
   before_script: &before-release
-  - apt-get update && apt-get install -y cmake
+  - apt-get update && apt-get install -y cmake libmagic-dev
   - echo "import Mix.Config" > config/prod.secret.exs
   - mix local.hex --force
   - mix local.rebar --force
@@ -217,7 +217,7 @@ amd64-musl:
   cache: *release-cache
   variables: *release-variables
   before_script: &before-release-musl
-  - apk add git gcc g++ musl-dev make cmake
+  - apk add git gcc g++ musl-dev make cmake file-dev
   - echo "import Mix.Config" > config/prod.secret.exs
   - mix local.hex --force
   - mix local.rebar --force
@@ -228,8 +228,8 @@ arm:
   artifacts: *release-artifacts
   only: *release-only
   tags:
-    - arm32
+    - arm32-specified
-  image: elixir:1.10.3
+  image: arm32v7/elixir:1.10.3
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release
@@ -240,8 +240,8 @@ arm-musl:
   artifacts: *release-artifacts
   only: *release-only
   tags:
-    - arm32
+    - arm32-specified
-  image: elixir:1.10.3-alpine
+  image: arm32v7/elixir:1.10.3-alpine
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release-musl
@@ -253,7 +253,7 @@ arm64:
   only: *release-only
   tags:
     - arm
-  image: elixir:1.10.3
+  image: arm64v8/elixir:1.10.3
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release
@@ -265,8 +265,7 @@ arm64-musl:
   only: *release-only
   tags:
     - arm
-  # TODO: Replace with upstream image when 1.9.0 comes out
+  image: arm64v8/elixir:1.10.3-alpine
-  image: elixir:1.10.3-alpine
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release-musl

CHANGELOG.md

@@ -1,41 +1,111 @@
 # Changelog
+
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+### Changed
+
+- Polls now always return a `voters_count`, even if they are single-choice.
+- Admin Emails: The ap id is used as the user link in emails now.
+- Improved registration workflow for email confirmation and account approval modes.
+- **Breaking:** Changed `mix pleroma.user toggle_confirmed` to `mix pleroma.user confirm`
+- Search: When using Postgres 11+, Pleroma will use the `websearch_to_tsvector` function to parse search queries.
+- Emoji: Support the full Unicode 13.1 set of Emoji for reactions, plus regional indicators.
+
 ### Added
-- Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
+
-- Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
+- Reports now generate notifications for admins and mods.
-- Mix task option for force-unfollowing relays
+- Experimental websocket-based federation between Pleroma instances.
+- Support for local-only statuses.
+- Support pagination of blocks and mutes.
+- Account backup.
+- Configuration: Add `:instance, autofollowing_nicknames` setting to provide a way to make accounts automatically follow new users that register on the local Pleroma instance.
+- Ability to view remote timelines, with ex. `/api/v1/timelines/public?instance=lain.com` and streams `public:remote` and `public:remote:media`.
+- The site title is now injected as a `title` tag like preloads or metadata.
+- Password reset tokens now are not accepted after a certain age.
+- Mix tasks to help with displaying and removing ConfigDB entries. See `mix pleroma.config`.
+- OAuth form improvements: users are remembered by their cookie, the CSS is overridable by the admin, and the style has been improved.
+- OAuth improvements and fixes: more secure session-based authentication (by token that could be revoked anytime), ability to revoke belonging OAuth token from any client etc.
 - Ability to set ActivityPub aliases for follower migration.
 
+<details>
+  <summary>API Changes</summary>
+- Admin API: (`GET /api/pleroma/admin/users`) filter users by `unconfirmed` status and `actor_type`.
+- Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
+- Pleroma API: (`GET /api/v1/pleroma/federation_status`) Add a way to get a list of unreachable instances.
+- Mastodon API: User and conversation mutes can now auto-expire if `expires_in` parameter was given while adding the mute.
+- Admin API: An endpoint to manage frontends.
+- Streaming API: Add follow relationships updates.
+</details>
+
+### Fixed
+
+- Users with `is_discoverable` field set to false (default value) will appear in in-service search results but be hidden from external services (search bots etc.).
+- Streaming API: Posts and notifications are not dropped, when CLI task is executing.
+- Creating incorrect IPv4 address-style HTTP links when encountering certain numbers.
+
+<details>
+  <summary>API Changes</summary>
+  - Mastodon API: Current user is now included in conversation if it's the only participant.
+  - Mastodon API: Fixed last_status.account being not filled with account data.
+</details>
+
+## Unreleased (Patch)
+
+### Fixed
+
+- Fix ability to update Pleroma Chat push notifications with PUT /api/v1/push/subscription and alert type pleroma:chat_mention
+- Emoji Reaction activity filtering from blocked and muted accounts.
+
+## [2.2.1] - 2020-12-22
+
+### Changed
+- Updated Pleroma FE
+
+### Fixed
+
+- Config generation: rename `Pleroma.Upload.Filter.ExifTool` to `Pleroma.Upload.Filter.Exiftool`.
+- S3 Uploads with Elixir 1.11.
+- Mix task pleroma.user delete_activities for source installations.
+- Search: RUM index search speed has been fixed.
+- Rich Media Previews sometimes showed the wrong preview due to a bug following redirects.
+- Fixes for the autolinker.
+- Forwarded reports duplication from Pleroma instances.
+
+- <details>
+    <summary>API</summary>
+  - Statuses were not displayed for Mastodon forwarded reports.
+  </details>
+
+### Upgrade notes
+
+1. Restart Pleroma
+
+## [2.2.0] - 2020-11-12
+
+### Security
+
+- Fixed the possibility of using file uploads to spoof posts.
+
 ### Changed
 
 - **Breaking** Requires `libmagic` (or `file`) to guess file types.
+- **Breaking:** App metrics endpoint (`/api/pleroma/app_metrics`) is disabled by default, check `docs/API/prometheus.md` on enabling and configuring.
 - **Breaking:** Pleroma Admin API: emoji packs and files routes changed.
 - **Breaking:** Sensitive/NSFW statuses no longer disable link previews.
 - Search: Users are now findable by their urls.
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
 - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.
 - The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false.
-- Users with the `discoverable` field set to false will not show up in searches.
+- Users with the `is_discoverable` field set to false will not show up in searches ([bug](https://git.pleroma.social/pleroma/pleroma/-/issues/2301)).
 - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
 - Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`.
-
+- <details>
-### Added
-- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
-- Pleroma API: Importing the mutes users from CSV files.
-- Experimental websocket-based federation between Pleroma instances.
-
-<details>
   <summary>API Changes</summary>
-
+- API: Empty parameter values for integer parameters are now ignored in non-strict validaton mode.
-- Pleroma API: Importing the mutes users from CSV files.
-- Admin API: Importing emoji from a zip file
-- Pleroma API: Pagination for remote/local packs and emoji.
-
 </details>
 
 ### Removed
@@ -46,15 +116,38 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Removed `:managed_config` option. In practice, it was accidentally removed with 2.0.0 release when frontends were
 switched to a new configuration mechanism, however it was not officially removed until now.
 
+### Added
+
+- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
+- Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
+- Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email resend_confirmation_emails`)
+- Mix task option for force-unfollowing relays
+- App metrics: ability to restrict access to specified IP whitelist.
+
+<details>
+  <summary>API Changes</summary>
+
+- Admin API: Importing emoji from a zip file
+- Pleroma API: Importing the mutes users from CSV files.
+- Pleroma API: Pagination for remote/local packs and emoji.
+
+</details>
+
 ### Fixed
 
 - Add documented-but-missing chat pagination.
 - Allow sending out emails again.
+- Allow sending chat messages to yourself
+- OStatus / static FE endpoints: fixed inaccessibility for anonymous users on non-federating instances, switched to handling per `:restrict_unauthenticated` setting.
+- Fix remote users with a whitespace name.
 
-## Unreleased (Patch)
+### Upgrade notes
 
-### Changed
+1. Install libmagic and development headers (`libmagic-dev` on Ubuntu/Debian, `file-dev` on Alpine Linux)
-- API: Empty parameter values for integer parameters are now ignored in non-strict validaton mode.
+2. Run database migrations (inside Pleroma directory):
+  - OTP: `./bin/pleroma_ctl migrate`
+  - From Source: `mix ecto.migrate`
+3. Restart Pleroma
 
 ## [2.1.2] - 2020-09-17
 

Dockerfile

@@ -4,7 +4,7 @@ COPY . .
 
 ENV MIX_ENV=prod
 
-RUN apk add git gcc g++ musl-dev make cmake &&\
+RUN apk add git gcc g++ musl-dev make cmake file-dev &&\
 	echo "import Mix.Config" > config/prod.secret.exs &&\
 	mix local.hex --force &&\
 	mix local.rebar --force &&\
@@ -31,9 +31,9 @@ LABEL maintainer="ops@pleroma.social" \
 ARG HOME=/opt/pleroma
 ARG DATA=/var/lib/pleroma
 
-RUN echo "https://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\
+RUN echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\
 	apk update &&\
-	apk add exiftool imagemagick ncurses postgresql-client &&\
+	apk add exiftool imagemagick libmagic ncurses postgresql-client &&\
 	adduser --system --shell /bin/false --home ${HOME} pleroma &&\
 	mkdir -p ${DATA}/uploads &&\
 	mkdir -p ${DATA}/static &&\

SECURITY.md

@@ -6,7 +6,7 @@ Currently, Pleroma offers bugfixes and security patches only for the latest mino
 
 | Version | Support 
 |---------| --------
-| 2.1     | Bugfixes and security patches
+| 2.2     | Bugfixes and security patches
 
 ## Reporting a vulnerability
 

benchmarks/load_testing/users.ex

@@ -109,8 +109,8 @@ defmodule Pleroma.LoadTesting.Users do
   end
 
   def make_friends(%User{} = main_user, %User{} = user) do
-    {:ok, _} = User.follow(main_user, user)
+    {:ok, _, _} = User.follow(main_user, user)
-    {:ok, _} = User.follow(user, main_user)
+    {:ok, _, _} = User.follow(user, main_user)
   end
 
   @spec get_users(User.t(), keyword()) :: [User.t()]

benchmarks/mix/tasks/pleroma/benchmarks/timelines.ex

@@ -50,7 +50,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Timelines do
     )
 
     users
-    |> Enum.each(fn {:ok, follower} -> Pleroma.User.follow(follower, user) end)
+    |> Enum.each(fn {:ok, follower, user} -> Pleroma.User.follow(follower, user) end)
 
     Benchee.run(
       %{

config/config.exs

@@ -47,7 +47,6 @@ use Mix.Config
 config :pleroma, ecto_repos: [Pleroma.Repo]
 
 config :pleroma, Pleroma.Repo,
-  types: Pleroma.PostgresTypes,
   telemetry_event: [Pleroma.Repo.Instrumenter],
   migration_lock: nil
 
@@ -123,14 +122,12 @@ websocket_config = [
 
 # Configures the endpoint
 config :pleroma, Pleroma.Web.Endpoint,
-  instrumenters: [Pleroma.Web.Endpoint.Instrumenter],
   url: [host: "localhost"],
   http: [
     ip: {127, 0, 0, 1},
     dispatch: [
       {:_,
        [
-         {"/api/fedsocket/v1", Pleroma.Web.FedSockets.IncomingHandler, []},
          {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []},
          {"/websocket", Phoenix.Endpoint.CowboyWebSocket,
           {Phoenix.Transports.WebSocket,
@@ -143,22 +140,12 @@ config :pleroma, Pleroma.Web.Endpoint,
   secret_key_base: "aK4Abxf29xU9TTDKre9coZPUgevcVCFQJe/5xP/7Lt4BEif6idBIbjupVbOrbKxl",
   signing_salt: "CqaoopA2",
   render_errors: [view: Pleroma.Web.ErrorView, accepts: ~w(json)],
-  pubsub: [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2],
+  pubsub_server: Pleroma.PubSub,
   secure_cookie_flag: true,
   extra_cookie_attrs: [
     "SameSite=Lax"
   ]
 
-config :pleroma, :fed_sockets,
-  enabled: false,
-  connection_duration: :timer.hours(8),
-  rejection_duration: :timer.minutes(15),
-  fed_socket_fetches: [
-    default: 12_000,
-    interval: 3_000,
-    lazy: false
-  ]
-
 # Configures Elixir's Logger
 config :logger, :console,
   level: :debug,
@@ -235,6 +222,7 @@ config :pleroma, :instance,
     "text/bbcode"
   ],
   autofollowed_nicknames: [],
+  autofollowing_nicknames: [],
   max_pinned_statuses: 1,
   attachment_links: false,
   max_report_comment_size: 1000,
@@ -264,7 +252,8 @@ config :pleroma, :instance,
       length: 16
     ]
   ],
-  show_reactions: true
+  show_reactions: true,
+  password_reset_token_validity: 60 * 60 * 24
 
 config :pleroma, :welcome,
   direct_message: [
@@ -316,7 +305,7 @@ config :pleroma, :frontend_configurations,
     hideSitename: false,
     hideUserStats: false,
     loginMethod: "password",
-    logo: "/static/logo.png",
+    logo: "/static/logo.svg",
     logoMargin: ".1em",
     logoMask: true,
     minimalScopesMode: false,
@@ -353,8 +342,8 @@ config :pleroma, :assets,
 config :pleroma, :manifest,
   icons: [
     %{
-      src: "/static/logo.png",
+      src: "/static/logo.svg",
-      type: "image/png"
+      type: "image/svg+xml"
     }
   ],
   theme_color: "#282c37",
@@ -551,6 +540,7 @@ config :pleroma, Oban,
   queues: [
     activity_expiration: 10,
     token_expiration: 5,
+    backup: 1,
     federator_incoming: 50,
     federator_outgoing: 50,
     ingestion_queue: 50,
@@ -561,7 +551,8 @@ config :pleroma, Oban,
     background: 5,
     remote_fetcher: 2,
     attachments_cleanup: 5,
-    new_users_digest: 1
+    new_users_digest: 1,
+    mute_expire: 5
   ],
   plugins: [Oban.Plugins.Pruner],
   crontab: [
@@ -636,7 +627,12 @@ config :pleroma, Pleroma.Emails.UserEmail,
 
 config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: false
 
-config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics"
+config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
+  enabled: false,
+  auth: false,
+  ip_whitelist: [],
+  path: "/api/pleroma/app_metrics",
+  format: :text
 
 config :pleroma, Pleroma.ScheduledActivity,
   daily_user_limit: 25,
@@ -651,7 +647,7 @@ config :pleroma, :email_notifications,
   }
 
 config :pleroma, :oauth2,
-  token_expires_in: 600,
+  token_expires_in: 3600 * 24 * 365 * 100,
   issue_new_refresh_token: true,
   clean_expired_tokens: false
 
@@ -814,7 +810,7 @@ config :pleroma, :restrict_unauthenticated,
 config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: false
 
 config :pleroma, :mrf,
-  policies: Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy,
+  policies: [Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy, Pleroma.Web.ActivityPub.MRF.TagPolicy],
   transparency: true,
   transparency_exclusions: []
 
@@ -830,6 +826,11 @@ config :floki, :html_parser, Floki.HTMLParser.FastHtml
 
 config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
 
+config :pleroma, Pleroma.User.Backup,
+  purge_after_days: 30,
+  limit_days: 7,
+  dir: nil
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/description.exs

@@ -1,5 +1,4 @@
 use Mix.Config
-alias Pleroma.Docs.Generator
 
 websocket_config = [
   path: "/websocket",
@@ -273,19 +272,6 @@ config :pleroma, :config_description, [
       }
     ]
   },
-  %{
-    group: :pleroma,
-    key: :fed_sockets,
-    type: :group,
-    description: "Websocket based federation",
-    children: [
-      %{
-        key: :enabled,
-        type: :boolean,
-        description: "Enable FedSockets"
-      }
-    ]
-  },
   %{
     group: :pleroma,
     key: Pleroma.Emails.Mailer,
@@ -829,13 +815,13 @@ config :pleroma, :config_description, [
         key: :autofollowed_nicknames,
         type: {:list, :string},
         description:
-          "Set to nicknames of (local) users that every new user should automatically follow",
+          "Set to nicknames of (local) users that every new user should automatically follow"
-        suggestions: [
+      },
-          "lain",
+      %{
-          "kaniini",
+        key: :autofollowing_nicknames,
-          "lanodan",
+        type: {:list, :string},
-          "rinpatch"
+        description:
-        ]
+          "Set to nicknames of (local) users that automatically follows every newly registered user"
       },
       %{
         key: :attachment_links,
@@ -1268,7 +1254,7 @@ config :pleroma, :config_description, [
             hideSitename: false,
             hideUserStats: false,
             loginMethod: "password",
-            logo: "/static/logo.png",
+            logo: "/static/logo.svg",
             logoMargin: ".1em",
             logoMask: true,
             minimalScopesMode: false,
@@ -1354,7 +1340,7 @@ config :pleroma, :config_description, [
             key: :logo,
             type: {:string, :image},
             description: "URL of the logo, defaults to Pleroma's logo",
-            suggestions: ["/static/logo.png"]
+            suggestions: ["/static/logo.svg"]
           },
           %{
             key: :logoMargin,
@@ -1555,289 +1541,6 @@ config :pleroma, :config_description, [
       }
     ]
   },
-  %{
-    group: :pleroma,
-    key: :mrf,
-    tab: :mrf,
-    label: "MRF",
-    type: :group,
-    description: "General MRF settings",
-    children: [
-      %{
-        key: :policies,
-        type: [:module, {:list, :module}],
-        description:
-          "A list of MRF policies enabled. Module names are shortened (removed leading `Pleroma.Web.ActivityPub.MRF.` part), but on adding custom module you need to use full name.",
-        suggestions: {:list_behaviour_implementations, Pleroma.Web.ActivityPub.MRF}
-      },
-      %{
-        key: :transparency,
-        label: "MRF transparency",
-        type: :boolean,
-        description:
-          "Make the content of your Message Rewrite Facility settings public (via nodeinfo)"
-      },
-      %{
-        key: :transparency_exclusions,
-        label: "MRF transparency exclusions",
-        type: {:list, :string},
-        description:
-          "Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.",
-        suggestions: [
-          "exclusion.com"
-        ]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_simple,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.SimplePolicy",
-    label: "MRF Simple",
-    type: :group,
-    description: "Simple ingress policies",
-    children: [
-      %{
-        key: :media_removal,
-        type: {:list, :string},
-        description: "List of instances to strip media attachments from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :media_nsfw,
-        label: "Media NSFW",
-        type: {:list, :string},
-        description: "List of instances to tag all media as NSFW (sensitive) from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :federated_timeline_removal,
-        type: {:list, :string},
-        description:
-          "List of instances to remove from the Federated (aka The Whole Known Network) Timeline",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :reject,
-        type: {:list, :string},
-        description: "List of instances to reject activities from (except deletes)",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :accept,
-        type: {:list, :string},
-        description: "List of instances to only accept activities from (except deletes)",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :followers_only,
-        type: {:list, :string},
-        description: "Force posts from the given instances to be visible by followers only",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :report_removal,
-        type: {:list, :string},
-        description: "List of instances to reject reports from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :avatar_removal,
-        type: {:list, :string},
-        description: "List of instances to strip avatars from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :banner_removal,
-        type: {:list, :string},
-        description: "List of instances to strip banners from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :reject_deletes,
-        type: {:list, :string},
-        description: "List of instances to reject deletions from",
-        suggestions: ["example.com", "*.example.com"]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_activity_expiration,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy",
-    label: "MRF Activity Expiration Policy",
-    type: :group,
-    description: "Adds automatic expiration to all local activities",
-    children: [
-      %{
-        key: :days,
-        type: :integer,
-        description: "Default global expiration time for all local activities (in days)",
-        suggestions: [90, 365]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_subchain,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.SubchainPolicy",
-    label: "MRF Subchain",
-    type: :group,
-    description:
-      "This policy processes messages through an alternate pipeline when a given message matches certain criteria." <>
-        " All criteria are configured as a map of regular expressions to lists of policy modules.",
-    children: [
-      %{
-        key: :match_actor,
-        type: {:map, {:list, :string}},
-        description: "Matches a series of regular expressions against the actor field",
-        suggestions: [
-          %{
-            ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy]
-          }
-        ]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_rejectnonpublic,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.RejectNonPublic",
-    description: "RejectNonPublic drops posts with non-public visibility settings.",
-    label: "MRF Reject Non Public",
-    type: :group,
-    children: [
-      %{
-        key: :allow_followersonly,
-        label: "Allow followers-only",
-        type: :boolean,
-        description: "Whether to allow followers-only posts"
-      },
-      %{
-        key: :allow_direct,
-        type: :boolean,
-        description: "Whether to allow direct messages"
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_hellthread,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.HellthreadPolicy",
-    label: "MRF Hellthread",
-    type: :group,
-    description: "Block messages with excessive user mentions",
-    children: [
-      %{
-        key: :delist_threshold,
-        type: :integer,
-        description:
-          "Number of mentioned users after which the message gets removed from timelines and" <>
-            "disables notifications. Set to 0 to disable.",
-        suggestions: [10]
-      },
-      %{
-        key: :reject_threshold,
-        type: :integer,
-        description:
-          "Number of mentioned users after which the messaged gets rejected. Set to 0 to disable.",
-        suggestions: [20]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_keyword,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy",
-    label: "MRF Keyword",
-    type: :group,
-    description: "Reject or Word-Replace messages with a keyword or regex",
-    children: [
-      %{
-        key: :reject,
-        type: {:list, :string},
-        description:
-          "A list of patterns which result in message being rejected. Each pattern can be a string or a regular expression.",
-        suggestions: ["foo", ~r/foo/iu]
-      },
-      %{
-        key: :federated_timeline_removal,
-        type: {:list, :string},
-        description:
-          "A list of patterns which result in message being removed from federated timelines (a.k.a unlisted). Each pattern can be a string or a regular expression.",
-        suggestions: ["foo", ~r/foo/iu]
-      },
-      %{
-        key: :replace,
-        type: {:list, :tuple},
-        description:
-          "A list of tuples containing {pattern, replacement}. Each pattern can be a string or a regular expression.",
-        suggestions: [{"foo", "bar"}, {~r/foo/iu, "bar"}]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_mention,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.MentionPolicy",
-    label: "MRF Mention",
-    type: :group,
-    description: "Block messages which mention a specific user",
-    children: [
-      %{
-        key: :actors,
-        type: {:list, :string},
-        description: "A list of actors for which any post mentioning them will be dropped",
-        suggestions: ["actor1", "actor2"]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_vocabulary,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.VocabularyPolicy",
-    label: "MRF Vocabulary",
-    type: :group,
-    description: "Filter messages which belong to certain activity vocabularies",
-    children: [
-      %{
-        key: :accept,
-        type: {:list, :string},
-        description:
-          "A list of ActivityStreams terms to accept. If empty, all supported messages are accepted.",
-        suggestions: ["Create", "Follow", "Mention", "Announce", "Like"]
-      },
-      %{
-        key: :reject,
-        type: {:list, :string},
-        description:
-          "A list of ActivityStreams terms to reject. If empty, no messages are rejected.",
-        suggestions: ["Create", "Follow", "Mention", "Announce", "Like"]
-      }
-    ]
-  },
-  # %{
-  #   group: :pleroma,
-  #   key: :mrf_user_allowlist,
-  #   tab: :mrf,
-  #   related_policy: "Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy",
-  #   type: :map,
-  #   description:
-  #     "The keys in this section are the domain names that the policy should apply to." <>
-  #       " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID",
-  #     suggestions: [
-  #       %{"example.org" => ["https://example.org/users/admin"]}
-  #     ]
-  #   ]
-  # },
   %{
     group: :pleroma,
     key: :media_proxy,
@@ -2250,14 +1953,8 @@ config :pleroma, :config_description, [
     group: :pleroma,
     key: Oban,
     type: :group,
-    description: """
+    description:
-    [Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration.
+      "[Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration.",
-
-    Note: if you are running PostgreSQL in [`silent_mode`](https://postgresqlco.nf/en/doc/param/silent_mode?version=9.1),
-      it's advised to set [`log_destination`](https://postgresqlco.nf/en/doc/param/log_destination?version=9.1) to `syslog`,
-      otherwise `postmaster.log` file may grow because of "you don't own a lock of type ShareLock" warnings
-      (see https://github.com/sorentwo/oban/issues/52).
-    """,
     children: [
       %{
         key: :log,
@@ -2288,6 +1985,12 @@ config :pleroma, :config_description, [
             description: "Activity expiration queue",
             suggestions: [10]
           },
+          %{
+            key: :backup,
+            type: :integer,
+            description: "Backup queue",
+            suggestions: [1]
+          },
           %{
             key: :attachments_cleanup,
             type: :integer,
@@ -2831,7 +2534,7 @@ config :pleroma, :config_description, [
         key: :token_expires_in,
         type: :integer,
         description: "The lifetime in seconds of the access token",
-        suggestions: [600]
+        suggestions: [2_592_000]
       },
       %{
         key: :issue_new_refresh_token,
@@ -3144,22 +2847,6 @@ config :pleroma, :config_description, [
       }
     ]
   },
-  %{
-    group: :pleroma,
-    key: :mrf_normalize_markup,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.NormalizeMarkup",
-    label: "MRF Normalize Markup",
-    description: "MRF NormalizeMarkup settings. Scrub configured hypertext markup.",
-    type: :group,
-    children: [
-      %{
-        key: :scrub_policy,
-        type: :module,
-        suggestions: [Pleroma.HTML.Scrubber.Default]
-      }
-    ]
-  },
   %{
     group: :pleroma,
     key: Pleroma.User,
@@ -3349,33 +3036,6 @@ config :pleroma, :config_description, [
       }
     ]
   },
-  %{
-    group: :pleroma,
-    key: :mrf_object_age,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy",
-    label: "MRF Object Age",
-    type: :group,
-    description:
-      "Rejects or delists posts based on their timestamp deviance from your server's clock.",
-    children: [
-      %{
-        key: :threshold,
-        type: :integer,
-        description: "Required age (in seconds) of a post before actions are taken.",
-        suggestions: [172_800]
-      },
-      %{
-        key: :actions,
-        type: {:list, :atom},
-        description:
-          "A list of actions to apply to the post. `:delist` removes the post from public timelines; " <>
-            "`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines; " <>
-            "`:reject` rejects the message entirely",
-        suggestions: [:delist, :strip_followers, :reject]
-      }
-    ]
-  },
   %{
     group: :pleroma,
     key: :modules,
@@ -3722,5 +3382,62 @@ config :pleroma, :config_description, [
         suggestions: [2]
       }
     ]
+  },
+  %{
+    group: :pleroma,
+    key: Pleroma.User.Backup,
+    type: :group,
+    description: "Account Backup",
+    children: [
+      %{
+        key: :purge_after_days,
+        type: :integer,
+        description: "Remove backup achives after N days",
+        suggestions: [30]
+      },
+      %{
+        key: :limit_days,
+        type: :integer,
+        description: "Limit user to export not more often than once per N days",
+        suggestions: [7]
+      }
+    ]
+  },
+  %{
+    group: :prometheus,
+    key: Pleroma.Web.Endpoint.MetricsExporter,
+    type: :group,
+    description: "Prometheus app metrics endpoint configuration",
+    children: [
+      %{
+        key: :enabled,
+        type: :boolean,
+        description: "[Pleroma extension] Enables app metrics endpoint."
+      },
+      %{
+        key: :ip_whitelist,
+        type: [{:list, :string}, {:list, :charlist}, {:list, :tuple}],
+        description:
+          "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses."
+      },
+      %{
+        key: :auth,
+        type: [:boolean, :tuple],
+        description: "Enables HTTP Basic Auth for app metrics endpoint.",
+        suggestion: [false, {:basic, "myusername", "mypassword"}]
+      },
+      %{
+        key: :path,
+        type: :string,
+        description: "App metrics endpoint URI path.",
+        suggestions: ["/api/pleroma/app_metrics"]
+      },
+      %{
+        key: :format,
+        type: :atom,
+        description: "App metrics endpoint output format.",
+        suggestions: [:text, :protobuf]
+      }
+    ]
   }
 ]

config/releases.exs

@@ -1,31 +0,0 @@
-import Config
-
-config :pleroma, :instance, static_dir: "/var/lib/pleroma/static"
-config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads"
-config :pleroma, :modules, runtime_dir: "/var/lib/pleroma/modules"
-
-config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
-
-config :pleroma, release: true, config_path: config_path
-
-if File.exists?(config_path) do
-  import_config config_path
-else
-  warning = [
-    IO.ANSI.red(),
-    IO.ANSI.bright(),
-    "!!! #{config_path} not found! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file",
-    IO.ANSI.reset()
-  ]
-
-  IO.puts(warning)
-end
-
-exported_config =
-  config_path
-  |> Path.dirname()
-  |> Path.join("prod.exported_from_db.secret.exs")
-
-if File.exists?(exported_config) do
-  import_config exported_config
-end

config/test.exs

@@ -19,11 +19,6 @@ config :logger, :console,
   level: :warn,
   format: "\n[$level] $message\n"
 
-config :pleroma, :fed_sockets,
-  enabled: false,
-  connection_duration: 5,
-  rejection_duration: 5
-
 config :pleroma, :auth, oauth_consumer_strategies: []
 
 config :pleroma, Pleroma.Upload,
@@ -52,7 +47,10 @@ config :pleroma, Pleroma.Repo,
   password: "postgres",
   database: "pleroma_test",
   hostname: System.get_env("DB_HOST") || "localhost",
-  pool: Ecto.Adapters.SQL.Sandbox
+  pool: Ecto.Adapters.SQL.Sandbox,
+  pool_size: 50
+
+config :pleroma, :dangerzone, override_repo_pool_size: true
 
 # Reduce hash rounds for testing
 config :pbkdf2_elixir, rounds: 1
@@ -126,6 +124,16 @@ config :tzdata, :autoupdate, :disabled
 
 config :pleroma, :mrf, policies: []
 
+config :pleroma, :pipeline,
+  object_validator: Pleroma.Web.ActivityPub.ObjectValidatorMock,
+  mrf: Pleroma.Web.ActivityPub.MRFMock,
+  activity_pub: Pleroma.Web.ActivityPub.ActivityPubMock,
+  side_effects: Pleroma.Web.ActivityPub.SideEffectsMock,
+  federator: Pleroma.Web.FederatorMock,
+  config: Pleroma.ConfigMock
+
+config :pleroma, :cachex, provider: Pleroma.CachexMock
+
 if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"
 else

docs/API/admin_api.md

@@ -20,12 +20,14 @@ Configuration options:
     - `external`: only external users
     - `active`: only active users
     - `need_approval`: only unapproved users
+    - `unconfirmed`: only unconfirmed users
     - `deactivated`: only deactivated users
     - `is_admin`: users with admin role
     - `is_moderator`: users with moderator role
   - *optional* `page`: **integer** page number
   - *optional* `page_size`: **integer** number of users per page (default is `50`)
   - *optional* `tags`: **[string]** tags list
+  - *optional* `actor_types`: **[string]** actor type list (`Person`, `Service`, `Application`)
   - *optional* `name`: **string** user display name
   - *optional* `email`: **string** user email
 - Example: `https://mypleroma.org/api/pleroma/admin/users?query=john&filters=local,active&page=1&page_size=10&tags[]=some_tag&tags[]=another_tag&name=display_name&email=email@example.com`
@@ -552,7 +554,7 @@ Response:
   * `show_role`
   * `skip_thread_containment`
   * `fields`
-  * `discoverable`
+  * `is_discoverable`
   * `actor_type`
 
 * Responses:
@@ -1497,3 +1499,66 @@ Returns the content of the document
   "url": "https://example.com/instance/panel.html"
 }
 ```
+
+## `GET /api/pleroma/admin/frontends
+
+### List available frontends
+
+- Response:
+
+```json
+[
+   {
+    "build_url": "https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build",
+    "git": "https://git.pleroma.social/pleroma/fedi-fe",
+    "installed": true,
+    "name": "fedi-fe",
+    "ref": "master"
+  },
+  {
+    "build_url": "https://git.pleroma.social/lambadalambda/kenoma/-/jobs/artifacts/${ref}/download?job=build",
+    "git": "https://git.pleroma.social/lambadalambda/kenoma",
+    "installed": false,
+    "name": "kenoma",
+    "ref": "master"
+  }
+]
+```
+
+## `POST /api/pleroma/admin/frontends/install`
+
+### Install a frontend
+
+- Params:
+  - `name`: frontend name, required
+  - `ref`: frontend ref
+  - `file`: path to a frontend zip file
+  - `build_url`: build URL
+  - `build_dir`: build directory
+
+- Response:
+
+```json
+[
+   {
+    "build_url": "https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build",
+    "git": "https://git.pleroma.social/pleroma/fedi-fe",
+    "installed": true,
+    "name": "fedi-fe",
+    "ref": "master"
+  },
+  {
+    "build_url": "https://git.pleroma.social/lambadalambda/kenoma/-/jobs/artifacts/${ref}/download?job=build",
+    "git": "https://git.pleroma.social/lambadalambda/kenoma",
+    "installed": false,
+    "name": "kenoma",
+    "ref": "master"
+  }
+]
+```
+
+```json
+{
+  "error": "Could not install frontend"
+}
+```

docs/API/chats.md

@@ -116,6 +116,10 @@ The modified chat message
 This will return a list of chats that you have been involved in, sorted by their
 last update (so new chats will be at the top).
 
+Parameters:
+
+- with_muted: Include chats from muted users (boolean).
+
 Returned data:
 
 ```json
@@ -173,11 +177,14 @@ Returned data:
     "created_at": "2020-04-21T15:06:45.000Z",
     "emojis": [],
     "id": "12",
-    "unread": false
+    "unread": false,
+    "idempotency_key": "75442486-0874-440c-9db1-a7006c25a31f"
   }
 ]
 ```
 
+- idempotency_key: The copy of the `idempotency-key` HTTP request header that can be used for optimistic message sending. Included only during the first few minutes after the message creation.
+
 ### Posting a chat message
 
 Posting a chat message for given Chat id works like this:

docs/API/differences_in_mastoapi_responses.md

@@ -4,17 +4,21 @@ A Pleroma instance can be identified by "<Mastodon version> (compatible; Pleroma
 
 ## Flake IDs
 
-Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mastodon's ids they are lexically sortable strings
+Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However, just like Mastodon's ids, they are lexically sortable strings
 
 ## Timelines
 
 Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users.
+
 Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`.
+
 Adding the parameter `reply_visibility` to the public and home timelines queries will filter replies. Possible values: without parameter (default) shows all replies, `following` - replies directed to you or users you follow, `self` - replies directed to you.
 
+Adding the parameter `instance=lain.com` to the public timeline will show only statuses originating from `lain.com` (or any remote instance).
+
 ## Statuses
 
-- `visibility`: has an additional possible value `list`
+- `visibility`: has additional possible values `list` and `local` (for local-only statuses)
 
 Has these additional fields under the `pleroma` object:
 
@@ -22,8 +26,8 @@ Has these additional fields under the `pleroma` object:
 - `conversation_id`: the ID of the AP context the status is associated with (if any)
 - `direct_conversation_id`: the ID of the Mastodon direct message conversation the status is associated with (if any)
 - `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any)
-- `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
+- `content`: a map consisting of alternate representations of the `content` property with the key being its mimetype. Currently, the only alternate representation supported is `text/plain`
-- `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
+- `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being its mimetype. Currently, the only alternate representation supported is `text/plain`
 - `expires_at`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire
 - `thread_muted`: true if the thread the post belongs to is muted
 - `emoji_reactions`: A list with emoji / reaction maps. The format is `{name: "☕", count: 1, me: true}`. Contains no information about the reacting users, for that use the `/statuses/:id/reactions` endpoint.
@@ -80,7 +84,7 @@ Has these additional fields under the `pleroma` object:
 
 - `show_role`: boolean, nullable, true when the user wants his role (e.g admin, moderator) to be shown
 - `no_rich_text` - boolean, nullable, true when html tags are stripped from all statuses requested from the API
-- `discoverable`: boolean, true when the user allows discovery of the account in search results and other services.
+- `discoverable`: boolean, true when the user allows external services (search bots) etc. to index / list the account (regardless of this setting, user will still appear in regular search results)
 - `actor_type`: string, the type of this account.
 
 ## Conversations
@@ -125,12 +129,30 @@ The `type` value is `pleroma:emoji_reaction`. Has these fields:
 - `account`: The account of the user who reacted
 - `status`: The status that was reacted on
 
+### ChatMention Notification (not default)
+
+This notification has to be requested explicitly.
+
+The `type` value is `pleroma:chat_mention`
+
+- `account`: The account who sent the message
+- `chat_message`: The chat message
+
+### Report Notification (not default)
+
+This notification has to be requested explicitly.
+
+The `type` value is `pleroma:report`
+
+- `account`: The account who reported
+- `report`: The report
+
 ## GET `/api/v1/notifications`
 
 Accepts additional parameters:
 
 - `exclude_visibilities`: will exclude the notifications for activities with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`). Usage example: `GET /api/v1/notifications?exclude_visibilities[]=direct&exclude_visibilities[]=private`.
-- `include_types`: will include the notifications for activities with the given types. The parameter accepts an array of types (`mention`, `follow`, `reblog`, `favourite`, `move`, `pleroma:emoji_reaction`). Usage example: `GET /api/v1/notifications?include_types[]=mention&include_types[]=reblog`.
+- `include_types`: will include the notifications for activities with the given types. The parameter accepts an array of types (`mention`, `follow`, `reblog`, `favourite`, `move`, `pleroma:emoji_reaction`, `pleroma:chat_mention`, `pleroma:report`). Usage example: `GET /api/v1/notifications?include_types[]=mention&include_types[]=reblog`.
 
 ## DELETE `/api/v1/notifications/destroy_multiple`
 
@@ -148,10 +170,10 @@ Returns on success: 200 OK `{}`
 
 Additional parameters can be added to the JSON body/Form data:
 
-- `preview`: boolean, if set to `true` the post won't be actually posted, but the status entitiy would still be rendered back. This could be useful for previewing rich text/custom emoji, for example.
+- `preview`: boolean, if set to `true` the post won't be actually posted, but the status entity would still be rendered back. This could be useful for previewing rich text/custom emoji, for example.
 - `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint.
-- `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply.
+- `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for post visibility are not affected by this and will still apply.
-- `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted` or `public`) it can be used to address a List by setting it to `list:LIST_ID`.
+- `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted`, `local` or `public`) it can be used to address a List by setting it to `list:LIST_ID`.
 - `expires_in`: The number of seconds the posted activity should expire in. When a posted activity expires it will be deleted from the server, and a delete request for it will be federated. This needs to be longer than an hour.
 - `in_reply_to_conversation_id`: Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`.
 
@@ -186,7 +208,7 @@ Additional parameters can be added to the JSON body/Form data:
 - `allow_following_move` - if true, allows automatically follow moved following accounts
 - `also_known_as` - array of ActivityPub IDs, needed for following move
 - `pleroma_background_image` - sets the background image of the user. Can be set to "" (an empty string) to reset.
-- `discoverable` - if true, discovery of this account in search results and other services is allowed.
+- `discoverable` - if true, external services (search bots) etc. are allowed to index / list the account (regardless of this setting, user will still appear in regular search results).
 - `actor_type` - the type of this account.
 - `accepts_chat_messages` - if false, this account will reject all chat messages.
 
@@ -212,7 +234,7 @@ Post here request with `grant_type=refresh_token` to obtain new access token. Re
 
 `POST /api/v1/accounts`
 
-Has theses additional parameters (which are the same as in Pleroma-API):
+Has these additional parameters (which are the same as in Pleroma-API):
 
 - `fullname`: optional
 - `bio`: optional
@@ -240,6 +262,16 @@ Has theses additional parameters (which are the same as in Pleroma-API):
 - `pleroma.metadata.post_formats`: A list of the allowed post format types
 - `vapid_public_key`: The public key needed for push messages
 
+## Push Subscription
+
+`POST /api/v1/push/subscription`
+`PUT /api/v1/push/subscription`
+
+Permits these additional alert types:
+
+- pleroma:chat_mention
+- pleroma:emoji_reaction
+
 ## Markers
 
 Has these additional fields under the `pleroma` object:
@@ -248,8 +280,31 @@ Has these additional fields under the `pleroma` object:
 
 ## Streaming
 
+### Chats
+
 There is an additional `user:pleroma_chat` stream. Incoming chat messages will make the current chat be sent to this `user` stream. The `event` of an incoming chat message is `pleroma:chat_update`. The payload is the updated chat with the incoming chat message in the `last_message` field.
 
+### Remote timelines
+
+For viewing remote server timelines, there are `public:remote` and `public:remote:media` streams. Each of these accept a parameter like `?instance=lain.com`.
+
+### Follow relationships updates
+
+Pleroma streams follow relationships updates as `pleroma:follow_relationships_update` events to the `user` stream.
+
+The message payload consist of:
+
+- `state`: a relationship state, one of `follow_pending`, `follow_accept` or `follow_reject`.
+
+- `follower` and `following` maps with following fields:
+  - `id`: user ID
+  - `follower_count`: follower count
+  - `following_count`: following count
+
+## User muting and thread muting
+
+Both user muting and thread muting can be done for only a certain time by adding an `expires_in` parameter to the API calls and giving the expiration time in seconds.
+
 ## Not implemented
 
 Pleroma is generally compatible with the Mastodon 2.7.2 API, but some newer features and non-essential features are omitted. These features usually return an HTTP 200 status code, but with an empty response. While they may be added in the future, they are considered low priority.

docs/API/pleroma_api.md

@@ -579,14 +579,14 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
 ### React to a post with a unicode emoji
 * Method: `PUT`
 * Authentication: required
-* Params: `emoji`: A single character unicode emoji
+* Params: `emoji`: A unicode RGI emoji or a regional indicator
 * Response: JSON, the status.
 
 ## `DELETE /api/v1/pleroma/statuses/:id/reactions/:emoji`
 ### Remove a reaction to a post with a unicode emoji
 * Method: `DELETE`
 * Authentication: required
-* Params: `emoji`: A single character unicode emoji
+* Params: `emoji`: A unicode RGI emoji or a regional indicator
 * Response: JSON, the status.
 
 ## `GET /api/v1/pleroma/statuses/:id/reactions`
@@ -615,3 +615,41 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
   {"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]}
 ]
 ```
+
+## `POST /api/v1/pleroma/backups`
+### Create a user backup archive
+
+* Method: `POST`
+* Authentication: required
+* Params: none
+* Response: JSON
+* Example response:
+
+```json
+[{
+    "content_type": "application/zip",
+    "file_size": 0,
+    "inserted_at": "2020-09-10T16:18:03.000Z",
+    "processed": false,
+    "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
+}]
+```
+
+## `GET /api/v1/pleroma/backups`
+### Lists user backups
+
+* Method: `GET`
+* Authentication: not required
+* Params: none
+* Response: JSON
+* Example response:
+
+```json
+[{
+    "content_type": "application/zip",
+    "file_size": 55457,
+    "inserted_at": "2020-09-10T16:18:03.000Z",
+    "processed": true,
+    "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
+}]
+```

docs/API/prometheus.md

@@ -2,15 +2,37 @@
 
 Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library.
 
+Config example:
+
+```
+config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
+  enabled: true,
+  auth: {:basic, "myusername", "mypassword"},
+  ip_whitelist: ["127.0.0.1"],
+  path: "/api/pleroma/app_metrics",
+  format: :text
+```
+
+* `enabled` (Pleroma extension) enables the endpoint
+* `ip_whitelist` (Pleroma extension) could be used to restrict access only to specified IPs
+* `auth` sets the authentication (`false` for no auth; configurable to HTTP Basic Auth, see [prometheus-plugs](https://github.com/deadtrickster/prometheus-plugs#exporting) documentation)
+* `format` sets the output format (`:text` or `:protobuf`)
+* `path` sets the path to app metrics page 
+
+
 ## `/api/pleroma/app_metrics`
+
 ### Exports Prometheus application metrics
+
 * Method: `GET`
-* Authentication: not required
+* Authentication: not required by default (see configuration options above)
 * Params: none
-* Response: JSON
+* Response: text
 
 ## Grafana
+
 ### Config example
+
 The following is a config example to use with [Grafana](https://grafana.com)
 
 ```

docs/administration/CLI_tasks/config.md

@@ -32,7 +32,7 @@
     config :pleroma, configurable_from_database: false
     ```
 
-To delete transfered settings from database optional flag `-d` can be used. `<env>` is `prod` by default.
+To delete transferred settings from database optional flag `-d` can be used. `<env>` is `prod` by default.
 
 === "OTP"
     ```sh
@@ -43,3 +43,111 @@ To delete transfered settings from database optional flag `-d` can be used. `<en
     ```sh
     mix pleroma.config migrate_from_db [--env=<env>] [-d]
     ```
+
+## Dump all of the config settings defined in the database
+
+=== "OTP"
+
+    ```sh
+     ./bin/pleroma_ctl config dump
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.config dump
+    ```
+
+## List individual configuration groups in the database
+
+=== "OTP"
+
+    ```sh
+     ./bin/pleroma_ctl config groups
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.config groups
+    ```
+
+## Dump the saved configuration values for a specific group or key
+
+e.g., this shows all the settings under `config :pleroma`
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl config dump pleroma
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.config dump pleroma
+    ```
+
+To get values under a specific key:
+
+e.g., this shows all the settings under `config :pleroma, :instance`
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl config dump pleroma instance
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.config dump pleroma instance
+    ```
+
+## Delete the saved configuration values for a specific group or key
+
+e.g., this deletes all the settings under `config :tesla`
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl config delete [--force] tesla
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.config delete [--force] tesla
+    ```
+
+To delete values under a specific key:
+
+e.g., this deletes all the settings under `config :phoenix, :stacktrace_depth`
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl config delete [--force] phoenix stacktrace_depth
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.config delete [--force] phoenix stacktrace_depth
+    ```
+
+## Remove all settings from the database
+
+This forcibly removes all saved values in the database.
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl config [--force] reset
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.config [--force] reset
+    ```

docs/administration/CLI_tasks/email.md

@@ -16,7 +16,6 @@
     mix pleroma.email test [--to <destination email address>]
     ```
 
-
 Example:
 
 === "OTP"
@@ -36,11 +35,11 @@ Example:
 === "OTP"
 
     ```sh
-     ./bin/pleroma_ctl email send_confirmation_mails
+     ./bin/pleroma_ctl email resend_confirmation_emails
     ```
 
 === "From Source"
 
     ```sh
-    mix pleroma.email send_confirmation_mails
+    mix pleroma.email resend_confirmation_emails
     ```

docs/administration/CLI_tasks/frontend.md

@@ -1,12 +1,23 @@
 # Managing frontends
 
-`mix pleroma.frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]`
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]
+    ```
 
 Frontend can be installed either from local zip file, or automatically downloaded from the web.
 
-You can give all the options directly on the command like, but missing information will be filled out by looking at the data configured under `frontends.available` in the config files.
+You can give all the options directly on the command line, but missing information will be filled out by looking at the data configured under `frontends.available` in the config files.
+
+Currently, known `<frontend>` values are:
 
-Currently known `<frontend>` values are:
 - [admin-fe](https://git.pleroma.social/pleroma/admin-fe)
 - [kenoma](http://git.pleroma.social/lambadalambda/kenoma)
 - [pleroma-fe](http://git.pleroma.social/pleroma/pleroma-fe)
@@ -19,51 +30,67 @@ You can still install frontends that are not configured, see below.
 
 For a frontend configured under the `available` key, it's enough to install it by name.
 
-```sh tab="OTP"
+=== "OTP"
+
+    ```sh
     ./bin/pleroma_ctl frontend install pleroma
     ```
 
-```sh tab="From Source"
+=== "From Source"
+
+    ```sh
     mix pleroma.frontend install pleroma
     ```
 
-This will download the latest build for the the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`).
+This will download the latest build for the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`).
 
-You can override any of the details. To install a pleroma build from a different url, you could do this:
+You can override any of the details. To install a pleroma build from a different URL, you could do this:
 
-```sh tab="OPT"
+=== "OTP"
+
+    ```sh
     ./bin/pleroma_ctl frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
     ```
 
-```sh tab="From Source"
+=== "From Source"
+
+    ```sh
     mix pleroma.frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
     ```
 
 Similarly, you can also install from a local zip file.
 
-```sh tab="OTP"
+=== "OTP"
+
+    ```sh
     ./bin/pleroma_ctl frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
     ```
 
-```sh tab="From Source"
+=== "From Source"
+
+    ```sh
     mix pleroma.frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
     ```
 
-The resulting frontend will always be installed into a folder of this template: `${instance_static}/frontends/${name}/${ref}`
+The resulting frontend will always be installed into a folder of this template: `${instance_static}/frontends/${name}/${ref}`.
 
-Careful: This folder will be completely replaced on installation
+Careful: This folder will be completely replaced on installation.
 
 ## Example installation for an unknown frontend
 
-The installation process is the same, but you will have to give all the needed options on the commond line. For example:
+The installation process is the same, but you will have to give all the needed options on the command line. For example:
 
-```sh tab="OTP"
+=== "OTP"
+
+    ```sh
     ./bin/pleroma_ctl frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
     ```
 
-```sh tab="From Source"
+=== "From Source"
+
+    ```sh
     mix pleroma.frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
     ```
 
-If you don't have a zip file but just want to install a frontend from a local path, you can simply copy the files over a folder of this template: `${instance_static}/frontends/${name}/${ref}`
+If you don't have a zip file but just want to install a frontend from a local path, you can simply copy the files over a folder of this template: `${instance_static}/frontends/${name}/${ref}`.
 

docs/administration/CLI_tasks/instance.md

@@ -40,3 +40,5 @@ If any of the options are left unspecified, you will be prompted interactively.
 - `--strip-uploads <Y|N>` - use ExifTool to strip uploads of sensitive location data
 - `--anonymize-uploads <Y|N>` - randomize uploaded filenames
 - `--dedupe-uploads <Y|N>` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames
+- `--skip-release-env` - skip generation the release environment file
+- `--release-env-file` - release environment file path

docs/administration/CLI_tasks/user.md

@@ -264,13 +264,13 @@
 === "OTP"
 
     ```sh
-     ./bin/pleroma_ctl user toggle_confirmed <nickname>
+     ./bin/pleroma_ctl user confirm <nickname>
     ```
 
 === "From Source"
 
     ```sh
-    mix pleroma.user toggle_confirmed <nickname>
+    mix pleroma.user confirm <nickname>
     ```
 
 ## Set confirmation status for all regular active users

docs/ap_extensions.md

@@ -1,11 +1,41 @@
-# ChatMessages
+# AP Extensions
+## Actor endpoints
 
-ChatMessages are the messages sent in 1-on-1 chats. They are similar to
+The following endpoints are additionally present into our actors.
+
+- `oauthRegistrationEndpoint` (`http://litepub.social/ns#oauthRegistrationEndpoint`)
+- `uploadMedia` (`https://www.w3.org/ns/activitystreams#uploadMedia`)
+
+### oauthRegistrationEndpoint
+
+Points to MastodonAPI `/api/v1/apps` for now.
+
+See <https://docs.joinmastodon.org/methods/apps/>
+
+### uploadMedia
+
+Inspired by <https://www.w3.org/wiki/SocialCG/ActivityPub/MediaUpload>, it is part of the ActivityStreams namespace because it used to be part of the ActivityPub specification and got removed from it.
+
+Content-Type: multipart/form-data
+
+Parameters:
+- (required) `file`: The file being uploaded
+- (optionnal) `description`: A plain-text description of the media, for accessibility purposes.
+
+Response: HTTP 201 Created with the object into the body, no `Location` header provided as it doesn't have an `id`
+
+The object given in the reponse should then be inserted into an Object's `attachment` field.
+
+## ChatMessages
+
+`ChatMessage`s are the messages sent in 1-on-1 chats. They are similar to
 `Note`s, but the addresing is done by having a single AP actor in the `to`
 field. Addressing multiple actors is not allowed. These messages are always
 private, there is no public version of them. They are created with a `Create`
 activity.
 
+They are part of the `litepub` namespace as `http://litepub.social/ns#ChatMessage`.
+
 Example:
 
 ```json

docs/clients.md

@@ -7,97 +7,105 @@ Feel free to contact us to be added to this list!
 - Homepage: <https://www.pleroma.com/#desktopApp>
 - Source Code: <https://github.com/roma-apps/roma-desktop>
 - Platforms: Windows, Mac, Linux
-- Features: Streaming Ready
+- Features: MastoAPI, Streaming Ready
 
 ### Social
 - Source Code: <https://gitlab.gnome.org/World/Social>
 - Contact: [@brainblasted@social.libre.fi](https://social.libre.fi/users/brainblasted)
 - Platforms: Linux (GNOME)
 - Note(2019-01-28): Not at a pre-alpha stage yet
+- Features: MastoAPI
 
 ### Whalebird
 - Homepage: <https://whalebird.org/>
 - Source Code: <https://github.com/h3poteto/whalebird-desktop>
 - Contact: [@h3poteto@pleroma.io](https://pleroma.io/users/h3poteto)
 - Platforms: Windows, Mac, Linux
-- Features: Streaming Ready
+- Features: MastoAPI, Streaming Ready
 
 ## Handheld
+### AndStatus
+- Homepage: <http://andstatus.org/>
+- Source Code: <https://github.com/andstatus/andstatus/>
+- Platforms: Android
+- Features: MastoAPI, ActivityPub (Client-to-Server)
+
 ### Amaroq
 - Homepage: <https://itunes.apple.com/us/app/amaroq-for-mastodon/id1214116200>
 - Source Code: <https://github.com/ReticentJohn/Amaroq>
 - Contact: [@eurasierboy@mastodon.social](https://mastodon.social/users/eurasierboy)
 - Platforms: iOS
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Fedilab
 - Homepage: <https://fedilab.app/>
 - Source Code: <https://framagit.org/tom79/fedilab/>
 - Contact: [@fedilab@framapiaf.org](https://framapiaf.org/users/fedilab)
 - Platforms: Android
-- Features: Streaming Ready, Moderation, Text Formatting
+- Features: MastoAPI, Streaming Ready, Moderation, Text Formatting
 
 ### Kyclos
 - Source Code: <https://git.pleroma.social/pleroma/harbour-kyclos>
 - Platforms: SailfishOS
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Husky
 - Source code: <https://git.mentality.rip/FWGS/Husky>
 - Contact: [@Husky@enigmatic.observer](https://enigmatic.observer/users/Husky)
 - Platforms: Android
-- Features: No Streaming, Emoji Reactions, Text Formatting, FE Stickers
+- Features: MastoAPI, No Streaming, Emoji Reactions, Text Formatting, FE Stickers
 
 ### Fedi
 - Homepage: <https://www.fediapp.com/>
 - Source Code: Proprietary, but gratis
 - Platforms: iOS, Android
-- Features: Pleroma-specific features like Reactions
+- Features: MastoAPI, Pleroma-specific features like Reactions
 
 ### Tusky
 - Homepage: <https://tuskyapp.github.io/>
 - Source Code: <https://github.com/tuskyapp/Tusky>
 - Contact: [@ConnyDuck@mastodon.social](https://mastodon.social/users/ConnyDuck)
 - Platforms: Android
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Twidere
 - Homepage: <https://twidere.mariotaku.org/>
 - Source Code: <https://github.com/TwidereProject/Twidere-Android/>
 - Contact: <me@mariotaku.org>
 - Platform: Android
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Indigenous
 - Homepage: <https://indigenous.realize.be/>
 - Source Code: <https://github.com/swentel/indigenous-android/>
-- Contact: [@realize.be@realize.be](@realize.be@realize.be)
+- Contact: [@swentel@realize.be](https://realize.be)
 - Platforms: Android
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ## Alternative Web Interfaces
 ### Brutaldon
 - Homepage: <https://jfm.carcosa.net/projects/software/brutaldon/>
 - Source Code: <https://git.carcosa.net/jmcbray/brutaldon>
 - Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc)
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Halcyon
 - Source Code: <https://notabug.org/halcyon-suite/halcyon>
 - Contact: [@halcyon@social.csswg.org](https://social.csswg.org/users/halcyon)
-- Features: Streaming Ready
+- Features: MastoAPI, Streaming Ready
 
 ### Pinafore
 - Homepage: <https://pinafore.social/>
 - Source Code: <https://github.com/nolanlawson/pinafore>
 - Contact: [@pinafore@mastodon.technology](https://mastodon.technology/users/pinafore)
 - Note: Pleroma support is a secondary goal
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Sengi
 - Homepage: <https://nicolasconstant.github.io/sengi/>
 - Source Code: <https://github.com/NicolasConstant/sengi>
 - Contact: [@sengi_app@mastodon.social](https://mastodon.social/users/sengi_app)
+- Features: MastoAPI
 
 ### DashFE
 - Source Code: <https://notabug.org/daisuke/DashboardFE>
@@ -107,3 +115,4 @@ Feel free to contact us to be added to this list!
 - Source Code: <https://git.freesoftwareextremist.com/bloat/>
 - Contact: [@r@freesoftwareextremist.com](https://freesoftwareextremist.com/users/r)
 - Features: Does not requires JavaScript
+- Features: MastoAPI

docs/configuration/cheatsheet.md

@@ -45,6 +45,7 @@ To add configuration to your config file, you can copy it from the base config.
     older software for theses nicknames.
 * `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature.
 * `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow.
+* `autofollowing_nicknames`: Set to nicknames of (local) users that automatically follows every newly registered user.
 * `attachment_links`: Set to true to enable automatically adding attachment link text to statuses.
 * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`).
 * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). Default: `false`.
@@ -62,6 +63,7 @@ To add configuration to your config file, you can copy it from the base config.
 * `external_user_synchronization`: Enabling following/followers counters synchronization for external users.
 * `cleanup_attachments`: Remove attachments along with statuses. Does not affect duplicate files and attachments without status. Enabling this will increase load to database when deleting statuses on larger instances.
 * `show_reactions`: Let favourites and emoji reactions be viewed through the API (default: `true`).
+* `password_reset_token_validity`: The time after which reset tokens aren't accepted anymore, in seconds (default: one day).
 
 ## Welcome
 * `direct_message`: - welcome message sent as a direct message.
@@ -219,18 +221,6 @@ config :pleroma, :mrf_user_allowlist, %{
 * `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`)
 * `enabled`: whether scheduled activities are sent to the job queue to be executed
 
-## FedSockets
-FedSockets is an experimental feature allowing for Pleroma backends to federate using a persistant websocket connection as opposed to making each federation a seperate http connection. This feature is currently off by default. It is configurable throught he following options.
-
-### :fedsockets
-* `enabled`: Enables FedSockets for this instance. `false` by default.
-* `connection_duration`: Time an idle websocket is kept open.
-* `rejection_duration`: Failures to connect via FedSockets will not be retried for this period of time.
-* `fed_socket_fetches` and `fed_socket_rejections`: Settings passed to `cachex` for the fetch registry, and rejection stacks. See `Pleroma.Web.FedSockets` for more details.
-
-
-## Frontends
-
 ### :frontend_configurations
 
 This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` and `masto_fe` are configured. You can find the documentation for `pleroma_fe` configuration into [Pleroma-FE configuration and customization for instance administrators](/frontend/CONFIGURATION/#options).
@@ -1077,6 +1067,20 @@ Control favicons for instances.
 
 * `enabled`: Allow/disallow displaying and getting instances favicons
 
+## Pleroma.User.Backup
+
+!!! note
+    Requires enabled email
+
+* `:purge_after_days` an integer, remove backup achives after N days.
+* `:limit_days` an integer, limit user to export not more often than once per N days.
+* `:dir` a string with a path to backup temporary directory or `nil` to let Pleroma choose temporary directory in the following order:
+    1. the directory named by the TMPDIR environment variable
+    2. the directory named by the TEMP environment variable
+    3. the directory named by the TMP environment variable
+    4. C:\TMP on Windows or /tmp on Unix-like operating systems
+    5. as a last resort, the current working directory
+
 ## Frontend management
 
 Frontends in Pleroma are swappable - you can specify which one to use here.

docs/configuration/howto_database_config.md

@@ -5,7 +5,7 @@ The configuration of Pleroma has traditionally been managed with a config file,
 
 ## Migration to database config
 
-1. Run the mix task to migrate to the database. You'll receive some debugging output and a few messages informing you of what happened.
+1. Run the mix task to migrate to the database.
 
   **Source:**
 
@@ -24,21 +24,8 @@ The configuration of Pleroma has traditionally been managed with a config file,
   ```
 
   ```
-   10:04:34.155 [debug] QUERY OK source="config" db=1.6ms decode=2.0ms queue=33.5ms idle=0.0ms
-    SELECT c0."id", c0."key", c0."group", c0."value", c0."inserted_at", c0."updated_at" FROM "config" AS c0 []
     Migrating settings from file: /home/pleroma/config/dev.secret.exs
 
-   10:04:34.240 [debug] QUERY OK db=4.5ms queue=0.3ms idle=92.2ms
-    TRUNCATE config; []
-  
-   10:04:34.244 [debug] QUERY OK db=2.8ms queue=0.3ms idle=97.2ms
-    ALTER SEQUENCE config_id_seq RESTART; []
-    
-   10:04:34.256 [debug] QUERY OK source="config" db=0.8ms queue=1.4ms idle=109.8ms
-   SELECT c0."id", c0."key", c0."group", c0."value", c0."inserted_at", c0."updated_at" FROM "config" AS c0 WHERE ((c0."group" = $1) AND (c0."key" = $2)) [":pleroma", ":instance"]
-    
-   10:04:34.292 [debug] QUERY OK db=2.6ms queue=1.7ms idle=137.7ms
-   INSERT INTO "config" ("group","key","value","inserted_at","updated_at") VALUES ($1,$2,$3,$4,$5) RETURNING "id" [":pleroma", ":instance", <<131, 108, 0, 0, 0, 1, 104, 2, 100, 0, 4, 110, 97, 109, 101, 109, 0, 0, 0, 7, 66, 108, 101, 114, 111, 109, 97, 106>>, ~N[2020-07-12 15:04:34], ~N[2020-07-12 15:04:34]]
    Settings for key instance migrated.
    Settings for group :pleroma migrated.
   ```
@@ -124,12 +111,20 @@ The configuration of Pleroma has traditionally been managed with a config file,
 ## Debugging
 
 ### Clearing database config
-You can clear the database config by truncating the `config` table in the database. e.g.,
+You can clear the database config with the following command:
+
+  **Source:**
 
   ```
-psql -d pleroma_dev
+  $ mix pleroma.config reset
-pleroma_dev=# TRUNCATE config;
+  ```
-TRUNCATE TABLE
+
+  or
+
+  **OTP:**
+
+  ```
+  $ ./bin/pleroma_ctl config reset
   ```
 
 Additionally, every time you migrate the configuration to the database the config table is automatically truncated to ensure a clean migration.
@@ -137,17 +132,24 @@ Additionally, every time you migrate the configuration to the database the confi
 ### Manually removing a setting
 If you encounter a situation where the server cannot run properly because of an invalid setting in the database and this is preventing you from accessing AdminFE, you can manually remove the offending setting if you know which one it is.
 
-e.g., here is an example showing a minimal configuration in the database. Only the `config :pleroma, :instance` settings are in the table:
+e.g., here is an example showing a the removal of the `config :pleroma, :instance` settings:
+
+  **Source:**
 
   ```
-psql -d pleroma_dev
+  $ mix pleroma.config delete pleroma instance
-pleroma_dev=# select * from config;
+  Are you sure you want to continue? [n]  y
- id |    key    |                           value                            |     inserted_at     |     updated_at      |  group
+  config :pleroma, :instance deleted from the ConfigDB.
-----+-----------+------------------------------------------------------------+---------------------+---------------------+----------
+  ```
-  1 | :instance | \x836c0000000168026400046e616d656d00000007426c65726f6d616a | 2020-07-12 15:33:29 | 2020-07-12 15:33:29 | :pleroma
+
-(1 row)
+  or
-pleroma_dev=# delete from config where key = ':instance' and group = ':pleroma';
+
-DELETE 1
+  **OTP:**
+
+  ```
+  $ ./bin/pleroma_ctl config delete pleroma instance
+  Are you sure you want to continue? [n]  y
+  config :pleroma, :instance deleted from the ConfigDB.
   ```
 
 Now the `config :pleroma, :instance` settings have been removed from the database.

docs/configuration/howto_ejabberd.md

@@ -0,0 +1,136 @@
+# Configuring Ejabberd (XMPP Server) to use Pleroma for authentication
+
+If you want to give your Pleroma users an XMPP (chat) account, you can configure [Ejabberd](https://github.com/processone/ejabberd) to use your Pleroma server for user authentication, automatically giving every local user an XMPP account.
+
+In general, you just have to follow the configuration described at [https://docs.ejabberd.im/admin/configuration/authentication/#external-script](https://docs.ejabberd.im/admin/configuration/authentication/#external-script). Please read this section carefully. 
+
+Copy the script below to suitable path on your system and set owner and permissions. Also do not forget adjusting `PLEROMA_HOST` and `PLEROMA_PORT`, if necessary.
+
+```bash
+cp pleroma_ejabberd_auth.py /etc/ejabberd/pleroma_ejabberd_auth.py
+chown ejabberd /etc/ejabberd/pleroma_ejabberd_auth.py
+chmod 700 /etc/ejabberd/pleroma_ejabberd_auth.py
+```
+
+Set external auth params in ejabberd.yaml file:
+
+```bash
+auth_method: [external]
+extauth_program: "python3 /etc/ejabberd/pleroma_ejabberd_auth.py"
+extauth_instances: 3
+auth_use_cache: false
+```
+
+Restart / reload your ejabberd service.
+
+After restarting your Ejabberd server, your users should now be able to connect with their Pleroma credentials.
+
+
+```python
+import sys
+import struct
+import http.client
+from base64 import b64encode
+import logging
+
+
+PLEROMA_HOST = "127.0.0.1"
+PLEROMA_PORT = "4000"
+AUTH_ENDPOINT = "/api/v1/accounts/verify_credentials"
+USER_ENDPOINT = "/api/v1/accounts"
+LOGFILE = "/var/log/ejabberd/pleroma_auth.log"
+
+logging.basicConfig(filename=LOGFILE, level=logging.INFO)
+
+
+# Pleroma functions
+def create_connection():
+    return http.client.HTTPConnection(PLEROMA_HOST, PLEROMA_PORT)
+
+
+def verify_credentials(user: str, password: str) -> bool:
+    user_pass_b64 = b64encode("{}:{}".format(
+        user, password).encode('utf-8')).decode("ascii")
+    params = {}
+    headers = {
+        "Authorization": "Basic {}".format(user_pass_b64)
+    }
+
+    try:
+        conn = create_connection()
+        conn.request("GET", AUTH_ENDPOINT, params, headers)
+
+        response = conn.getresponse()
+        if response.status == 200:
+            return True
+
+        return False
+    except Exception as e:
+        logging.info("Can not connect: %s", str(e))
+        return False
+
+
+def does_user_exist(user: str) -> bool:
+    conn = create_connection()
+    conn.request("GET", "{}/{}".format(USER_ENDPOINT, user))
+
+    response = conn.getresponse()
+    if response.status == 200:
+        return True
+
+    return False
+
+
+def auth(username: str, server: str, password: str) -> bool:
+    return verify_credentials(username, password)
+
+
+def isuser(username, server):
+    return does_user_exist(username)
+
+
+def read():
+    (pkt_size,) = struct.unpack('>H', bytes(sys.stdin.read(2), encoding='utf8'))
+    pkt = sys.stdin.read(pkt_size)
+    cmd = pkt.split(':')[0]
+    if cmd == 'auth':
+        username, server, password = pkt.split(':', 3)[1:]
+        write(auth(username, server, password))
+    elif cmd == 'isuser':
+        username, server = pkt.split(':', 2)[1:]
+        write(isuser(username, server))
+    elif cmd == 'setpass':
+        # u, s, p = pkt.split(':', 3)[1:]
+        write(False)
+    elif cmd == 'tryregister':
+        # u, s, p = pkt.split(':', 3)[1:]
+        write(False)
+    elif cmd == 'removeuser':
+        # u, s = pkt.split(':', 2)[1:]
+        write(False)
+    elif cmd == 'removeuser3':
+        # u, s, p = pkt.split(':', 3)[1:]
+        write(False)
+    else:
+        write(False)
+
+
+def write(result):
+    if result:
+        sys.stdout.write('\x00\x02\x00\x01')
+    else:
+        sys.stdout.write('\x00\x02\x00\x00')
+    sys.stdout.flush()
+
+
+if __name__ == "__main__":
+    logging.info("Starting pleroma ejabberd auth daemon...")
+    while True:
+        try:
+            read()
+        except Exception as e:
+            logging.info(
+                "Error while processing data from ejabberd %s", str(e))
+            pass
+
+```

docs/configuration/optimizing_beam.md

@@ -0,0 +1,66 @@
+# Optimizing the BEAM
+
+Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly optimized for latency, but this has drawbacks in environments without dedicated hardware. One of the tricks used by the BEAM VM is [busy waiting](https://en.wikipedia.org/wiki/Busy_waiting). This allows the application to pretend to be busy working so the OS kernel does not pause the application process and switch to another process waiting for the CPU to execute its workload. It does this by spinning for a period of time which inflates the apparent CPU usage of the application so it is immediately ready to execute another task. This can be observed with utilities like **top(1)** which will show consistently high CPU usage for the process. Switching between procesess is a rather expensive operation and also clears CPU caches further affecting latency and performance. The goal of busy waiting is to avoid this penalty.
+
+This strategy is very successful in making a performant and responsive application, but is not desirable on Virtual Machines or hardware with few CPU cores. Pleroma instances are often deployed on the same server as the required PostgreSQL database which can lead to situations where the Pleroma application is holding the CPU in a busy-wait loop and as a result the database cannot process requests in a timely manner. The fewer CPUs available, the more this problem is exacerbated. The latency is further amplified by the OS being installed on a Virtual Machine as the Hypervisor uses CPU time-slicing to pause the entire OS and switch between other tasks.
+
+More adventurous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process creating latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security.
+
+Please only change these settings if you are experiencing issues or really know what you are doing. In general, there's no need to change these settings.
+
+## VPS Provider Recommendations
+
+### Good
+
+* Hetzner Cloud
+
+### Bad
+
+* AWS (known to use burst scheduling)
+
+
+## Example configurations
+
+Tuning the BEAM requires you provide a config file normally called [vm.args](http://erlang.org/doc/man/erl.html#emulator-flags). If you are using systemd to manage the service you can modify the unit file as such:
+
+`ExecStart=/usr/bin/elixir --erl '-args_file /opt/pleroma/config/vm.args' -S /usr/bin/mix phx.server`
+
+Check your OS documentation to adopt a similar strategy on other platforms.
+
+### Virtual Machine and/or few CPU cores
+
+Disable the busy-waiting. This should generally only be done if you're on a platform that does burst scheduling, like AWS.
+
+**vm.args:**
+
+```
++sbwt none
++sbwtdcpu none
++sbwtdio none
+```
+
+### Dedicated Hardware
+
+Enable more busy waiting, increase the internal maximum limit of BEAM processes and ports. You can use this if you run on dedicated hardware, but it is not necessary.
+
+**vm.args:**
+
+```
++P 16777216
++Q 16777216
++K true
++A 128
++sbt db
++sbwt very_long
++swt very_low
++sub true
++Mulmbcs 32767
++Mumbcgs 1
++Musmbcs 2047
+```
+
+## Additional Reading
+
+* [WhatsApp: Scaling to Millions of Simultaneous Connections](https://www.erlang-factory.com/upload/presentations/558/efsf2012-whatsapp-scaling.pdf)
+* [Preemptive Scheduling and Spinlocks](https://www.uio.no/studier/emner/matnat/ifi/nedlagte-emner/INF3150/h03/annet/slides/preemptive.pdf)
+* [The Curious Case of BEAM CPU Usage](https://stressgrid.com/blog/beam_cpu_usage/)

docs/configuration/static_dir.md

@@ -88,3 +88,8 @@ config :pleroma, :frontend_configurations,
     Note the extra `static` folder for the terms-of-service.html
 
 Terms of Service will be shown to all users on the registration page. It's the best place where to write down the rules for your instance. You can modify the rules by adding and changing `$static_dir/static/terms-of-service.html`.
+
+ 	
+## Styling rendered pages
+
+To overwrite the CSS stylesheet of the OAuth form and other static pages, you can upload your own CSS file to `instance/static/static.css`. This will completely replace the CSS used by those pages, so it might be a good idea to copy the one from `priv/static/instance/static.css` and make your changes.

docs/dev.md

@@ -14,10 +14,33 @@ This document contains notes and guidelines for Pleroma developers.
 
   For `:api` pipeline routes, it'll be verified whether `OAuthScopesPlug` was called or explicitly skipped, and if it was not then auth information will be dropped for request. Then `EnsurePublicOrAuthenticatedPlug` will be called to ensure that either the instance is not private or user is authenticated (unless explicitly skipped). Such automated checks help to prevent human errors and result in higher security / privacy for users.
 
-## [HTTP Basic Authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization)
+## Non-OAuth authentication
 
-* With HTTP Basic Auth, OAuth scopes check is _not_ performed for any action (since password is provided during the auth, requester is able to obtain a token with full permissions anyways). `Pleroma.Web.Plugs.AuthenticationPlug` and `Pleroma.Web.Plugs.LegacyAuthenticationPlug` both call `Pleroma.Web.Plugs.OAuthScopesPlug.skip_plug(conn)` when password is provided.
+* With non-OAuth authentication ([HTTP Basic Authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization) or HTTP header- or params-provided auth), OAuth scopes check is _not_ performed for any action (since password is provided during the auth, requester is able to obtain a token with full permissions anyways); auth plugs invoke `Pleroma.Helpers.AuthHelper.skip_oauth(conn)` in this case.
 
 ## Auth-related configuration, OAuth consumer mode etc.
 
 See `Authentication` section of [the configuration cheatsheet](configuration/cheatsheet.md#authentication).
+
+## MRF policies descriptions
+
+If MRF policy depends on config, it can be added into MRF tab to adminFE by adding `config_description/0` method, which returns map with special structure.
+
+Example:
+
+```elixir
+%{
+      key: :mrf_activity_expiration,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy",
+      label: "MRF Activity Expiration Policy",
+      description: "Adds automatic expiration to all local activities",
+      children: [
+        %{
+          key: :days,
+          type: :integer,
+          description: "Default global expiration time for all local activities (in days)",
+          suggestions: [90, 365]
+        }
+      ]
+    }
+```

docs/installation/debian_based_en.md

@@ -35,7 +35,7 @@ sudo apt full-upgrade
 * Install some of the above mentioned programs:
 
 ```shell
-sudo apt install git build-essential postgresql postgresql-contrib cmake libmagic-devel
+sudo apt install git build-essential postgresql postgresql-contrib cmake libmagic-dev
 ```
 
 ### Install Elixir and Erlang
@@ -101,6 +101,7 @@ sudo -Hu pleroma mix deps.get
 mv config/{generated_config.exs,prod.secret.exs}
 ```
 
+
 * The previous command creates also the file `config/setup_db.psql`, with which you can create the database:
 
 ```shell

docs/installation/otp_en.md

@@ -43,7 +43,7 @@ Other than things bundled in the OTP release Pleroma depends on:
 
 ### Installing optional packages
 
-Per [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md):
+Per [`docs/installation/optional/media_graphics_packages.md`](optional/media_graphics_packages.md):
   * ImageMagick
   * ffmpeg
   * exiftool
@@ -159,7 +159,7 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
 # su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
 
 # Start the instance to verify that everything is working as expected
-su pleroma -s $SHELL -lc "./bin/pleroma daemon"
+su pleroma -s $SHELL -lc "export $(cat /opt/pleroma/config/pleroma.env); ./bin/pleroma daemon"
 
 # Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly
 sleep 20 && curl http://localhost:4000/api/v1/instance
@@ -311,4 +311,3 @@ This will create an account withe the username of 'joeuser' with the email addre
 ## Questions
 
 Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
-

installation/download-mastofe-build.sh

@@ -9,29 +9,32 @@ static_dir="instance/static"
 # project_branch="pleroma"
 # static_dir="priv/static"
 
-if [[ ! -d "${static_dir}" ]]
+if [ ! -d "${static_dir}" ]
 then
 	echo "Error: ${static_dir} directory is missing, are you sure you are running this script at the root of pleroma’s repository?"
 	exit 1
 fi
 
-last_modified="$(curl -s -I 'https://git.pleroma.social/api/v4/projects/'${project_id}'/jobs/artifacts/'${project_branch}'/download?job=build' | grep '^Last-Modified:' | cut -d: -f2-)"
+last_modified="$(curl --fail -s -I 'https://git.pleroma.social/api/v4/projects/'${project_id}'/jobs/artifacts/'${project_branch}'/download?job=build' | grep '^Last-Modified:' | cut -d: -f2-)"
 
 echo "branch:${project_branch}"
 echo "Last-Modified:${last_modified}"
 
 artifact="mastofe.zip"
 
-if [[ -e mastofe.timestamp ]] && [[ "${last_modified}" != "" ]]
+if [ "${last_modified}x" = "x" ]
 then
-	if [[ "$(cat mastofe.timestamp)" == "${last_modified}" ]]
+	echo "ERROR: Couldn't get the modification date of the latest build archive, maybe it expired, exiting..."
-	then
+	exit 1
-		echo "MastoFE is up-to-date, exiting…"
-		exit 0
-	fi
 fi
 
-curl -c - "https://git.pleroma.social/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=build" -o "${artifact}" || exit
+if [ -e mastofe.timestamp ] && [ "$(cat mastofe.timestamp)" = "${last_modified}" ]
+then
+	echo "MastoFE is up-to-date, exiting..."
+	exit 0
+fi
+
+curl --fail -c - "https://git.pleroma.social/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=build" -o "${artifact}" || exit
 
 # TODO: Update the emoji as well
 rm -fr "${static_dir}/sw.js" "${static_dir}/packs" || exit

installation/pleroma.nginx

@@ -93,9 +93,4 @@ server {
         chunked_transfer_encoding on;
         proxy_pass         http://phoenix;
     }
-
-    location /api/fedsocket/v1 {
-        proxy_request_buffering off;
-        proxy_pass http://phoenix/api/fedsocket/v1;
-    }
 }

installation/pleroma.service

@@ -29,8 +29,6 @@ ProtectHome=true
 ProtectSystem=full
 ; Sets up a new /dev mount for the process and only adds API pseudo devices like /dev/null, /dev/zero or /dev/random but not physical devices. Disabled by default because it may not work on devices like the Raspberry Pi.
 PrivateDevices=false
-; Ensures that the service process and all its children can never gain new privileges through execve().
-NoNewPrivileges=true
 ; Drops the sysadmin capability from the daemon.
 CapabilityBoundingSet=~CAP_SYS_ADMIN
 

lib/mix/pleroma.ex

@@ -12,17 +12,19 @@ defmodule Mix.Pleroma do
     :cachex,
     :flake_id,
     :swoosh,
-    :timex
+    :timex,
+    :fast_html
   ]
-  @cachex_children ["object", "user", "scrubber"]
+  @cachex_children ["object", "user", "scrubber", "web_resp"]
   @doc "Common functions to be reused in mix tasks"
   def start_pleroma do
     Pleroma.Config.Holder.save_default()
     Pleroma.Config.Oban.warn()
+    Pleroma.Application.limiters_setup()
     Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
 
-    if Pleroma.Config.get(:env) != :test do
+    unless System.get_env("DEBUG") do
-      Application.put_env(:logger, :console, level: :debug)
+      Logger.remove_backend(:console)
     end
 
     adapter = Application.get_env(:tesla, :adapter)
@@ -36,12 +38,23 @@ defmodule Mix.Pleroma do
 
     Enum.each(apps, &Application.ensure_all_started/1)
 
+    oban_config = [
+      crontab: [],
+      repo: Pleroma.Repo,
+      log: false,
+      queues: [],
+      plugins: []
+    ]
+
     children =
       [
         Pleroma.Repo,
+        Pleroma.Emoji,
         {Pleroma.Config.TransferTask, false},
         Pleroma.Web.Endpoint,
-        {Oban, Pleroma.Config.get(Oban)}
+        {Oban, oban_config},
+        {Majic.Pool,
+         [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]}
       ] ++
         http_children(adapter)
 
@@ -97,12 +110,6 @@ defmodule Mix.Pleroma do
     end
   end
 
-  def shell_yes?(message) do
-    if mix_shell?(),
-      do: Mix.shell().yes?("Continue?"),
-      else: shell_prompt(message, "Continue?") in ~w(Yn Y y)
-  end
-
   def shell_info(message) do
     if mix_shell?(),
       do: Mix.shell().info(message),

lib/mix/tasks/pleroma/config.ex

@@ -5,6 +5,7 @@
 defmodule Mix.Tasks.Pleroma.Config do
   use Mix.Task
 
+  import Ecto.Query
   import Mix.Pleroma
 
   alias Pleroma.ConfigDB
@@ -14,11 +15,14 @@ defmodule Mix.Tasks.Pleroma.Config do
   @moduledoc File.read!("docs/administration/CLI_tasks/config.md")
 
   def run(["migrate_to_db"]) do
+    check_configdb(fn ->
       start_pleroma()
       migrate_to_db()
+    end)
   end
 
   def run(["migrate_from_db" | options]) do
+    check_configdb(fn ->
       start_pleroma()
 
       {opts, _} =
@@ -28,12 +32,182 @@ defmodule Mix.Tasks.Pleroma.Config do
         )
 
       migrate_from_db(opts)
+    end)
+  end
+
+  def run(["dump"]) do
+    check_configdb(fn ->
+      start_pleroma()
+
+      header = config_header()
+
+      settings =
+        ConfigDB
+        |> Repo.all()
+        |> Enum.sort()
+
+      unless settings == [] do
+        shell_info("#{header}")
+
+        Enum.each(settings, &dump(&1))
+      else
+        shell_error("No settings in ConfigDB.")
+      end
+    end)
+  end
+
+  def run(["dump", group, key]) do
+    check_configdb(fn ->
+      start_pleroma()
+
+      group = maybe_atomize(group)
+      key = maybe_atomize(key)
+
+      group
+      |> ConfigDB.get_by_group_and_key(key)
+      |> dump()
+    end)
+  end
+
+  def run(["dump", group]) do
+    check_configdb(fn ->
+      start_pleroma()
+
+      group = maybe_atomize(group)
+
+      dump_group(group)
+    end)
+  end
+
+  def run(["groups"]) do
+    check_configdb(fn ->
+      start_pleroma()
+
+      groups =
+        ConfigDB
+        |> distinct([c], true)
+        |> select([c], c.group)
+        |> Repo.all()
+
+      if length(groups) > 0 do
+        shell_info("The following configuration groups are set in ConfigDB:\r\n")
+        groups |> Enum.each(fn x -> shell_info("-  #{x}") end)
+        shell_info("\r\n")
+      end
+    end)
+  end
+
+  def run(["reset", "--force"]) do
+    check_configdb(fn ->
+      start_pleroma()
+      truncatedb()
+      shell_info("The ConfigDB settings have been removed from the database.")
+    end)
+  end
+
+  def run(["reset"]) do
+    check_configdb(fn ->
+      start_pleroma()
+
+      shell_info("The following settings will be permanently removed:")
+
+      ConfigDB
+      |> Repo.all()
+      |> Enum.sort()
+      |> Enum.each(&dump(&1))
+
+      shell_error("\nTHIS CANNOT BE UNDONE!")
+
+      if shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
+        truncatedb()
+
+        shell_info("The ConfigDB settings have been removed from the database.")
+      else
+        shell_error("No changes made.")
+      end
+    end)
+  end
+
+  def run(["delete", "--force", group, key]) do
+    start_pleroma()
+
+    group = maybe_atomize(group)
+    key = maybe_atomize(key)
+
+    with true <- key_exists?(group, key) do
+      shell_info("The following settings will be removed from ConfigDB:\n")
+
+      group
+      |> ConfigDB.get_by_group_and_key(key)
+      |> dump()
+
+      delete_key(group, key)
+    else
+      _ ->
+        shell_error("No settings in ConfigDB for #{inspect(group)}, #{inspect(key)}. Aborting.")
+    end
+  end
+
+  def run(["delete", "--force", group]) do
+    start_pleroma()
+
+    group = maybe_atomize(group)
+
+    with true <- group_exists?(group) do
+      shell_info("The following settings will be removed from ConfigDB:\n")
+      dump_group(group)
+      delete_group(group)
+    else
+      _ -> shell_error("No settings in ConfigDB for #{inspect(group)}. Aborting.")
+    end
+  end
+
+  def run(["delete", group, key]) do
+    start_pleroma()
+
+    group = maybe_atomize(group)
+    key = maybe_atomize(key)
+
+    with true <- key_exists?(group, key) do
+      shell_info("The following settings will be removed from ConfigDB:\n")
+
+      group
+      |> ConfigDB.get_by_group_and_key(key)
+      |> dump()
+
+      if shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
+        delete_key(group, key)
+      else
+        shell_error("No changes made.")
+      end
+    else
+      _ ->
+        shell_error("No settings in ConfigDB for #{inspect(group)}, #{inspect(key)}. Aborting.")
+    end
+  end
+
+  def run(["delete", group]) do
+    start_pleroma()
+
+    group = maybe_atomize(group)
+
+    with true <- group_exists?(group) do
+      shell_info("The following settings will be removed from ConfigDB:\n")
+      dump_group(group)
+
+      if shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
+        delete_group(group)
+      else
+        shell_error("No changes made.")
+      end
+    else
+      _ -> shell_error("No settings in ConfigDB for #{inspect(group)}. Aborting.")
+    end
   end
 
   @spec migrate_to_db(Path.t() | nil) :: any()
   def migrate_to_db(file_path \\ nil) do
-    with true <- Pleroma.Config.get([:configurable_from_database]),
+    with :ok <- Pleroma.Config.DeprecationWarnings.warn() do
-         :ok <- Pleroma.Config.DeprecationWarnings.warn() do
       config_file =
         if file_path do
           file_path
@@ -47,16 +221,15 @@ defmodule Mix.Tasks.Pleroma.Config do
 
       do_migrate_to_db(config_file)
     else
-      :error -> deprecation_error()
+      _ ->
-      _ -> migration_error()
+        shell_error("Migration is not allowed until all deprecation warnings have been resolved.")
     end
   end
 
   defp do_migrate_to_db(config_file) do
     if File.exists?(config_file) do
       shell_info("Migrating settings from file: #{Path.expand(config_file)}")
-      Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;")
+      truncatedb()
-      Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")
 
       custom_config =
         config_file
@@ -80,11 +253,10 @@ defmodule Mix.Tasks.Pleroma.Config do
       shell_info("Settings for key #{key} migrated.")
     end)
 
-    shell_info("Settings for group :#{group} migrated.")
+    shell_info("Settings for group #{inspect(group)} migrated.")
   end
 
   defp migrate_from_db(opts) do
-    if Pleroma.Config.get([:configurable_from_database]) do
     env = opts[:env] || Pleroma.Config.get(:env)
 
     config_path =
@@ -111,19 +283,6 @@ defmodule Mix.Tasks.Pleroma.Config do
     shell_info(
       "Database configuration settings have been exported to config/#{env}.exported_from_db.secret.exs"
     )
-    else
-      migration_error()
-    end
-  end
-
-  defp migration_error do
-    shell_error(
-      "Migration is not allowed in config. You can change this behavior by setting `config :pleroma, configurable_from_database: true`"
-    )
-  end
-
-  defp deprecation_error do
-    shell_error("Migration is not allowed until all deprecation warnings have been resolved.")
   end
 
   if Code.ensure_loaded?(Config.Reader) do
@@ -150,8 +309,80 @@ defmodule Mix.Tasks.Pleroma.Config do
 
   defp delete(config, true) do
     {:ok, _} = Repo.delete(config)
-    shell_info("#{config.key} deleted from DB.")
+
+    shell_info(
+      "config #{inspect(config.group)}, #{inspect(config.key)} was deleted from the ConfigDB."
+    )
   end
 
   defp delete(_config, _), do: :ok
+
+  defp dump(%ConfigDB{} = config) do
+    value = inspect(config.value, limit: :infinity)
+
+    shell_info("config #{inspect(config.group)}, #{inspect(config.key)}, #{value}\r\n\r\n")
+  end
+
+  defp dump(_), do: :noop
+
+  defp dump_group(group) when is_atom(group) do
+    group
+    |> ConfigDB.get_all_by_group()
+    |> Enum.each(&dump/1)
+  end
+
+  defp group_exists?(group) do
+    group
+    |> ConfigDB.get_all_by_group()
+    |> Enum.any?()
+  end
+
+  defp key_exists?(group, key) do
+    group
+    |> ConfigDB.get_by_group_and_key(key)
+    |> is_nil
+    |> Kernel.!()
+  end
+
+  defp maybe_atomize(arg) when is_atom(arg), do: arg
+
+  defp maybe_atomize(":" <> arg), do: maybe_atomize(arg)
+
+  defp maybe_atomize(arg) when is_binary(arg) do
+    if ConfigDB.module_name?(arg) do
+      String.to_existing_atom("Elixir." <> arg)
+    else
+      String.to_atom(arg)
+    end
+  end
+
+  defp check_configdb(callback) do
+    with true <- Pleroma.Config.get([:configurable_from_database]) do
+      callback.()
+    else
+      _ ->
+        shell_error(
+          "ConfigDB not enabled. Please check the value of :configurable_from_database in your configuration."
+        )
+    end
+  end
+
+  defp delete_key(group, key) do
+    check_configdb(fn ->
+      ConfigDB.delete(%{group: group, key: key})
+    end)
+  end
+
+  defp delete_group(group) do
+    check_configdb(fn ->
+      group
+      |> ConfigDB.get_all_by_group()
+      |> Enum.each(&ConfigDB.delete/1)
+    end)
+  end
+
+  defp truncatedb do
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;")
+    Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")
+  end
 end

lib/mix/tasks/pleroma/database.ex

@@ -48,9 +48,15 @@ defmodule Mix.Tasks.Pleroma.Database do
   def run(["update_users_following_followers_counts"]) do
     start_pleroma()
 
-    User
+    Repo.transaction(
-    |> Repo.all()
+      fn ->
-    |> Enum.each(&User.update_follower_count/1)
+        from(u in User, select: u)
+        |> Repo.stream()
+        |> Stream.each(&User.update_follower_count/1)
+        |> Stream.run()
+      end,
+      timeout: :infinity
+    )
   end
 
   def run(["prune_objects" | args]) do

lib/mix/tasks/pleroma/frontend.ex

@@ -17,8 +17,6 @@ defmodule Mix.Tasks.Pleroma.Frontend do
   end
 
   def run(["install", frontend | args]) do
-    log_level = Logger.level()
-    Logger.configure(level: :warn)
     start_pleroma()
 
     {options, [], []} =
@@ -33,109 +31,6 @@ defmodule Mix.Tasks.Pleroma.Frontend do
         ]
       )
 
-    instance_static_dir =
+    Pleroma.Frontend.install(frontend, options)
-      with nil <- options[:static_dir] do
-        Pleroma.Config.get!([:instance, :static_dir])
-      end
-
-    cmd_frontend_info = %{
-      "name" => frontend,
-      "ref" => options[:ref],
-      "build_url" => options[:build_url],
-      "build_dir" => options[:build_dir]
-    }
-
-    config_frontend_info = Pleroma.Config.get([:frontends, :available, frontend], %{})
-
-    frontend_info =
-      Map.merge(config_frontend_info, cmd_frontend_info, fn _key, config, cmd ->
-        # This only overrides things that are actually set
-        cmd || config
-      end)
-
-    ref = frontend_info["ref"]
-
-    unless ref do
-      raise "No ref given or configured"
-    end
-
-    dest =
-      Path.join([
-        instance_static_dir,
-        "frontends",
-        frontend,
-        ref
-      ])
-
-    fe_label = "#{frontend} (#{ref})"
-
-    tmp_dir = Path.join([instance_static_dir, "frontends", "tmp"])
-
-    with {_, :ok} <-
-           {:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, options[:file])},
-         shell_info("Installing #{fe_label} to #{dest}"),
-         :ok <- install_frontend(frontend_info, tmp_dir, dest) do
-      File.rm_rf!(tmp_dir)
-      shell_info("Frontend #{fe_label} installed to #{dest}")
-
-      Logger.configure(level: log_level)
-    else
-      {:download_or_unzip, _} ->
-        shell_info("Could not download or unzip the frontend")
-
-      _e ->
-        shell_info("Could not install the frontend")
-    end
-  end
-
-  defp download_or_unzip(frontend_info, temp_dir, file) do
-    if file do
-      with {:ok, zip} <- File.read(Path.expand(file)) do
-        unzip(zip, temp_dir)
-      end
-    else
-      download_build(frontend_info, temp_dir)
-    end
-  end
-
-  def unzip(zip, dest) do
-    with {:ok, unzipped} <- :zip.unzip(zip, [:memory]) do
-      File.rm_rf!(dest)
-      File.mkdir_p!(dest)
-
-      Enum.each(unzipped, fn {filename, data} ->
-        path = filename
-
-        new_file_path = Path.join(dest, path)
-
-        new_file_path
-        |> Path.dirname()
-        |> File.mkdir_p!()
-
-        File.write!(new_file_path, data)
-      end)
-
-      :ok
-    end
-  end
-
-  defp download_build(frontend_info, dest) do
-    shell_info("Downloading pre-built bundle for #{frontend_info["name"]}")
-    url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
-
-    with {:ok, %{status: 200, body: zip_body}} <-
-           Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do
-      unzip(zip_body, dest)
-    else
-      e -> {:error, e}
-    end
-  end
-
-  defp install_frontend(frontend_info, source, dest) do
-    from = frontend_info["build_dir"] || "dist"
-    File.rm_rf!(dest)
-    File.mkdir_p!(dest)
-    File.cp_r!(Path.join([source, from]), dest)
-    :ok
   end
 end

lib/mix/tasks/pleroma/instance.ex

@@ -161,12 +161,21 @@ defmodule Mix.Tasks.Pleroma.Instance do
         )
         |> Path.expand()
 
+      {strip_uploads_message, strip_uploads_default} =
+        if Pleroma.Utils.command_available?("exiftool") do
+          {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as installed. (y/n)",
+           "y"}
+        else
+          {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
+           "n"}
+        end
+
       strip_uploads =
         get_option(
           options,
           :strip_uploads,
-          "Do you want to strip location (GPS) data from uploaded images? (y/n)",
+          strip_uploads_message,
-          "y"
+          strip_uploads_default
         ) === "y"
 
       anonymize_uploads =
@@ -253,7 +262,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
     else
       shell_error(
         "The task would have overwritten the following files:\n" <>
-          (Enum.map(paths, &"- #{&1}\n") |> Enum.join("")) <>
+          (Enum.map(will_overwrite, &"- #{&1}\n") |> Enum.join("")) <>
           "Rerun with `--force` to overwrite them."
       )
     end
@@ -284,7 +293,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
   defp upload_filters(filters) when is_map(filters) do
     enabled_filters =
       if filters.strip do
-        [Pleroma.Upload.Filter.ExifTool]
+        [Pleroma.Upload.Filter.Exiftool]
       else
         []
       end

lib/mix/tasks/pleroma/user.ex

@@ -60,7 +60,7 @@ defmodule Mix.Tasks.Pleroma.User do
       - admin: #{if(admin?, do: "true", else: "false")}
     """)
 
-    proceed? = assume_yes? or shell_yes?("Continue?")
+    proceed? = assume_yes? or shell_prompt("Continue?", "n") in ~w(Yn Y y)
 
     if proceed? do
       start_pleroma()
@@ -345,11 +345,11 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
-  def run(["toggle_confirmed", nickname]) do
+  def run(["confirm", nickname]) do
     start_pleroma()
 
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
-      {:ok, user} = User.toggle_confirmation(user)
+      {:ok, user} = User.confirm(user)
 
       message = if user.confirmation_pending, do: "needs", else: "doesn't need"
 

lib/phoenix/transports/web_socket/raw.ex

@@ -31,7 +31,12 @@ defmodule Phoenix.Transports.WebSocket.Raw do
 
     case conn do
       %{halted: false} = conn ->
-        case Transport.connect(endpoint, handler, transport, __MODULE__, nil, conn.params) do
+        case handler.connect(%{
+               endpoint: endpoint,
+               transport: transport,
+               options: [serializer: nil],
+               params: conn.params
+             }) do
           {:ok, socket} ->
             {:ok, conn, {__MODULE__, {socket, opts}}}
 

lib/pleroma/activity.ex

@@ -14,6 +14,7 @@ defmodule Pleroma.Activity do
   alias Pleroma.ReportNote
   alias Pleroma.ThreadMute
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
 
   import Ecto.Changeset
   import Ecto.Query
@@ -23,6 +24,8 @@ defmodule Pleroma.Activity do
 
   @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   schema "activities" do
     field(:data, :map)
     field(:local, :boolean, default: true)
@@ -153,6 +156,18 @@ defmodule Pleroma.Activity do
 
   def get_bookmark(_, _), do: nil
 
+  def get_report(activity_id) do
+    opts = %{
+      type: "Flag",
+      skip_preload: true,
+      preload_report_notes: true
+    }
+
+    ActivityPub.fetch_activities_query([], opts)
+    |> where(id: ^activity_id)
+    |> Repo.one()
+  end
+
   def change(struct, params \\ %{}) do
     struct
     |> cast(params, [:data, :recipients])
@@ -181,6 +196,19 @@ defmodule Pleroma.Activity do
     end
   end
 
+  def get_by_id_with_user_actor(id) do
+    case FlakeId.flake_id?(id) do
+      true ->
+        Activity
+        |> where([a], a.id == ^id)
+        |> with_preloaded_user_actor()
+        |> Repo.one()
+
+      _ ->
+        nil
+    end
+  end
+
   def get_by_id_with_object(id) do
     Activity
     |> where(id: ^id)
@@ -272,7 +300,7 @@ defmodule Pleroma.Activity do
 
   defp purge_web_resp_cache(%Activity{} = activity) do
     %{path: path} = URI.parse(activity.data["id"])
-    Cachex.del(:web_resp_cache, path)
+    @cachex.del(:web_resp_cache, path)
     activity
   end
 
@@ -343,4 +371,15 @@ defmodule Pleroma.Activity do
     actor = user_actor(activity)
     activity.id in actor.pinned_activities
   end
+
+  @spec get_by_object_ap_id_with_object(String.t()) :: t() | nil
+  def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
+    ap_id
+    |> Queries.by_object_id()
+    |> with_preloaded_object()
+    |> first()
+    |> Repo.one()
+  end
+
+  def get_by_object_ap_id_with_object(_), do: nil
 end

lib/pleroma/activity/ir/topics.ex

@@ -40,7 +40,8 @@ defmodule Pleroma.Activity.Ir.Topics do
   end
 
   defp item_creation_tags(tags, object, %{data: %{"type" => "Create"}} = activity) do
-    tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
+    tags ++
+      remote_topics(activity) ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
   end
 
   defp item_creation_tags(tags, _, _) do
@@ -55,9 +56,19 @@ defmodule Pleroma.Activity.Ir.Topics do
 
   defp hashtags_to_topics(_), do: []
 
+  defp remote_topics(%{local: true}), do: []
+
+  defp remote_topics(%{actor: actor}) when is_binary(actor),
+    do: ["public:remote:" <> URI.parse(actor).host]
+
+  defp remote_topics(_), do: []
+
   defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: []
 
   defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"]
 
+  defp attachment_topics(_object, %{actor: actor}) when is_binary(actor),
+    do: ["public:media", "public:remote:media:" <> URI.parse(actor).host]
+
   defp attachment_topics(_object, _act), do: ["public:media"]
 end

lib/pleroma/activity/search.ex

@@ -19,15 +19,25 @@ defmodule Pleroma.Activity.Search do
     offset = Keyword.get(options, :offset, 0)
     author = Keyword.get(options, :author)
 
+    search_function =
+      if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do
+        :websearch
+      else
+        :plain
+      end
+
     Activity
     |> Activity.with_preloaded_object()
     |> Activity.restrict_deactivated_users()
     |> restrict_public()
-    |> query_with(index_type, search_query)
+    |> query_with(index_type, search_query, search_function)
     |> maybe_restrict_local(user)
     |> maybe_restrict_author(author)
     |> maybe_restrict_blocked(user)
-    |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => limit}, :offset)
+    |> Pagination.fetch_paginated(
+      %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
+      :offset
+    )
     |> maybe_fetch(user, search_query)
   end
 
@@ -50,7 +60,7 @@ defmodule Pleroma.Activity.Search do
     )
   end
 
-  defp query_with(q, :gin, search_query) do
+  defp query_with(q, :gin, search_query, :plain) do
     from([a, o] in q,
       where:
         fragment(
@@ -61,7 +71,18 @@ defmodule Pleroma.Activity.Search do
     )
   end
 
-  defp query_with(q, :rum, search_query) do
+  defp query_with(q, :gin, search_query, :websearch) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "to_tsvector('english', ?->>'content') @@ websearch_to_tsquery('english', ?)",
+          o.data,
+          ^search_query
+        )
+    )
+  end
+
+  defp query_with(q, :rum, search_query, :plain) do
     from([a, o] in q,
       where:
         fragment(
@@ -73,6 +94,18 @@ defmodule Pleroma.Activity.Search do
     )
   end
 
+  defp query_with(q, :rum, search_query, :websearch) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "? @@ websearch_to_tsquery('english', ?)",
+          o.fts_content,
+          ^search_query
+        ),
+      order_by: [fragment("? <=> now()::date", o.inserted_at)]
+    )
+  end
+
   defp maybe_restrict_local(q, user) do
     limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
 

lib/pleroma/application.ex

@@ -57,6 +57,7 @@ defmodule Pleroma.Application do
     setup_instrumenters()
     load_custom_modules()
     Pleroma.Docs.JSON.compile()
+    limiters_setup()
 
     adapter = Application.get_env(:tesla, :adapter)
 
@@ -100,7 +101,7 @@ defmodule Pleroma.Application do
         ] ++
         task_children(@env) ++
         dont_run_in_test(@env) ++
-        chat_child(@env, chat_enabled?()) ++
+        chat_child(chat_enabled?()) ++
         [
           Pleroma.Web.Endpoint,
           Pleroma.Gopher.Server
@@ -109,7 +110,28 @@ defmodule Pleroma.Application do
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
-    Supervisor.start_link(children, opts)
+    result = Supervisor.start_link(children, opts)
+
+    set_postgres_server_version()
+
+    result
+  end
+
+  defp set_postgres_server_version do
+    version =
+      with %{rows: [[version]]} <- Ecto.Adapters.SQL.query!(Pleroma.Repo, "show server_version"),
+           {num, _} <- Float.parse(version) do
+        num
+      else
+        e ->
+          Logger.warn(
+            "Could not get the postgres version: #{inspect(e)}.\nSetting the default value of 9.6"
+          )
+
+          9.6
+      end
+
+    :persistent_term.put({Pleroma.Repo, :postgres_version}, version)
   end
 
   def load_custom_modules do
@@ -151,7 +173,10 @@ defmodule Pleroma.Application do
 
     Pleroma.Web.Endpoint.MetricsExporter.setup()
     Pleroma.Web.Endpoint.PipelineInstrumenter.setup()
-    Pleroma.Web.Endpoint.Instrumenter.setup()
+
+    # Note: disabled until prometheus-phx is integrated into prometheus-phoenix:
+    # Pleroma.Web.Endpoint.Instrumenter.setup()
+    PrometheusPhx.setup()
   end
 
   defp cachex_children do
@@ -165,7 +190,11 @@ defmodule Pleroma.Application do
       build_cachex("web_resp", limit: 2500),
       build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
       build_cachex("failed_proxy_url", limit: 2500),
-      build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
+      build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
+      build_cachex("chat_message_id_idempotency_key",
+        expiration: chat_message_id_idempotency_key_expiration(),
+        limit: 500_000
+      )
     ]
   end
 
@@ -175,6 +204,9 @@ defmodule Pleroma.Application do
   defp idempotency_expiration,
     do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
 
+  defp chat_message_id_idempotency_key_expiration,
+    do: expiration(default: :timer.minutes(2), interval: :timer.seconds(60))
+
   defp seconds_valid_interval,
     do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
 
@@ -197,16 +229,18 @@ defmodule Pleroma.Application do
          name: Pleroma.Web.Streamer.registry(),
          keys: :duplicate,
          partitions: System.schedulers_online()
-       ]},
+       ]}
-      Pleroma.Web.FedSockets.Supervisor
     ]
   end
 
-  defp chat_child(_env, true) do
+  defp chat_child(true) do
-    [Pleroma.Web.ChatChannel.ChatChannelState]
+    [
+      Pleroma.Web.ChatChannel.ChatChannelState,
+      {Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]}
+    ]
   end
 
-  defp chat_child(_, _), do: []
+  defp chat_child(_), do: []
 
   defp task_children(:test) do
     [
@@ -260,4 +294,10 @@ defmodule Pleroma.Application do
   end
 
   defp http_children(_, _), do: []
+
+  @spec limiters_setup() :: :ok
+  def limiters_setup do
+    [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy]
+    |> Enum.each(&ConcurrentLimiter.new(&1, 1, 0))
+  end
 end

lib/pleroma/application_requirements.ex

@@ -24,6 +24,7 @@ defmodule Pleroma.ApplicationRequirements do
     |> check_migrations_applied!()
     |> check_welcome_message_config!()
     |> check_rum!()
+    |> check_repo_pool_size!()
     |> handle_result()
   end
 
@@ -188,6 +189,30 @@ defmodule Pleroma.ApplicationRequirements do
 
   defp check_system_commands!(result), do: result
 
+  defp check_repo_pool_size!(:ok) do
+    if Pleroma.Config.get([Pleroma.Repo, :pool_size], 10) != 10 and
+         not Pleroma.Config.get([:dangerzone, :override_repo_pool_size], false) do
+      Logger.error("""
+      !!!CONFIG WARNING!!!
+
+      The database pool size has been altered from the recommended value of 10.
+
+      Please revert or ensure your database is tuned appropriately and then set
+      `config :pleroma, :dangerzone, override_repo_pool_size: true`.
+
+      If you are experiencing database timeouts, please check the "Optimizing
+      your PostgreSQL performance" section in the documentation. If you still
+      encounter issues after that, please open an issue on the tracker.
+      """)
+
+      {:error, "Repo.pool_size different than recommended value."}
+    else
+      :ok
+    end
+  end
+
+  defp check_repo_pool_size!(result), do: result
+
   defp check_filter(filter, command_required) do
     filters = Config.get([Pleroma.Upload, :filters])
 

lib/pleroma/caching.ex

@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Caching do
+  @callback get!(Cachex.cache(), any()) :: any()
+  @callback get(Cachex.cache(), any()) :: {atom(), any()}
+  @callback put(Cachex.cache(), any(), any(), Keyword.t()) :: {Cachex.status(), boolean()}
+  @callback put(Cachex.cache(), any(), any()) :: {Cachex.status(), boolean()}
+  @callback fetch!(Cachex.cache(), any(), function() | nil) :: any()
+  # @callback del(Cachex.cache(), any(), Keyword.t()) :: {Cachex.status(), boolean()}
+  @callback del(Cachex.cache(), any()) :: {Cachex.status(), boolean()}
+  @callback stream!(Cachex.cache(), any()) :: Enumerable.t()
+  @callback expire_at(Cachex.cache(), binary(), number()) :: {Cachex.status(), boolean()}
+  @callback exists?(Cachex.cache(), any()) :: {Cachex.status(), boolean()}
+  @callback execute!(Cachex.cache(), function()) :: any()
+  @callback get_and_update(Cachex.cache(), any(), function()) ::
+              {:commit | :ignore, any()}
+end

lib/pleroma/captcha.ex

@@ -7,6 +7,8 @@ defmodule Pleroma.Captcha do
   alias Plug.Crypto.KeyGenerator
   alias Plug.Crypto.MessageEncryptor
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   @doc """
   Ask the configured captcha service for a new captcha
   """
@@ -86,7 +88,7 @@ defmodule Pleroma.Captcha do
   end
 
   defp validate_usage(token) do
-    if is_nil(Cachex.get!(:used_captcha_cache, token)) do
+    if is_nil(@cachex.get!(:used_captcha_cache, token)) do
       :ok
     else
       {:error, :already_used}
@@ -95,7 +97,7 @@ defmodule Pleroma.Captcha do
 
   defp mark_captcha_as_used(token) do
     ttl = seconds_valid() |> :timer.seconds()
-    Cachex.put(:used_captcha_cache, token, true, ttl: ttl)
+    @cachex.put(:used_captcha_cache, token, true, ttl: ttl)
   end
 
   defp method, do: Pleroma.Config.get!([__MODULE__, :method])

lib/pleroma/captcha/kocaptcha.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.Captcha.Kocaptcha do
   def new do
     endpoint = Pleroma.Config.get!([__MODULE__, :endpoint])
 
-    case Tesla.get(endpoint <> "/new") do
+    case Pleroma.HTTP.get(endpoint <> "/new") do
       {:error, _} ->
         %{error: :kocaptcha_service_unavailable}
 

lib/pleroma/config.ex

@@ -3,14 +3,18 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Config do
+  @behaviour Pleroma.Config.Getting
   defmodule Error do
     defexception [:message]
   end
 
+  @impl true
   def get(key), do: get(key, nil)
 
+  @impl true
   def get([key], default), do: get(key, default)
 
+  @impl true
   def get([_ | _] = path, default) do
     case fetch(path) do
       {:ok, value} -> value
@@ -18,6 +22,7 @@ defmodule Pleroma.Config do
     end
   end
 
+  @impl true
   def get(key, default) do
     Application.get_env(:pleroma, key, default)
   end

lib/pleroma/config/getting.ex

@@ -0,0 +1,8 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Config.Getting do
+  @callback get(any()) :: any()
+  @callback get(any(), any()) :: any()
+end

lib/pleroma/config/holder.ex

@@ -9,12 +9,7 @@ defmodule Pleroma.Config.Holder do
   def save_default do
     default_config =
       if System.get_env("RELEASE_NAME") do
-        release_config =
+        Pleroma.Config.Loader.merge(@config, release_defaults())
-          [:code.root_dir(), "releases", System.get_env("RELEASE_VSN"), "releases.exs"]
-          |> Path.join()
-          |> Pleroma.Config.Loader.read()
-
-        Pleroma.Config.Loader.merge(@config, release_config)
       else
         @config
       end
@@ -32,4 +27,16 @@ defmodule Pleroma.Config.Holder do
   def default_config(group, key), do: get_in(get_default(), [group, key])
 
   defp get_default, do: Pleroma.Config.get(:default_config)
+
+  @spec release_defaults() :: keyword()
+  def release_defaults do
+    [
+      pleroma: [
+        {:instance, [static_dir: "/var/lib/pleroma/static"]},
+        {Pleroma.Uploaders.Local, [uploads: "/var/lib/pleroma/uploads"]},
+        {:modules, [runtime_dir: "/var/lib/pleroma/modules"]},
+        {:release, true}
+      ]
+    ]
+  end
 end

lib/pleroma/config/release_runtime_provider.ex

@@ -0,0 +1,50 @@
+defmodule Pleroma.Config.ReleaseRuntimeProvider do
+  @moduledoc """
+  Imports `runtime.exs` and `{env}.exported_from_db.secret.exs` for elixir releases.
+  """
+  @behaviour Config.Provider
+
+  @impl true
+  def init(opts), do: opts
+
+  @impl true
+  def load(config, _opts) do
+    with_defaults = Config.Reader.merge(config, Pleroma.Config.Holder.release_defaults())
+
+    config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
+
+    with_runtime_config =
+      if File.exists?(config_path) do
+        runtime_config = Config.Reader.read!(config_path)
+
+        with_defaults
+        |> Config.Reader.merge(pleroma: [config_path: config_path])
+        |> Config.Reader.merge(runtime_config)
+      else
+        warning = [
+          IO.ANSI.red(),
+          IO.ANSI.bright(),
+          "!!! #{config_path} not found! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file",
+          IO.ANSI.reset()
+        ]
+
+        IO.puts(warning)
+        with_defaults
+      end
+
+    exported_config_path =
+      config_path
+      |> Path.dirname()
+      |> Path.join("prod.exported_from_db.secret.exs")
+
+    with_exported =
+      if File.exists?(exported_config_path) do
+        exported_config = Config.Reader.read!(with_runtime_config)
+        Config.Reader.merge(with_runtime_config, exported_config)
+      else
+        with_runtime_config
+      end
+
+    with_exported
+  end
+end

lib/pleroma/config_db.ex

@@ -6,7 +6,7 @@ defmodule Pleroma.ConfigDB do
   use Ecto.Schema
 
   import Ecto.Changeset
-  import Ecto.Query, only: [select: 3]
+  import Ecto.Query, only: [select: 3, from: 2]
   import Pleroma.Web.Gettext
 
   alias __MODULE__
@@ -41,8 +41,18 @@ defmodule Pleroma.ConfigDB do
     end)
   end
 
+  @spec get_all_by_group(atom() | String.t()) :: [t()]
+  def get_all_by_group(group) do
+    from(c in ConfigDB, where: c.group == ^group) |> Repo.all()
+  end
+
+  @spec get_by_group_and_key(atom() | String.t(), atom() | String.t()) :: t() | nil
+  def get_by_group_and_key(group, key) do
+    get_by_params(%{group: group, key: key})
+  end
+
   @spec get_by_params(map()) :: ConfigDB.t() | nil
-  def get_by_params(params), do: Repo.get_by(ConfigDB, params)
+  def get_by_params(%{group: _, key: _} = params), do: Repo.get_by(ConfigDB, params)
 
   @spec changeset(ConfigDB.t(), map()) :: Changeset.t()
   def changeset(config, params \\ %{}) do

lib/pleroma/constants.ex

@@ -26,4 +26,6 @@ defmodule Pleroma.Constants do
     do:
       ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css)
   )
+
+  def as_local_public, do: Pleroma.Web.base_url() <> "/#Public"
 end

lib/pleroma/conversation.ex

@@ -43,7 +43,7 @@ defmodule Pleroma.Conversation do
   def maybe_create_recipientships(participation, activity) do
     participation = Repo.preload(participation, :recipients)
 
-    if participation.recipients |> Enum.empty?() do
+    if Enum.empty?(participation.recipients) do
       recipients = User.get_all_by_ap_id(activity.recipients)
       RecipientShip.create(recipients, participation)
     end
@@ -69,10 +69,6 @@ defmodule Pleroma.Conversation do
         Enum.map(users, fn user ->
           invisible_conversation = Enum.any?(users, &User.blocks?(user, &1))
 
-          unless invisible_conversation do
-            User.increment_unread_conversation_count(conversation, user)
-          end
-
           opts = Keyword.put(opts, :invisible_conversation, invisible_conversation)
 
           {:ok, participation} =

lib/pleroma/conversation/participation.ex

@@ -63,21 +63,10 @@ defmodule Pleroma.Conversation.Participation do
     end
   end
 
-  def mark_as_read(participation) do
+  def mark_as_read(%__MODULE__{} = participation) do
-    __MODULE__
+    participation
-    |> where(id: ^participation.id)
+    |> change(read: true)
-    |> update(set: [read: true])
+    |> Repo.update()
-    |> select([p], p)
-    |> Repo.update_all([])
-    |> case do
-      {1, [participation]} ->
-        participation = Repo.preload(participation, :user)
-        User.set_unread_conversation_count(participation.user)
-        {:ok, participation}
-
-      error ->
-        error
-    end
   end
 
   def mark_all_as_read(%User{local: true} = user, %User{} = target_user) do
@@ -93,7 +82,6 @@ defmodule Pleroma.Conversation.Participation do
     |> update([p], set: [read: true])
     |> Repo.update_all([])
 
-    {:ok, user} = User.set_unread_conversation_count(user)
     {:ok, user, []}
   end
 
@@ -108,7 +96,6 @@ defmodule Pleroma.Conversation.Participation do
       |> select([p], p)
       |> Repo.update_all([])
 
-    {:ok, user} = User.set_unread_conversation_count(user)
     {:ok, user, participations}
   end
 
@@ -220,6 +207,12 @@ defmodule Pleroma.Conversation.Participation do
     {:ok, Repo.preload(participation, :recipients, force: true)}
   end
 
+  @spec unread_count(User.t()) :: integer()
+  def unread_count(%User{id: user_id}) do
+    from(q in __MODULE__, where: q.user_id == ^user_id and q.read == false)
+    |> Repo.aggregate(:count, :id)
+  end
+
   def unread_conversation_count_for_user(user) do
     from(p in __MODULE__,
       where: p.user_id == ^user.id,

lib/pleroma/docs/json.ex

@@ -11,7 +11,11 @@ defmodule Pleroma.Docs.JSON do
 
   @spec compile :: :ok
   def compile do
-    :persistent_term.put(@term, Pleroma.Docs.Generator.convert_to_strings(@raw_descriptions))
+    descriptions =
+      Pleroma.Web.ActivityPub.MRF.config_descriptions()
+      |> Enum.reduce(@raw_descriptions, fn description, acc -> [description | acc] end)
+
+    :persistent_term.put(@term, Pleroma.Docs.Generator.convert_to_strings(descriptions))
   end
 
   @spec compiled_descriptions :: Map.t()

lib/pleroma/emails/admin_email.ex

@@ -18,10 +18,6 @@ defmodule Pleroma.Emails.AdminEmail do
     Keyword.get(instance_config(), :notify_email, instance_config()[:email])
   end
 
-  defp user_url(user) do
-    Helpers.user_feed_url(Pleroma.Web.Endpoint, :feed_redirect, user.id)
-  end
-
   def test_email(mail_to \\ nil) do
     html_body = """
     <h3>Instance Test Email</h3>
@@ -52,6 +48,9 @@ defmodule Pleroma.Emails.AdminEmail do
               status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, id)
               "<li><a href=\"#{status_url}\">#{status_url}</li>"
 
+            %{"id" => id} when is_binary(id) ->
+              "<li><a href=\"#{id}\">#{id}</li>"
+
             id when is_binary(id) ->
               "<li><a href=\"#{id}\">#{id}</li>"
           end)
@@ -69,8 +68,8 @@ defmodule Pleroma.Emails.AdminEmail do
       end
 
     html_body = """
-    <p>Reported by: <a href="#{user_url(reporter)}">#{reporter.nickname}</a></p>
+    <p>Reported by: <a href="#{reporter.ap_id}">#{reporter.nickname}</a></p>
-    <p>Reported Account: <a href="#{user_url(account)}">#{account.nickname}</a></p>
+    <p>Reported Account: <a href="#{account.ap_id}">#{account.nickname}</a></p>
     #{comment_html}
     #{statuses_html}
     <p>
@@ -86,7 +85,7 @@ defmodule Pleroma.Emails.AdminEmail do
 
   def new_unapproved_registration(to, account) do
     html_body = """
-    <p>New account for review: <a href="#{user_url(account)}">@#{account.nickname}</a></p>
+    <p>New account for review: <a href="#{account.ap_id}">@#{account.nickname}</a></p>
     <blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote>
     <a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
     """

lib/pleroma/emails/user_email.ex

@@ -93,6 +93,19 @@ defmodule Pleroma.Emails.UserEmail do
     |> html_body(html_body)
   end
 
+  def approval_pending_email(user) do
+    html_body = """
+    <h3>Awaiting Approval</h3>
+    <p>Your account at #{instance_name()} is being reviewed by staff. You will receive another email once your account is approved.</p>
+    """
+
+    new()
+    |> to(recipient(user))
+    |> from(sender())
+    |> subject("Your account is awaiting approval")
+    |> html_body(html_body)
+  end
+
   @doc """
   Email used in digest email notifications
   Includes Mentions and New Followers data
@@ -151,7 +164,7 @@ defmodule Pleroma.Emails.UserEmail do
 
       logo_path =
         if is_nil(logo) do
-          Path.join(:code.priv_dir(:pleroma), "static/static/logo.png")
+          Path.join(:code.priv_dir(:pleroma), "static/static/logo.svg")
         else
           Path.join(Config.get([:instance, :static_dir]), logo)
         end
@@ -162,7 +175,7 @@ defmodule Pleroma.Emails.UserEmail do
       |> subject("Your digest from #{instance_name()}")
       |> put_layout(false)
       |> render_body("digest.html", html_data)
-      |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.png", type: :inline))
+      |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.svg", type: :inline))
     end
   end
 
@@ -189,4 +202,30 @@ defmodule Pleroma.Emails.UserEmail do
 
     Router.Helpers.subscription_url(Endpoint, :unsubscribe, token)
   end
+
+  def backup_is_ready_email(backup, admin_user_id \\ nil) do
+    %{user: user} = Pleroma.Repo.preload(backup, :user)
+    download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
+
+    html_body =
+      if is_nil(admin_user_id) do
+        """
+        <p>You requested a full backup of your Pleroma account. It's ready for download:</p>
+        <p><a href="#{download_url}">#{download_url}</a></p>
+        """
+      else
+        admin = Pleroma.Repo.get(User, admin_user_id)
+
+        """
+        <p>Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:</p>
+        <p><a href="#{download_url}">#{download_url}</a></p>
+        """
+      end
+
+    new()
+    |> to(recipient(user))
+    |> from(sender())
+    |> subject("Your account archive is ready")
+    |> html_body(html_body)
+  end
 end

lib/pleroma/emoji-data.txt

@@ -1,769 +0,0 @@
-# emoji-data.txt
-# Date: 2019-01-15, 12:10:05 GMT
-# © 2019 Unicode®, Inc.
-# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
-# For terms of use, see http://www.unicode.org/terms_of_use.html
-#
-# Emoji Data for UTS #51
-# Version: 12.0
-#
-# For documentation and usage, see http://www.unicode.org/reports/tr51
-#
-# Format: 
-# <codepoint(s)> ; <property> # <comments> 
-# Note: there is no guarantee as to the structure of whitespace or comments
-#
-# Characters and sequences are listed in code point order. Users should be shown a more natural order.
-# See the CLDR collation order for Emoji.
-
-
-# ================================================
-
-# All omitted code points have Emoji=No 
-# @missing: 0000..10FFFF  ; Emoji ; No
-
-0023          ; Emoji                #  1.1  [1] (#️)       number sign
-002A          ; Emoji                #  1.1  [1] (*️)       asterisk
-0030..0039    ; Emoji                #  1.1 [10] (0️..9️)    digit zero..digit nine
-00A9          ; Emoji                #  1.1  [1] (©️)       copyright
-00AE          ; Emoji                #  1.1  [1] (®️)       registered
-203C          ; Emoji                #  1.1  [1] (‼️)       double exclamation mark
-2049          ; Emoji                #  3.0  [1] (⁉️)       exclamation question mark
-2122          ; Emoji                #  1.1  [1] (™️)       trade mark
-2139          ; Emoji                #  3.0  [1] (ℹ️)       information
-2194..2199    ; Emoji                #  1.1  [6] (↔️..↙️)    left-right arrow..down-left arrow
-21A9..21AA    ; Emoji                #  1.1  [2] (↩️..↪️)    right arrow curving left..left arrow curving right
-231A..231B    ; Emoji                #  1.1  [2] (⌚..⌛)    watch..hourglass done
-2328          ; Emoji                #  1.1  [1] (⌨️)       keyboard
-23CF          ; Emoji                #  4.0  [1] (⏏️)       eject button
-23E9..23F3    ; Emoji                #  6.0 [11] (⏩..⏳)    fast-forward button..hourglass not done
-23F8..23FA    ; Emoji                #  7.0  [3] (⏸️..⏺️)    pause button..record button
-24C2          ; Emoji                #  1.1  [1] (Ⓜ️)       circled M
-25AA..25AB    ; Emoji                #  1.1  [2] (▪️..▫️)    black small square..white small square
-25B6          ; Emoji                #  1.1  [1] (▶️)       play button
-25C0          ; Emoji                #  1.1  [1] (◀️)       reverse button
-25FB..25FE    ; Emoji                #  3.2  [4] (◻️..◾)    white medium square..black medium-small square
-2600..2604    ; Emoji                #  1.1  [5] (☀️..☄️)    sun..comet
-260E          ; Emoji                #  1.1  [1] (☎️)       telephone
-2611          ; Emoji                #  1.1  [1] (☑️)       check box with check
-2614..2615    ; Emoji                #  4.0  [2] (☔..☕)    umbrella with rain drops..hot beverage
-2618          ; Emoji                #  4.1  [1] (☘️)       shamrock
-261D          ; Emoji                #  1.1  [1] (☝️)       index pointing up
-2620          ; Emoji                #  1.1  [1] (☠️)       skull and crossbones
-2622..2623    ; Emoji                #  1.1  [2] (☢️..☣️)    radioactive..biohazard
-2626          ; Emoji                #  1.1  [1] (☦️)       orthodox cross
-262A          ; Emoji                #  1.1  [1] (☪️)       star and crescent
-262E..262F    ; Emoji                #  1.1  [2] (☮️..☯️)    peace symbol..yin yang
-2638..263A    ; Emoji                #  1.1  [3] (☸️..☺️)    wheel of dharma..smiling face
-2640          ; Emoji                #  1.1  [1] (♀️)       female sign
-2642          ; Emoji                #  1.1  [1] (♂️)       male sign
-2648..2653    ; Emoji                #  1.1 [12] (♈..♓)    Aries..Pisces
-265F..2660    ; Emoji                #  1.1  [2] (♟️..♠️)    chess pawn..spade suit
-2663          ; Emoji                #  1.1  [1] (♣️)       club suit
-2665..2666    ; Emoji                #  1.1  [2] (♥️..♦️)    heart suit..diamond suit
-2668          ; Emoji                #  1.1  [1] (♨️)       hot springs
-267B          ; Emoji                #  3.2  [1] (♻️)       recycling symbol
-267E..267F    ; Emoji                #  4.1  [2] (♾️..♿)    infinity..wheelchair symbol
-2692..2697    ; Emoji                #  4.1  [6] (⚒️..⚗️)    hammer and pick..alembic
-2699          ; Emoji                #  4.1  [1] (⚙️)       gear
-269B..269C    ; Emoji                #  4.1  [2] (⚛️..⚜️)    atom symbol..fleur-de-lis
-26A0..26A1    ; Emoji                #  4.0  [2] (⚠️..⚡)    warning..high voltage
-26AA..26AB    ; Emoji                #  4.1  [2] (⚪..⚫)    white circle..black circle
-26B0..26B1    ; Emoji                #  4.1  [2] (⚰️..⚱️)    coffin..funeral urn
-26BD..26BE    ; Emoji                #  5.2  [2] (⚽..⚾)    soccer ball..baseball
-26C4..26C5    ; Emoji                #  5.2  [2] (⛄..⛅)    snowman without snow..sun behind cloud
-26C8          ; Emoji                #  5.2  [1] (⛈️)       cloud with lightning and rain
-26CE          ; Emoji                #  6.0  [1] (⛎)       Ophiuchus
-26CF          ; Emoji                #  5.2  [1] (⛏️)       pick
-26D1          ; Emoji                #  5.2  [1] (⛑️)       rescue worker’s helmet
-26D3..26D4    ; Emoji                #  5.2  [2] (⛓️..⛔)    chains..no entry
-26E9..26EA    ; Emoji                #  5.2  [2] (⛩️..⛪)    shinto shrine..church
-26F0..26F5    ; Emoji                #  5.2  [6] (⛰️..⛵)    mountain..sailboat
-26F7..26FA    ; Emoji                #  5.2  [4] (⛷️..⛺)    skier..tent
-26FD          ; Emoji                #  5.2  [1] (⛽)       fuel pump
-2702          ; Emoji                #  1.1  [1] (✂️)       scissors
-2705          ; Emoji                #  6.0  [1] (✅)       check mark button
-2708..2709    ; Emoji                #  1.1  [2] (✈️..✉️)    airplane..envelope
-270A..270B    ; Emoji                #  6.0  [2] (✊..✋)    raised fist..raised hand
-270C..270D    ; Emoji                #  1.1  [2] (✌️..✍️)    victory hand..writing hand
-270F          ; Emoji                #  1.1  [1] (✏️)       pencil
-2712          ; Emoji                #  1.1  [1] (✒️)       black nib
-2714          ; Emoji                #  1.1  [1] (✔️)       check mark
-2716          ; Emoji                #  1.1  [1] (✖️)       multiplication sign
-271D          ; Emoji                #  1.1  [1] (✝️)       latin cross
-2721          ; Emoji                #  1.1  [1] (✡️)       star of David
-2728          ; Emoji                #  6.0  [1] (✨)       sparkles
-2733..2734    ; Emoji                #  1.1  [2] (✳️..✴️)    eight-spoked asterisk..eight-pointed star
-2744          ; Emoji                #  1.1  [1] (❄️)       snowflake
-2747          ; Emoji                #  1.1  [1] (❇️)       sparkle
-274C          ; Emoji                #  6.0  [1] (❌)       cross mark
-274E          ; Emoji                #  6.0  [1] (❎)       cross mark button
-2753..2755    ; Emoji                #  6.0  [3] (❓..❕)    question mark..white exclamation mark
-2757          ; Emoji                #  5.2  [1] (❗)       exclamation mark
-2763..2764    ; Emoji                #  1.1  [2] (❣️..❤️)    heart exclamation..red heart
-2795..2797    ; Emoji                #  6.0  [3] (➕..➗)    plus sign..division sign
-27A1          ; Emoji                #  1.1  [1] (➡️)       right arrow
-27B0          ; Emoji                #  6.0  [1] (➰)       curly loop
-27BF          ; Emoji                #  6.0  [1] (➿)       double curly loop
-2934..2935    ; Emoji                #  3.2  [2] (⤴️..⤵️)    right arrow curving up..right arrow curving down
-2B05..2B07    ; Emoji                #  4.0  [3] (⬅️..⬇️)    left arrow..down arrow
-2B1B..2B1C    ; Emoji                #  5.1  [2] (⬛..⬜)    black large square..white large square
-2B50          ; Emoji                #  5.1  [1] (⭐)       star
-2B55          ; Emoji                #  5.2  [1] (⭕)       hollow red circle
-3030          ; Emoji                #  1.1  [1] (〰️)       wavy dash
-303D          ; Emoji                #  3.2  [1] (〽️)       part alternation mark
-3297          ; Emoji                #  1.1  [1] (㊗️)       Japanese “congratulations” button
-3299          ; Emoji                #  1.1  [1] (㊙️)       Japanese “secret” button
-1F004         ; Emoji                #  5.1  [1] (🀄)       mahjong red dragon
-1F0CF         ; Emoji                #  6.0  [1] (🃏)       joker
-1F170..1F171  ; Emoji                #  6.0  [2] (🅰️..🅱️)    A button (blood type)..B button (blood type)
-1F17E         ; Emoji                #  6.0  [1] (🅾️)       O button (blood type)
-1F17F         ; Emoji                #  5.2  [1] (🅿️)       P button
-1F18E         ; Emoji                #  6.0  [1] (🆎)       AB button (blood type)
-1F191..1F19A  ; Emoji                #  6.0 [10] (🆑..🆚)    CL button..VS button
-1F1E6..1F1FF  ; Emoji                #  6.0 [26] (🇦..🇿)    regional indicator symbol letter a..regional indicator symbol letter z
-1F201..1F202  ; Emoji                #  6.0  [2] (🈁..🈂️)    Japanese “here” button..Japanese “service charge” button
-1F21A         ; Emoji                #  5.2  [1] (🈚)       Japanese “free of charge” button
-1F22F         ; Emoji                #  5.2  [1] (🈯)       Japanese “reserved” button
-1F232..1F23A  ; Emoji                #  6.0  [9] (🈲..🈺)    Japanese “prohibited” button..Japanese “open for business” button
-1F250..1F251  ; Emoji                #  6.0  [2] (🉐..🉑)    Japanese “bargain” button..Japanese “acceptable” button
-1F300..1F320  ; Emoji                #  6.0 [33] (🌀..🌠)    cyclone..shooting star
-1F321         ; Emoji                #  7.0  [1] (🌡️)       thermometer
-1F324..1F32C  ; Emoji                #  7.0  [9] (🌤️..🌬️)    sun behind small cloud..wind face
-1F32D..1F32F  ; Emoji                #  8.0  [3] (🌭..🌯)    hot dog..burrito
-1F330..1F335  ; Emoji                #  6.0  [6] (🌰..🌵)    chestnut..cactus
-1F336         ; Emoji                #  7.0  [1] (🌶️)       hot pepper
-1F337..1F37C  ; Emoji                #  6.0 [70] (🌷..🍼)    tulip..baby bottle
-1F37D         ; Emoji                #  7.0  [1] (🍽️)       fork and knife with plate
-1F37E..1F37F  ; Emoji                #  8.0  [2] (🍾..🍿)    bottle with popping cork..popcorn
-1F380..1F393  ; Emoji                #  6.0 [20] (🎀..🎓)    ribbon..graduation cap
-1F396..1F397  ; Emoji                #  7.0  [2] (🎖️..🎗️)    military medal..reminder ribbon
-1F399..1F39B  ; Emoji                #  7.0  [3] (🎙️..🎛️)    studio microphone..control knobs
-1F39E..1F39F  ; Emoji                #  7.0  [2] (🎞️..🎟️)    film frames..admission tickets
-1F3A0..1F3C4  ; Emoji                #  6.0 [37] (🎠..🏄)    carousel horse..person surfing
-1F3C5         ; Emoji                #  7.0  [1] (🏅)       sports medal
-1F3C6..1F3CA  ; Emoji                #  6.0  [5] (🏆..🏊)    trophy..person swimming
-1F3CB..1F3CE  ; Emoji                #  7.0  [4] (🏋️..🏎️)    person lifting weights..racing car
-1F3CF..1F3D3  ; Emoji                #  8.0  [5] (🏏..🏓)    cricket game..ping pong
-1F3D4..1F3DF  ; Emoji                #  7.0 [12] (🏔️..🏟️)    snow-capped mountain..stadium
-1F3E0..1F3F0  ; Emoji                #  6.0 [17] (🏠..🏰)    house..castle
-1F3F3..1F3F5  ; Emoji                #  7.0  [3] (🏳️..🏵️)    white flag..rosette
-1F3F7         ; Emoji                #  7.0  [1] (🏷️)       label
-1F3F8..1F3FF  ; Emoji                #  8.0  [8] (🏸..🏿)    badminton..dark skin tone
-1F400..1F43E  ; Emoji                #  6.0 [63] (🐀..🐾)    rat..paw prints
-1F43F         ; Emoji                #  7.0  [1] (🐿️)       chipmunk
-1F440         ; Emoji                #  6.0  [1] (👀)       eyes
-1F441         ; Emoji                #  7.0  [1] (👁️)       eye
-1F442..1F4F7  ; Emoji                #  6.0[182] (👂..📷)    ear..camera
-1F4F8         ; Emoji                #  7.0  [1] (📸)       camera with flash
-1F4F9..1F4FC  ; Emoji                #  6.0  [4] (📹..📼)    video camera..videocassette
-1F4FD         ; Emoji                #  7.0  [1] (📽️)       film projector
-1F4FF         ; Emoji                #  8.0  [1] (📿)       prayer beads
-1F500..1F53D  ; Emoji                #  6.0 [62] (🔀..🔽)    shuffle tracks button..downwards button
-1F549..1F54A  ; Emoji                #  7.0  [2] (🕉️..🕊️)    om..dove
-1F54B..1F54E  ; Emoji                #  8.0  [4] (🕋..🕎)    kaaba..menorah
-1F550..1F567  ; Emoji                #  6.0 [24] (🕐..🕧)    one o’clock..twelve-thirty
-1F56F..1F570  ; Emoji                #  7.0  [2] (🕯️..🕰️)    candle..mantelpiece clock
-1F573..1F579  ; Emoji                #  7.0  [7] (🕳️..🕹️)    hole..joystick
-1F57A         ; Emoji                #  9.0  [1] (🕺)       man dancing
-1F587         ; Emoji                #  7.0  [1] (🖇️)       linked paperclips
-1F58A..1F58D  ; Emoji                #  7.0  [4] (🖊️..🖍️)    pen..crayon
-1F590         ; Emoji                #  7.0  [1] (🖐️)       hand with fingers splayed
-1F595..1F596  ; Emoji                #  7.0  [2] (🖕..🖖)    middle finger..vulcan salute
-1F5A4         ; Emoji                #  9.0  [1] (🖤)       black heart
-1F5A5         ; Emoji                #  7.0  [1] (🖥️)       desktop computer
-1F5A8         ; Emoji                #  7.0  [1] (🖨️)       printer
-1F5B1..1F5B2  ; Emoji                #  7.0  [2] (🖱️..🖲️)    computer mouse..trackball
-1F5BC         ; Emoji                #  7.0  [1] (🖼️)       framed picture
-1F5C2..1F5C4  ; Emoji                #  7.0  [3] (🗂️..🗄️)    card index dividers..file cabinet
-1F5D1..1F5D3  ; Emoji                #  7.0  [3] (🗑️..🗓️)    wastebasket..spiral calendar
-1F5DC..1F5DE  ; Emoji                #  7.0  [3] (🗜️..🗞️)    clamp..rolled-up newspaper
-1F5E1         ; Emoji                #  7.0  [1] (🗡️)       dagger
-1F5E3         ; Emoji                #  7.0  [1] (🗣️)       speaking head
-1F5E8         ; Emoji                #  7.0  [1] (🗨️)       left speech bubble
-1F5EF         ; Emoji                #  7.0  [1] (🗯️)       right anger bubble
-1F5F3         ; Emoji                #  7.0  [1] (🗳️)       ballot box with ballot
-1F5FA         ; Emoji                #  7.0  [1] (🗺️)       world map
-1F5FB..1F5FF  ; Emoji                #  6.0  [5] (🗻..🗿)    mount fuji..moai
-1F600         ; Emoji                #  6.1  [1] (😀)       grinning face
-1F601..1F610  ; Emoji                #  6.0 [16] (😁..😐)    beaming face with smiling eyes..neutral face
-1F611         ; Emoji                #  6.1  [1] (😑)       expressionless face
-1F612..1F614  ; Emoji                #  6.0  [3] (😒..😔)    unamused face..pensive face
-1F615         ; Emoji                #  6.1  [1] (😕)       confused face
-1F616         ; Emoji                #  6.0  [1] (😖)       confounded face
-1F617         ; Emoji                #  6.1  [1] (😗)       kissing face
-1F618         ; Emoji                #  6.0  [1] (😘)       face blowing a kiss
-1F619         ; Emoji                #  6.1  [1] (😙)       kissing face with smiling eyes
-1F61A         ; Emoji                #  6.0  [1] (😚)       kissing face with closed eyes
-1F61B         ; Emoji                #  6.1  [1] (😛)       face with tongue
-1F61C..1F61E  ; Emoji                #  6.0  [3] (😜..😞)    winking face with tongue..disappointed face
-1F61F         ; Emoji                #  6.1  [1] (😟)       worried face
-1F620..1F625  ; Emoji                #  6.0  [6] (😠..😥)    angry face..sad but relieved face
-1F626..1F627  ; Emoji                #  6.1  [2] (😦..😧)    frowning face with open mouth..anguished face
-1F628..1F62B  ; Emoji                #  6.0  [4] (😨..😫)    fearful face..tired face
-1F62C         ; Emoji                #  6.1  [1] (😬)       grimacing face
-1F62D         ; Emoji                #  6.0  [1] (😭)       loudly crying face
-1F62E..1F62F  ; Emoji                #  6.1  [2] (😮..😯)    face with open mouth..hushed face
-1F630..1F633  ; Emoji                #  6.0  [4] (😰..😳)    anxious face with sweat..flushed face
-1F634         ; Emoji                #  6.1  [1] (😴)       sleeping face
-1F635..1F640  ; Emoji                #  6.0 [12] (😵..🙀)    dizzy face..weary cat
-1F641..1F642  ; Emoji                #  7.0  [2] (🙁..🙂)    slightly frowning face..slightly smiling face
-1F643..1F644  ; Emoji                #  8.0  [2] (🙃..🙄)    upside-down face..face with rolling eyes
-1F645..1F64F  ; Emoji                #  6.0 [11] (🙅..🙏)    person gesturing NO..folded hands
-1F680..1F6C5  ; Emoji                #  6.0 [70] (🚀..🛅)    rocket..left luggage
-1F6CB..1F6CF  ; Emoji                #  7.0  [5] (🛋️..🛏️)    couch and lamp..bed
-1F6D0         ; Emoji                #  8.0  [1] (🛐)       place of worship
-1F6D1..1F6D2  ; Emoji                #  9.0  [2] (🛑..🛒)    stop sign..shopping cart
-1F6D5         ; Emoji                # 12.0  [1] (🛕)       hindu temple
-1F6E0..1F6E5  ; Emoji                #  7.0  [6] (🛠️..🛥️)    hammer and wrench..motor boat
-1F6E9         ; Emoji                #  7.0  [1] (🛩️)       small airplane
-1F6EB..1F6EC  ; Emoji                #  7.0  [2] (🛫..🛬)    airplane departure..airplane arrival
-1F6F0         ; Emoji                #  7.0  [1] (🛰️)       satellite
-1F6F3         ; Emoji                #  7.0  [1] (🛳️)       passenger ship
-1F6F4..1F6F6  ; Emoji                #  9.0  [3] (🛴..🛶)    kick scooter..canoe
-1F6F7..1F6F8  ; Emoji                # 10.0  [2] (🛷..🛸)    sled..flying saucer
-1F6F9         ; Emoji                # 11.0  [1] (🛹)       skateboard
-1F6FA         ; Emoji                # 12.0  [1] (🛺)       auto rickshaw
-1F7E0..1F7EB  ; Emoji                # 12.0 [12] (🟠..🟫)    orange circle..brown square
-1F90D..1F90F  ; Emoji                # 12.0  [3] (🤍..🤏)    white heart..pinching hand
-1F910..1F918  ; Emoji                #  8.0  [9] (🤐..🤘)    zipper-mouth face..sign of the horns
-1F919..1F91E  ; Emoji                #  9.0  [6] (🤙..🤞)    call me hand..crossed fingers
-1F91F         ; Emoji                # 10.0  [1] (🤟)       love-you gesture
-1F920..1F927  ; Emoji                #  9.0  [8] (🤠..🤧)    cowboy hat face..sneezing face
-1F928..1F92F  ; Emoji                # 10.0  [8] (🤨..🤯)    face with raised eyebrow..exploding head
-1F930         ; Emoji                #  9.0  [1] (🤰)       pregnant woman
-1F931..1F932  ; Emoji                # 10.0  [2] (🤱..🤲)    breast-feeding..palms up together
-1F933..1F93A  ; Emoji                #  9.0  [8] (🤳..🤺)    selfie..person fencing
-1F93C..1F93E  ; Emoji                #  9.0  [3] (🤼..🤾)    people wrestling..person playing handball
-1F93F         ; Emoji                # 12.0  [1] (🤿)       diving mask
-1F940..1F945  ; Emoji                #  9.0  [6] (🥀..🥅)    wilted flower..goal net
-1F947..1F94B  ; Emoji                #  9.0  [5] (🥇..🥋)    1st place medal..martial arts uniform
-1F94C         ; Emoji                # 10.0  [1] (🥌)       curling stone
-1F94D..1F94F  ; Emoji                # 11.0  [3] (🥍..🥏)    lacrosse..flying disc
-1F950..1F95E  ; Emoji                #  9.0 [15] (🥐..🥞)    croissant..pancakes
-1F95F..1F96B  ; Emoji                # 10.0 [13] (🥟..🥫)    dumpling..canned food
-1F96C..1F970  ; Emoji                # 11.0  [5] (🥬..🥰)    leafy green..smiling face with hearts
-1F971         ; Emoji                # 12.0  [1] (🥱)       yawning face
-1F973..1F976  ; Emoji                # 11.0  [4] (🥳..🥶)    partying face..cold face
-1F97A         ; Emoji                # 11.0  [1] (🥺)       pleading face
-1F97B         ; Emoji                # 12.0  [1] (🥻)       sari
-1F97C..1F97F  ; Emoji                # 11.0  [4] (🥼..🥿)    lab coat..flat shoe
-1F980..1F984  ; Emoji                #  8.0  [5] (🦀..🦄)    crab..unicorn
-1F985..1F991  ; Emoji                #  9.0 [13] (🦅..🦑)    eagle..squid
-1F992..1F997  ; Emoji                # 10.0  [6] (🦒..🦗)    giraffe..cricket
-1F998..1F9A2  ; Emoji                # 11.0 [11] (🦘..🦢)    kangaroo..swan
-1F9A5..1F9AA  ; Emoji                # 12.0  [6] (🦥..🦪)    sloth..oyster
-1F9AE..1F9AF  ; Emoji                # 12.0  [2] (🦮..🦯)    guide dog..probing cane
-1F9B0..1F9B9  ; Emoji                # 11.0 [10] (🦰..🦹)    red hair..supervillain
-1F9BA..1F9BF  ; Emoji                # 12.0  [6] (🦺..🦿)    safety vest..mechanical leg
-1F9C0         ; Emoji                #  8.0  [1] (🧀)       cheese wedge
-1F9C1..1F9C2  ; Emoji                # 11.0  [2] (🧁..🧂)    cupcake..salt
-1F9C3..1F9CA  ; Emoji                # 12.0  [8] (🧃..🧊)    beverage box..ice cube
-1F9CD..1F9CF  ; Emoji                # 12.0  [3] (🧍..🧏)    person standing..deaf person
-1F9D0..1F9E6  ; Emoji                # 10.0 [23] (🧐..🧦)    face with monocle..socks
-1F9E7..1F9FF  ; Emoji                # 11.0 [25] (🧧..🧿)    red envelope..nazar amulet
-1FA70..1FA73  ; Emoji                # 12.0  [4] (🩰..🩳)    ballet shoes..shorts
-1FA78..1FA7A  ; Emoji                # 12.0  [3] (🩸..🩺)    drop of blood..stethoscope
-1FA80..1FA82  ; Emoji                # 12.0  [3] (🪀..🪂)    yo-yo..parachute
-1FA90..1FA95  ; Emoji                # 12.0  [6] (🪐..🪕)    ringed planet..banjo
-
-# Total elements: 1311
-
-# ================================================
-
-# All omitted code points have Emoji_Presentation=No 
-# @missing: 0000..10FFFF  ; Emoji_Presentation ; No
-
-231A..231B    ; Emoji_Presentation   #  1.1  [2] (⌚..⌛)    watch..hourglass done
-23E9..23EC    ; Emoji_Presentation   #  6.0  [4] (⏩..⏬)    fast-forward button..fast down button
-23F0          ; Emoji_Presentation   #  6.0  [1] (⏰)       alarm clock
-23F3          ; Emoji_Presentation   #  6.0  [1] (⏳)       hourglass not done
-25FD..25FE    ; Emoji_Presentation   #  3.2  [2] (◽..◾)    white medium-small square..black medium-small square
-2614..2615    ; Emoji_Presentation   #  4.0  [2] (☔..☕)    umbrella with rain drops..hot beverage
-2648..2653    ; Emoji_Presentation   #  1.1 [12] (♈..♓)    Aries..Pisces
-267F          ; Emoji_Presentation   #  4.1  [1] (♿)       wheelchair symbol
-2693          ; Emoji_Presentation   #  4.1  [1] (⚓)       anchor
-26A1          ; Emoji_Presentation   #  4.0  [1] (⚡)       high voltage
-26AA..26AB    ; Emoji_Presentation   #  4.1  [2] (⚪..⚫)    white circle..black circle
-26BD..26BE    ; Emoji_Presentation   #  5.2  [2] (⚽..⚾)    soccer ball..baseball
-26C4..26C5    ; Emoji_Presentation   #  5.2  [2] (⛄..⛅)    snowman without snow..sun behind cloud
-26CE          ; Emoji_Presentation   #  6.0  [1] (⛎)       Ophiuchus
-26D4          ; Emoji_Presentation   #  5.2  [1] (⛔)       no entry
-26EA          ; Emoji_Presentation   #  5.2  [1] (⛪)       church
-26F2..26F3    ; Emoji_Presentation   #  5.2  [2] (⛲..⛳)    fountain..flag in hole
-26F5          ; Emoji_Presentation   #  5.2  [1] (⛵)       sailboat
-26FA          ; Emoji_Presentation   #  5.2  [1] (⛺)       tent
-26FD          ; Emoji_Presentation   #  5.2  [1] (⛽)       fuel pump
-2705          ; Emoji_Presentation   #  6.0  [1] (✅)       check mark button
-270A..270B    ; Emoji_Presentation   #  6.0  [2] (✊..✋)    raised fist..raised hand
-2728          ; Emoji_Presentation   #  6.0  [1] (✨)       sparkles
-274C          ; Emoji_Presentation   #  6.0  [1] (❌)       cross mark
-274E          ; Emoji_Presentation   #  6.0  [1] (❎)       cross mark button
-2753..2755    ; Emoji_Presentation   #  6.0  [3] (❓..❕)    question mark..white exclamation mark
-2757          ; Emoji_Presentation   #  5.2  [1] (❗)       exclamation mark
-2795..2797    ; Emoji_Presentation   #  6.0  [3] (➕..➗)    plus sign..division sign
-27B0          ; Emoji_Presentation   #  6.0  [1] (➰)       curly loop
-27BF          ; Emoji_Presentation   #  6.0  [1] (➿)       double curly loop
-2B1B..2B1C    ; Emoji_Presentation   #  5.1  [2] (⬛..⬜)    black large square..white large square
-2B50          ; Emoji_Presentation   #  5.1  [1] (⭐)       star
-2B55          ; Emoji_Presentation   #  5.2  [1] (⭕)       hollow red circle
-1F004         ; Emoji_Presentation   #  5.1  [1] (🀄)       mahjong red dragon
-1F0CF         ; Emoji_Presentation   #  6.0  [1] (🃏)       joker
-1F18E         ; Emoji_Presentation   #  6.0  [1] (🆎)       AB button (blood type)
-1F191..1F19A  ; Emoji_Presentation   #  6.0 [10] (🆑..🆚)    CL button..VS button
-1F1E6..1F1FF  ; Emoji_Presentation   #  6.0 [26] (🇦..🇿)    regional indicator symbol letter a..regional indicator symbol letter z
-1F201         ; Emoji_Presentation   #  6.0  [1] (🈁)       Japanese “here” button
-1F21A         ; Emoji_Presentation   #  5.2  [1] (🈚)       Japanese “free of charge” button
-1F22F         ; Emoji_Presentation   #  5.2  [1] (🈯)       Japanese “reserved” button
-1F232..1F236  ; Emoji_Presentation   #  6.0  [5] (🈲..🈶)    Japanese “prohibited” button..Japanese “not free of charge” button
-1F238..1F23A  ; Emoji_Presentation   #  6.0  [3] (🈸..🈺)    Japanese “application” button..Japanese “open for business” button
-1F250..1F251  ; Emoji_Presentation   #  6.0  [2] (🉐..🉑)    Japanese “bargain” button..Japanese “acceptable” button
-1F300..1F320  ; Emoji_Presentation   #  6.0 [33] (🌀..🌠)    cyclone..shooting star
-1F32D..1F32F  ; Emoji_Presentation   #  8.0  [3] (🌭..🌯)    hot dog..burrito
-1F330..1F335  ; Emoji_Presentation   #  6.0  [6] (🌰..🌵)    chestnut..cactus
-1F337..1F37C  ; Emoji_Presentation   #  6.0 [70] (🌷..🍼)    tulip..baby bottle
-1F37E..1F37F  ; Emoji_Presentation   #  8.0  [2] (🍾..🍿)    bottle with popping cork..popcorn
-1F380..1F393  ; Emoji_Presentation   #  6.0 [20] (🎀..🎓)    ribbon..graduation cap
-1F3A0..1F3C4  ; Emoji_Presentation   #  6.0 [37] (🎠..🏄)    carousel horse..person surfing
-1F3C5         ; Emoji_Presentation   #  7.0  [1] (🏅)       sports medal
-1F3C6..1F3CA  ; Emoji_Presentation   #  6.0  [5] (🏆..🏊)    trophy..person swimming
-1F3CF..1F3D3  ; Emoji_Presentation   #  8.0  [5] (🏏..🏓)    cricket game..ping pong
-1F3E0..1F3F0  ; Emoji_Presentation   #  6.0 [17] (🏠..🏰)    house..castle
-1F3F4         ; Emoji_Presentation   #  7.0  [1] (🏴)       black flag
-1F3F8..1F3FF  ; Emoji_Presentation   #  8.0  [8] (🏸..🏿)    badminton..dark skin tone
-1F400..1F43E  ; Emoji_Presentation   #  6.0 [63] (🐀..🐾)    rat..paw prints
-1F440         ; Emoji_Presentation   #  6.0  [1] (👀)       eyes
-1F442..1F4F7  ; Emoji_Presentation   #  6.0[182] (👂..📷)    ear..camera
-1F4F8         ; Emoji_Presentation   #  7.0  [1] (📸)       camera with flash
-1F4F9..1F4FC  ; Emoji_Presentation   #  6.0  [4] (📹..📼)    video camera..videocassette
-1F4FF         ; Emoji_Presentation   #  8.0  [1] (📿)       prayer beads
-1F500..1F53D  ; Emoji_Presentation   #  6.0 [62] (🔀..🔽)    shuffle tracks button..downwards button
-1F54B..1F54E  ; Emoji_Presentation   #  8.0  [4] (🕋..🕎)    kaaba..menorah
-1F550..1F567  ; Emoji_Presentation   #  6.0 [24] (🕐..🕧)    one o’clock..twelve-thirty
-1F57A         ; Emoji_Presentation   #  9.0  [1] (🕺)       man dancing
-1F595..1F596  ; Emoji_Presentation   #  7.0  [2] (🖕..🖖)    middle finger..vulcan salute
-1F5A4         ; Emoji_Presentation   #  9.0  [1] (🖤)       black heart
-1F5FB..1F5FF  ; Emoji_Presentation   #  6.0  [5] (🗻..🗿)    mount fuji..moai
-1F600         ; Emoji_Presentation   #  6.1  [1] (😀)       grinning face
-1F601..1F610  ; Emoji_Presentation   #  6.0 [16] (😁..😐)    beaming face with smiling eyes..neutral face
-1F611         ; Emoji_Presentation   #  6.1  [1] (😑)       expressionless face
-1F612..1F614  ; Emoji_Presentation   #  6.0  [3] (😒..😔)    unamused face..pensive face
-1F615         ; Emoji_Presentation   #  6.1  [1] (😕)       confused face
-1F616         ; Emoji_Presentation   #  6.0  [1] (😖)       confounded face
-1F617         ; Emoji_Presentation   #  6.1  [1] (😗)       kissing face
-1F618         ; Emoji_Presentation   #  6.0  [1] (😘)       face blowing a kiss
-1F619         ; Emoji_Presentation   #  6.1  [1] (😙)       kissing face with smiling eyes
-1F61A         ; Emoji_Presentation   #  6.0  [1] (😚)       kissing face with closed eyes
-1F61B         ; Emoji_Presentation   #  6.1  [1] (😛)       face with tongue
-1F61C..1F61E  ; Emoji_Presentation   #  6.0  [3] (😜..😞)    winking face with tongue..disappointed face
-1F61F         ; Emoji_Presentation   #  6.1  [1] (😟)       worried face
-1F620..1F625  ; Emoji_Presentation   #  6.0  [6] (😠..😥)    angry face..sad but relieved face
-1F626..1F627  ; Emoji_Presentation   #  6.1  [2] (😦..😧)    frowning face with open mouth..anguished face
-1F628..1F62B  ; Emoji_Presentation   #  6.0  [4] (😨..😫)    fearful face..tired face
-1F62C         ; Emoji_Presentation   #  6.1  [1] (😬)       grimacing face
-1F62D         ; Emoji_Presentation   #  6.0  [1] (😭)       loudly crying face
-1F62E..1F62F  ; Emoji_Presentation   #  6.1  [2] (😮..😯)    face with open mouth..hushed face
-1F630..1F633  ; Emoji_Presentation   #  6.0  [4] (😰..😳)    anxious face with sweat..flushed face
-1F634         ; Emoji_Presentation   #  6.1  [1] (😴)       sleeping face
-1F635..1F640  ; Emoji_Presentation   #  6.0 [12] (😵..🙀)    dizzy face..weary cat
-1F641..1F642  ; Emoji_Presentation   #  7.0  [2] (🙁..🙂)    slightly frowning face..slightly smiling face
-1F643..1F644  ; Emoji_Presentation   #  8.0  [2] (🙃..🙄)    upside-down face..face with rolling eyes
-1F645..1F64F  ; Emoji_Presentation   #  6.0 [11] (🙅..🙏)    person gesturing NO..folded hands
-1F680..1F6C5  ; Emoji_Presentation   #  6.0 [70] (🚀..🛅)    rocket..left luggage
-1F6CC         ; Emoji_Presentation   #  7.0  [1] (🛌)       person in bed
-1F6D0         ; Emoji_Presentation   #  8.0  [1] (🛐)       place of worship
-1F6D1..1F6D2  ; Emoji_Presentation   #  9.0  [2] (🛑..🛒)    stop sign..shopping cart
-1F6D5         ; Emoji_Presentation   # 12.0  [1] (🛕)       hindu temple
-1F6EB..1F6EC  ; Emoji_Presentation   #  7.0  [2] (🛫..🛬)    airplane departure..airplane arrival
-1F6F4..1F6F6  ; Emoji_Presentation   #  9.0  [3] (🛴..🛶)    kick scooter..canoe
-1F6F7..1F6F8  ; Emoji_Presentation   # 10.0  [2] (🛷..🛸)    sled..flying saucer
-1F6F9         ; Emoji_Presentation   # 11.0  [1] (🛹)       skateboard
-1F6FA         ; Emoji_Presentation   # 12.0  [1] (🛺)       auto rickshaw
-1F7E0..1F7EB  ; Emoji_Presentation   # 12.0 [12] (🟠..🟫)    orange circle..brown square
-1F90D..1F90F  ; Emoji_Presentation   # 12.0  [3] (🤍..🤏)    white heart..pinching hand
-1F910..1F918  ; Emoji_Presentation   #  8.0  [9] (🤐..🤘)    zipper-mouth face..sign of the horns
-1F919..1F91E  ; Emoji_Presentation   #  9.0  [6] (🤙..🤞)    call me hand..crossed fingers
-1F91F         ; Emoji_Presentation   # 10.0  [1] (🤟)       love-you gesture
-1F920..1F927  ; Emoji_Presentation   #  9.0  [8] (🤠..🤧)    cowboy hat face..sneezing face
-1F928..1F92F  ; Emoji_Presentation   # 10.0  [8] (🤨..🤯)    face with raised eyebrow..exploding head
-1F930         ; Emoji_Presentation   #  9.0  [1] (🤰)       pregnant woman
-1F931..1F932  ; Emoji_Presentation   # 10.0  [2] (🤱..🤲)    breast-feeding..palms up together
-1F933..1F93A  ; Emoji_Presentation   #  9.0  [8] (🤳..🤺)    selfie..person fencing
-1F93C..1F93E  ; Emoji_Presentation   #  9.0  [3] (🤼..🤾)    people wrestling..person playing handball
-1F93F         ; Emoji_Presentation   # 12.0  [1] (🤿)       diving mask
-1F940..1F945  ; Emoji_Presentation   #  9.0  [6] (🥀..🥅)    wilted flower..goal net
-1F947..1F94B  ; Emoji_Presentation   #  9.0  [5] (🥇..🥋)    1st place medal..martial arts uniform
-1F94C         ; Emoji_Presentation   # 10.0  [1] (🥌)       curling stone
-1F94D..1F94F  ; Emoji_Presentation   # 11.0  [3] (🥍..🥏)    lacrosse..flying disc
-1F950..1F95E  ; Emoji_Presentation   #  9.0 [15] (🥐..🥞)    croissant..pancakes
-1F95F..1F96B  ; Emoji_Presentation   # 10.0 [13] (🥟..🥫)    dumpling..canned food
-1F96C..1F970  ; Emoji_Presentation   # 11.0  [5] (🥬..🥰)    leafy green..smiling face with hearts
-1F971         ; Emoji_Presentation   # 12.0  [1] (🥱)       yawning face
-1F973..1F976  ; Emoji_Presentation   # 11.0  [4] (🥳..🥶)    partying face..cold face
-1F97A         ; Emoji_Presentation   # 11.0  [1] (🥺)       pleading face
-1F97B         ; Emoji_Presentation   # 12.0  [1] (🥻)       sari
-1F97C..1F97F  ; Emoji_Presentation   # 11.0  [4] (🥼..🥿)    lab coat..flat shoe
-1F980..1F984  ; Emoji_Presentation   #  8.0  [5] (🦀..🦄)    crab..unicorn
-1F985..1F991  ; Emoji_Presentation   #  9.0 [13] (🦅..🦑)    eagle..squid
-1F992..1F997  ; Emoji_Presentation   # 10.0  [6] (🦒..🦗)    giraffe..cricket
-1F998..1F9A2  ; Emoji_Presentation   # 11.0 [11] (🦘..🦢)    kangaroo..swan
-1F9A5..1F9AA  ; Emoji_Presentation   # 12.0  [6] (🦥..🦪)    sloth..oyster
-1F9AE..1F9AF  ; Emoji_Presentation   # 12.0  [2] (🦮..🦯)    guide dog..probing cane
-1F9B0..1F9B9  ; Emoji_Presentation   # 11.0 [10] (🦰..🦹)    red hair..supervillain
-1F9BA..1F9BF  ; Emoji_Presentation   # 12.0  [6] (🦺..🦿)    safety vest..mechanical leg
-1F9C0         ; Emoji_Presentation   #  8.0  [1] (🧀)       cheese wedge
-1F9C1..1F9C2  ; Emoji_Presentation   # 11.0  [2] (🧁..🧂)    cupcake..salt
-1F9C3..1F9CA  ; Emoji_Presentation   # 12.0  [8] (🧃..🧊)    beverage box..ice cube
-1F9CD..1F9CF  ; Emoji_Presentation   # 12.0  [3] (🧍..🧏)    person standing..deaf person
-1F9D0..1F9E6  ; Emoji_Presentation   # 10.0 [23] (🧐..🧦)    face with monocle..socks
-1F9E7..1F9FF  ; Emoji_Presentation   # 11.0 [25] (🧧..🧿)    red envelope..nazar amulet
-1FA70..1FA73  ; Emoji_Presentation   # 12.0  [4] (🩰..🩳)    ballet shoes..shorts
-1FA78..1FA7A  ; Emoji_Presentation   # 12.0  [3] (🩸..🩺)    drop of blood..stethoscope
-1FA80..1FA82  ; Emoji_Presentation   # 12.0  [3] (🪀..🪂)    yo-yo..parachute
-1FA90..1FA95  ; Emoji_Presentation   # 12.0  [6] (🪐..🪕)    ringed planet..banjo
-
-# Total elements: 1093
-
-# ================================================
-
-# All omitted code points have Emoji_Modifier=No 
-# @missing: 0000..10FFFF  ; Emoji_Modifier ; No
-
-1F3FB..1F3FF  ; Emoji_Modifier       #  8.0  [5] (🏻..🏿)    light skin tone..dark skin tone
-
-# Total elements: 5
-
-# ================================================
-
-# All omitted code points have Emoji_Modifier_Base=No 
-# @missing: 0000..10FFFF  ; Emoji_Modifier_Base ; No
-
-261D          ; Emoji_Modifier_Base  #  1.1  [1] (☝️)       index pointing up
-26F9          ; Emoji_Modifier_Base  #  5.2  [1] (⛹️)       person bouncing ball
-270A..270B    ; Emoji_Modifier_Base  #  6.0  [2] (✊..✋)    raised fist..raised hand
-270C..270D    ; Emoji_Modifier_Base  #  1.1  [2] (✌️..✍️)    victory hand..writing hand
-1F385         ; Emoji_Modifier_Base  #  6.0  [1] (🎅)       Santa Claus
-1F3C2..1F3C4  ; Emoji_Modifier_Base  #  6.0  [3] (🏂..🏄)    snowboarder..person surfing
-1F3C7         ; Emoji_Modifier_Base  #  6.0  [1] (🏇)       horse racing
-1F3CA         ; Emoji_Modifier_Base  #  6.0  [1] (🏊)       person swimming
-1F3CB..1F3CC  ; Emoji_Modifier_Base  #  7.0  [2] (🏋️..🏌️)    person lifting weights..person golfing
-1F442..1F443  ; Emoji_Modifier_Base  #  6.0  [2] (👂..👃)    ear..nose
-1F446..1F450  ; Emoji_Modifier_Base  #  6.0 [11] (👆..👐)    backhand index pointing up..open hands
-1F466..1F478  ; Emoji_Modifier_Base  #  6.0 [19] (👦..👸)    boy..princess
-1F47C         ; Emoji_Modifier_Base  #  6.0  [1] (👼)       baby angel
-1F481..1F483  ; Emoji_Modifier_Base  #  6.0  [3] (💁..💃)    person tipping hand..woman dancing
-1F485..1F487  ; Emoji_Modifier_Base  #  6.0  [3] (💅..💇)    nail polish..person getting haircut
-1F48F         ; Emoji_Modifier_Base  #  6.0  [1] (💏)       kiss
-1F491         ; Emoji_Modifier_Base  #  6.0  [1] (💑)       couple with heart
-1F4AA         ; Emoji_Modifier_Base  #  6.0  [1] (💪)       flexed biceps
-1F574..1F575  ; Emoji_Modifier_Base  #  7.0  [2] (🕴️..🕵️)    man in suit levitating..detective
-1F57A         ; Emoji_Modifier_Base  #  9.0  [1] (🕺)       man dancing
-1F590         ; Emoji_Modifier_Base  #  7.0  [1] (🖐️)       hand with fingers splayed
-1F595..1F596  ; Emoji_Modifier_Base  #  7.0  [2] (🖕..🖖)    middle finger..vulcan salute
-1F645..1F647  ; Emoji_Modifier_Base  #  6.0  [3] (🙅..🙇)    person gesturing NO..person bowing
-1F64B..1F64F  ; Emoji_Modifier_Base  #  6.0  [5] (🙋..🙏)    person raising hand..folded hands
-1F6A3         ; Emoji_Modifier_Base  #  6.0  [1] (🚣)       person rowing boat
-1F6B4..1F6B6  ; Emoji_Modifier_Base  #  6.0  [3] (🚴..🚶)    person biking..person walking
-1F6C0         ; Emoji_Modifier_Base  #  6.0  [1] (🛀)       person taking bath
-1F6CC         ; Emoji_Modifier_Base  #  7.0  [1] (🛌)       person in bed
-1F90F         ; Emoji_Modifier_Base  # 12.0  [1] (🤏)       pinching hand
-1F918         ; Emoji_Modifier_Base  #  8.0  [1] (🤘)       sign of the horns
-1F919..1F91E  ; Emoji_Modifier_Base  #  9.0  [6] (🤙..🤞)    call me hand..crossed fingers
-1F91F         ; Emoji_Modifier_Base  # 10.0  [1] (🤟)       love-you gesture
-1F926         ; Emoji_Modifier_Base  #  9.0  [1] (🤦)       person facepalming
-1F930         ; Emoji_Modifier_Base  #  9.0  [1] (🤰)       pregnant woman
-1F931..1F932  ; Emoji_Modifier_Base  # 10.0  [2] (🤱..🤲)    breast-feeding..palms up together
-1F933..1F939  ; Emoji_Modifier_Base  #  9.0  [7] (🤳..🤹)    selfie..person juggling
-1F93C..1F93E  ; Emoji_Modifier_Base  #  9.0  [3] (🤼..🤾)    people wrestling..person playing handball
-1F9B5..1F9B6  ; Emoji_Modifier_Base  # 11.0  [2] (🦵..🦶)    leg..foot
-1F9B8..1F9B9  ; Emoji_Modifier_Base  # 11.0  [2] (🦸..🦹)    superhero..supervillain
-1F9BB         ; Emoji_Modifier_Base  # 12.0  [1] (🦻)       ear with hearing aid
-1F9CD..1F9CF  ; Emoji_Modifier_Base  # 12.0  [3] (🧍..🧏)    person standing..deaf person
-1F9D1..1F9DD  ; Emoji_Modifier_Base  # 10.0 [13] (🧑..🧝)    person..elf
-
-# Total elements: 120
-
-# ================================================
-
-# All omitted code points have Emoji_Component=No 
-# @missing: 0000..10FFFF  ; Emoji_Component ; No
-
-0023          ; Emoji_Component      #  1.1  [1] (#️)       number sign
-002A          ; Emoji_Component      #  1.1  [1] (*️)       asterisk
-0030..0039    ; Emoji_Component      #  1.1 [10] (0️..9️)    digit zero..digit nine
-200D          ; Emoji_Component      #  1.1  [1] (‍)        zero width joiner
-20E3          ; Emoji_Component      #  3.0  [1] (⃣)       combining enclosing keycap
-FE0F          ; Emoji_Component      #  3.2  [1] ()        VARIATION SELECTOR-16
-1F1E6..1F1FF  ; Emoji_Component      #  6.0 [26] (🇦..🇿)    regional indicator symbol letter a..regional indicator symbol letter z
-1F3FB..1F3FF  ; Emoji_Component      #  8.0  [5] (🏻..🏿)    light skin tone..dark skin tone
-1F9B0..1F9B3  ; Emoji_Component      # 11.0  [4] (🦰..🦳)    red hair..white hair
-E0020..E007F  ; Emoji_Component      #  3.1 [96] (󠀠..󠁿)      tag space..cancel tag
-
-# Total elements: 146
-
-# ================================================
-
-# All omitted code points have Extended_Pictographic=No 
-# @missing: 0000..10FFFF  ; Extended_Pictographic ; No
-
-00A9          ; Extended_Pictographic#  1.1  [1] (©️)       copyright
-00AE          ; Extended_Pictographic#  1.1  [1] (®️)       registered
-203C          ; Extended_Pictographic#  1.1  [1] (‼️)       double exclamation mark
-2049          ; Extended_Pictographic#  3.0  [1] (⁉️)       exclamation question mark
-2122          ; Extended_Pictographic#  1.1  [1] (™️)       trade mark
-2139          ; Extended_Pictographic#  3.0  [1] (ℹ️)       information
-2194..2199    ; Extended_Pictographic#  1.1  [6] (↔️..↙️)    left-right arrow..down-left arrow
-21A9..21AA    ; Extended_Pictographic#  1.1  [2] (↩️..↪️)    right arrow curving left..left arrow curving right
-231A..231B    ; Extended_Pictographic#  1.1  [2] (⌚..⌛)    watch..hourglass done
-2328          ; Extended_Pictographic#  1.1  [1] (⌨️)       keyboard
-2388          ; Extended_Pictographic#  3.0  [1] (⎈)       HELM SYMBOL
-23CF          ; Extended_Pictographic#  4.0  [1] (⏏️)       eject button
-23E9..23F3    ; Extended_Pictographic#  6.0 [11] (⏩..⏳)    fast-forward button..hourglass not done
-23F8..23FA    ; Extended_Pictographic#  7.0  [3] (⏸️..⏺️)    pause button..record button
-24C2          ; Extended_Pictographic#  1.1  [1] (Ⓜ️)       circled M
-25AA..25AB    ; Extended_Pictographic#  1.1  [2] (▪️..▫️)    black small square..white small square
-25B6          ; Extended_Pictographic#  1.1  [1] (▶️)       play button
-25C0          ; Extended_Pictographic#  1.1  [1] (◀️)       reverse button
-25FB..25FE    ; Extended_Pictographic#  3.2  [4] (◻️..◾)    white medium square..black medium-small square
-2600..2605    ; Extended_Pictographic#  1.1  [6] (☀️..★)    sun..BLACK STAR
-2607..2612    ; Extended_Pictographic#  1.1 [12] (☇..☒)    LIGHTNING..BALLOT BOX WITH X
-2614..2615    ; Extended_Pictographic#  4.0  [2] (☔..☕)    umbrella with rain drops..hot beverage
-2616..2617    ; Extended_Pictographic#  3.2  [2] (☖..☗)    WHITE SHOGI PIECE..BLACK SHOGI PIECE
-2618          ; Extended_Pictographic#  4.1  [1] (☘️)       shamrock
-2619          ; Extended_Pictographic#  3.0  [1] (☙)       REVERSED ROTATED FLORAL HEART BULLET
-261A..266F    ; Extended_Pictographic#  1.1 [86] (☚..♯)    BLACK LEFT POINTING INDEX..MUSIC SHARP SIGN
-2670..2671    ; Extended_Pictographic#  3.0  [2] (♰..♱)    WEST SYRIAC CROSS..EAST SYRIAC CROSS
-2672..267D    ; Extended_Pictographic#  3.2 [12] (♲..♽)    UNIVERSAL RECYCLING SYMBOL..PARTIALLY-RECYCLED PAPER SYMBOL
-267E..267F    ; Extended_Pictographic#  4.1  [2] (♾️..♿)    infinity..wheelchair symbol
-2680..2685    ; Extended_Pictographic#  3.2  [6] (⚀..⚅)    DIE FACE-1..DIE FACE-6
-2690..2691    ; Extended_Pictographic#  4.0  [2] (⚐..⚑)    WHITE FLAG..BLACK FLAG
-2692..269C    ; Extended_Pictographic#  4.1 [11] (⚒️..⚜️)    hammer and pick..fleur-de-lis
-269D          ; Extended_Pictographic#  5.1  [1] (⚝)       OUTLINED WHITE STAR
-269E..269F    ; Extended_Pictographic#  5.2  [2] (⚞..⚟)    THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT
-26A0..26A1    ; Extended_Pictographic#  4.0  [2] (⚠️..⚡)    warning..high voltage
-26A2..26B1    ; Extended_Pictographic#  4.1 [16] (⚢..⚱️)    DOUBLED FEMALE SIGN..funeral urn
-26B2          ; Extended_Pictographic#  5.0  [1] (⚲)       NEUTER
-26B3..26BC    ; Extended_Pictographic#  5.1 [10] (⚳..⚼)    CERES..SESQUIQUADRATE
-26BD..26BF    ; Extended_Pictographic#  5.2  [3] (⚽..⚿)    soccer ball..SQUARED KEY
-26C0..26C3    ; Extended_Pictographic#  5.1  [4] (⛀..⛃)    WHITE DRAUGHTS MAN..BLACK DRAUGHTS KING
-26C4..26CD    ; Extended_Pictographic#  5.2 [10] (⛄..⛍)    snowman without snow..DISABLED CAR
-26CE          ; Extended_Pictographic#  6.0  [1] (⛎)       Ophiuchus
-26CF..26E1    ; Extended_Pictographic#  5.2 [19] (⛏️..⛡)    pick..RESTRICTED LEFT ENTRY-2
-26E2          ; Extended_Pictographic#  6.0  [1] (⛢)       ASTRONOMICAL SYMBOL FOR URANUS
-26E3          ; Extended_Pictographic#  5.2  [1] (⛣)       HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE
-26E4..26E7    ; Extended_Pictographic#  6.0  [4] (⛤..⛧)    PENTAGRAM..INVERTED PENTAGRAM
-26E8..26FF    ; Extended_Pictographic#  5.2 [24] (⛨..⛿)    BLACK CROSS ON SHIELD..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE
-2700          ; Extended_Pictographic#  7.0  [1] (✀)       BLACK SAFETY SCISSORS
-2701..2704    ; Extended_Pictographic#  1.1  [4] (✁..✄)    UPPER BLADE SCISSORS..WHITE SCISSORS
-2705          ; Extended_Pictographic#  6.0  [1] (✅)       check mark button
-2708..2709    ; Extended_Pictographic#  1.1  [2] (✈️..✉️)    airplane..envelope
-270A..270B    ; Extended_Pictographic#  6.0  [2] (✊..✋)    raised fist..raised hand
-270C..2712    ; Extended_Pictographic#  1.1  [7] (✌️..✒️)    victory hand..black nib
-2714          ; Extended_Pictographic#  1.1  [1] (✔️)       check mark
-2716          ; Extended_Pictographic#  1.1  [1] (✖️)       multiplication sign
-271D          ; Extended_Pictographic#  1.1  [1] (✝️)       latin cross
-2721          ; Extended_Pictographic#  1.1  [1] (✡️)       star of David
-2728          ; Extended_Pictographic#  6.0  [1] (✨)       sparkles
-2733..2734    ; Extended_Pictographic#  1.1  [2] (✳️..✴️)    eight-spoked asterisk..eight-pointed star
-2744          ; Extended_Pictographic#  1.1  [1] (❄️)       snowflake
-2747          ; Extended_Pictographic#  1.1  [1] (❇️)       sparkle
-274C          ; Extended_Pictographic#  6.0  [1] (❌)       cross mark
-274E          ; Extended_Pictographic#  6.0  [1] (❎)       cross mark button
-2753..2755    ; Extended_Pictographic#  6.0  [3] (❓..❕)    question mark..white exclamation mark
-2757          ; Extended_Pictographic#  5.2  [1] (❗)       exclamation mark
-2763..2767    ; Extended_Pictographic#  1.1  [5] (❣️..❧)    heart exclamation..ROTATED FLORAL HEART BULLET
-2795..2797    ; Extended_Pictographic#  6.0  [3] (➕..➗)    plus sign..division sign
-27A1          ; Extended_Pictographic#  1.1  [1] (➡️)       right arrow
-27B0          ; Extended_Pictographic#  6.0  [1] (➰)       curly loop
-27BF          ; Extended_Pictographic#  6.0  [1] (➿)       double curly loop
-2934..2935    ; Extended_Pictographic#  3.2  [2] (⤴️..⤵️)    right arrow curving up..right arrow curving down
-2B05..2B07    ; Extended_Pictographic#  4.0  [3] (⬅️..⬇️)    left arrow..down arrow
-2B1B..2B1C    ; Extended_Pictographic#  5.1  [2] (⬛..⬜)    black large square..white large square
-2B50          ; Extended_Pictographic#  5.1  [1] (⭐)       star
-2B55          ; Extended_Pictographic#  5.2  [1] (⭕)       hollow red circle
-3030          ; Extended_Pictographic#  1.1  [1] (〰️)       wavy dash
-303D          ; Extended_Pictographic#  3.2  [1] (〽️)       part alternation mark
-3297          ; Extended_Pictographic#  1.1  [1] (㊗️)       Japanese “congratulations” button
-3299          ; Extended_Pictographic#  1.1  [1] (㊙️)       Japanese “secret” button
-1F000..1F02B  ; Extended_Pictographic#  5.1 [44] (🀀..🀫)    MAHJONG TILE EAST WIND..MAHJONG TILE BACK
-1F02C..1F02F  ; Extended_Pictographic#   NA  [4] (🀬..🀯)    <reserved-1F02C>..<reserved-1F02F>
-1F030..1F093  ; Extended_Pictographic#  5.1[100] (🀰..🂓)    DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06
-1F094..1F09F  ; Extended_Pictographic#   NA [12] (🂔..🂟)    <reserved-1F094>..<reserved-1F09F>
-1F0A0..1F0AE  ; Extended_Pictographic#  6.0 [15] (🂠..🂮)    PLAYING CARD BACK..PLAYING CARD KING OF SPADES
-1F0AF..1F0B0  ; Extended_Pictographic#   NA  [2] (🂯..🂰)    <reserved-1F0AF>..<reserved-1F0B0>
-1F0B1..1F0BE  ; Extended_Pictographic#  6.0 [14] (🂱..🂾)    PLAYING CARD ACE OF HEARTS..PLAYING CARD KING OF HEARTS
-1F0BF         ; Extended_Pictographic#  7.0  [1] (🂿)       PLAYING CARD RED JOKER
-1F0C0         ; Extended_Pictographic#   NA  [1] (🃀)       <reserved-1F0C0>
-1F0C1..1F0CF  ; Extended_Pictographic#  6.0 [15] (🃁..🃏)    PLAYING CARD ACE OF DIAMONDS..joker
-1F0D0         ; Extended_Pictographic#   NA  [1] (🃐)       <reserved-1F0D0>
-1F0D1..1F0DF  ; Extended_Pictographic#  6.0 [15] (🃑..🃟)    PLAYING CARD ACE OF CLUBS..PLAYING CARD WHITE JOKER
-1F0E0..1F0F5  ; Extended_Pictographic#  7.0 [22] (🃠..🃵)    PLAYING CARD FOOL..PLAYING CARD TRUMP-21
-1F0F6..1F0FF  ; Extended_Pictographic#   NA [10] (🃶..🃿)    <reserved-1F0F6>..<reserved-1F0FF>
-1F10D..1F10F  ; Extended_Pictographic#   NA  [3] (🄍..🄏)    <reserved-1F10D>..<reserved-1F10F>
-1F12F         ; Extended_Pictographic# 11.0  [1] (🄯)       COPYLEFT SYMBOL
-1F16C         ; Extended_Pictographic# 12.0  [1] (🅬)       RAISED MR SIGN
-1F16D..1F16F  ; Extended_Pictographic#   NA  [3] (🅭..🅯)    <reserved-1F16D>..<reserved-1F16F>
-1F170..1F171  ; Extended_Pictographic#  6.0  [2] (🅰️..🅱️)    A button (blood type)..B button (blood type)
-1F17E         ; Extended_Pictographic#  6.0  [1] (🅾️)       O button (blood type)
-1F17F         ; Extended_Pictographic#  5.2  [1] (🅿️)       P button
-1F18E         ; Extended_Pictographic#  6.0  [1] (🆎)       AB button (blood type)
-1F191..1F19A  ; Extended_Pictographic#  6.0 [10] (🆑..🆚)    CL button..VS button
-1F1AD..1F1E5  ; Extended_Pictographic#   NA [57] (🆭..🇥)    <reserved-1F1AD>..<reserved-1F1E5>
-1F201..1F202  ; Extended_Pictographic#  6.0  [2] (🈁..🈂️)    Japanese “here” button..Japanese “service charge” button
-1F203..1F20F  ; Extended_Pictographic#   NA [13] (🈃..🈏)    <reserved-1F203>..<reserved-1F20F>
-1F21A         ; Extended_Pictographic#  5.2  [1] (🈚)       Japanese “free of charge” button
-1F22F         ; Extended_Pictographic#  5.2  [1] (🈯)       Japanese “reserved” button
-1F232..1F23A  ; Extended_Pictographic#  6.0  [9] (🈲..🈺)    Japanese “prohibited” button..Japanese “open for business” button
-1F23C..1F23F  ; Extended_Pictographic#   NA  [4] (🈼..🈿)    <reserved-1F23C>..<reserved-1F23F>
-1F249..1F24F  ; Extended_Pictographic#   NA  [7] (🉉..🉏)    <reserved-1F249>..<reserved-1F24F>
-1F250..1F251  ; Extended_Pictographic#  6.0  [2] (🉐..🉑)    Japanese “bargain” button..Japanese “acceptable” button
-1F252..1F25F  ; Extended_Pictographic#   NA [14] (🉒..🉟)    <reserved-1F252>..<reserved-1F25F>
-1F260..1F265  ; Extended_Pictographic# 10.0  [6] (🉠..🉥)    ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI
-1F266..1F2FF  ; Extended_Pictographic#   NA[154] (🉦..🋿)    <reserved-1F266>..<reserved-1F2FF>
-1F300..1F320  ; Extended_Pictographic#  6.0 [33] (🌀..🌠)    cyclone..shooting star
-1F321..1F32C  ; Extended_Pictographic#  7.0 [12] (🌡️..🌬️)    thermometer..wind face
-1F32D..1F32F  ; Extended_Pictographic#  8.0  [3] (🌭..🌯)    hot dog..burrito
-1F330..1F335  ; Extended_Pictographic#  6.0  [6] (🌰..🌵)    chestnut..cactus
-1F336         ; Extended_Pictographic#  7.0  [1] (🌶️)       hot pepper
-1F337..1F37C  ; Extended_Pictographic#  6.0 [70] (🌷..🍼)    tulip..baby bottle
-1F37D         ; Extended_Pictographic#  7.0  [1] (🍽️)       fork and knife with plate
-1F37E..1F37F  ; Extended_Pictographic#  8.0  [2] (🍾..🍿)    bottle with popping cork..popcorn
-1F380..1F393  ; Extended_Pictographic#  6.0 [20] (🎀..🎓)    ribbon..graduation cap
-1F394..1F39F  ; Extended_Pictographic#  7.0 [12] (🎔..🎟️)    HEART WITH TIP ON THE LEFT..admission tickets
-1F3A0..1F3C4  ; Extended_Pictographic#  6.0 [37] (🎠..🏄)    carousel horse..person surfing
-1F3C5         ; Extended_Pictographic#  7.0  [1] (🏅)       sports medal
-1F3C6..1F3CA  ; Extended_Pictographic#  6.0  [5] (🏆..🏊)    trophy..person swimming
-1F3CB..1F3CE  ; Extended_Pictographic#  7.0  [4] (🏋️..🏎️)    person lifting weights..racing car
-1F3CF..1F3D3  ; Extended_Pictographic#  8.0  [5] (🏏..🏓)    cricket game..ping pong
-1F3D4..1F3DF  ; Extended_Pictographic#  7.0 [12] (🏔️..🏟️)    snow-capped mountain..stadium
-1F3E0..1F3F0  ; Extended_Pictographic#  6.0 [17] (🏠..🏰)    house..castle
-1F3F1..1F3F7  ; Extended_Pictographic#  7.0  [7] (🏱..🏷️)    WHITE PENNANT..label
-1F3F8..1F3FA  ; Extended_Pictographic#  8.0  [3] (🏸..🏺)    badminton..amphora
-1F400..1F43E  ; Extended_Pictographic#  6.0 [63] (🐀..🐾)    rat..paw prints
-1F43F         ; Extended_Pictographic#  7.0  [1] (🐿️)       chipmunk
-1F440         ; Extended_Pictographic#  6.0  [1] (👀)       eyes
-1F441         ; Extended_Pictographic#  7.0  [1] (👁️)       eye
-1F442..1F4F7  ; Extended_Pictographic#  6.0[182] (👂..📷)    ear..camera
-1F4F8         ; Extended_Pictographic#  7.0  [1] (📸)       camera with flash
-1F4F9..1F4FC  ; Extended_Pictographic#  6.0  [4] (📹..📼)    video camera..videocassette
-1F4FD..1F4FE  ; Extended_Pictographic#  7.0  [2] (📽️..📾)    film projector..PORTABLE STEREO
-1F4FF         ; Extended_Pictographic#  8.0  [1] (📿)       prayer beads
-1F500..1F53D  ; Extended_Pictographic#  6.0 [62] (🔀..🔽)    shuffle tracks button..downwards button
-1F546..1F54A  ; Extended_Pictographic#  7.0  [5] (🕆..🕊️)    WHITE LATIN CROSS..dove
-1F54B..1F54F  ; Extended_Pictographic#  8.0  [5] (🕋..🕏)    kaaba..BOWL OF HYGIEIA
-1F550..1F567  ; Extended_Pictographic#  6.0 [24] (🕐..🕧)    one o’clock..twelve-thirty
-1F568..1F579  ; Extended_Pictographic#  7.0 [18] (🕨..🕹️)    RIGHT SPEAKER..joystick
-1F57A         ; Extended_Pictographic#  9.0  [1] (🕺)       man dancing
-1F57B..1F5A3  ; Extended_Pictographic#  7.0 [41] (🕻..🖣)    LEFT HAND TELEPHONE RECEIVER..BLACK DOWN POINTING BACKHAND INDEX
-1F5A4         ; Extended_Pictographic#  9.0  [1] (🖤)       black heart
-1F5A5..1F5FA  ; Extended_Pictographic#  7.0 [86] (🖥️..🗺️)    desktop computer..world map
-1F5FB..1F5FF  ; Extended_Pictographic#  6.0  [5] (🗻..🗿)    mount fuji..moai
-1F600         ; Extended_Pictographic#  6.1  [1] (😀)       grinning face
-1F601..1F610  ; Extended_Pictographic#  6.0 [16] (😁..😐)    beaming face with smiling eyes..neutral face
-1F611         ; Extended_Pictographic#  6.1  [1] (😑)       expressionless face
-1F612..1F614  ; Extended_Pictographic#  6.0  [3] (😒..😔)    unamused face..pensive face
-1F615         ; Extended_Pictographic#  6.1  [1] (😕)       confused face
-1F616         ; Extended_Pictographic#  6.0  [1] (😖)       confounded face
-1F617         ; Extended_Pictographic#  6.1  [1] (😗)       kissing face
-1F618         ; Extended_Pictographic#  6.0  [1] (😘)       face blowing a kiss
-1F619         ; Extended_Pictographic#  6.1  [1] (😙)       kissing face with smiling eyes
-1F61A         ; Extended_Pictographic#  6.0  [1] (😚)       kissing face with closed eyes
-1F61B         ; Extended_Pictographic#  6.1  [1] (😛)       face with tongue
-1F61C..1F61E  ; Extended_Pictographic#  6.0  [3] (😜..😞)    winking face with tongue..disappointed face
-1F61F         ; Extended_Pictographic#  6.1  [1] (😟)       worried face
-1F620..1F625  ; Extended_Pictographic#  6.0  [6] (😠..😥)    angry face..sad but relieved face
-1F626..1F627  ; Extended_Pictographic#  6.1  [2] (😦..😧)    frowning face with open mouth..anguished face
-1F628..1F62B  ; Extended_Pictographic#  6.0  [4] (😨..😫)    fearful face..tired face
-1F62C         ; Extended_Pictographic#  6.1  [1] (😬)       grimacing face
-1F62D         ; Extended_Pictographic#  6.0  [1] (😭)       loudly crying face
-1F62E..1F62F  ; Extended_Pictographic#  6.1  [2] (😮..😯)    face with open mouth..hushed face
-1F630..1F633  ; Extended_Pictographic#  6.0  [4] (😰..😳)    anxious face with sweat..flushed face
-1F634         ; Extended_Pictographic#  6.1  [1] (😴)       sleeping face
-1F635..1F640  ; Extended_Pictographic#  6.0 [12] (😵..🙀)    dizzy face..weary cat
-1F641..1F642  ; Extended_Pictographic#  7.0  [2] (🙁..🙂)    slightly frowning face..slightly smiling face
-1F643..1F644  ; Extended_Pictographic#  8.0  [2] (🙃..🙄)    upside-down face..face with rolling eyes
-1F645..1F64F  ; Extended_Pictographic#  6.0 [11] (🙅..🙏)    person gesturing NO..folded hands
-1F680..1F6C5  ; Extended_Pictographic#  6.0 [70] (🚀..🛅)    rocket..left luggage
-1F6C6..1F6CF  ; Extended_Pictographic#  7.0 [10] (🛆..🛏️)    TRIANGLE WITH ROUNDED CORNERS..bed
-1F6D0         ; Extended_Pictographic#  8.0  [1] (🛐)       place of worship
-1F6D1..1F6D2  ; Extended_Pictographic#  9.0  [2] (🛑..🛒)    stop sign..shopping cart
-1F6D3..1F6D4  ; Extended_Pictographic# 10.0  [2] (🛓..🛔)    STUPA..PAGODA
-1F6D5         ; Extended_Pictographic# 12.0  [1] (🛕)       hindu temple
-1F6D6..1F6DF  ; Extended_Pictographic#   NA [10] (🛖..🛟)    <reserved-1F6D6>..<reserved-1F6DF>
-1F6E0..1F6EC  ; Extended_Pictographic#  7.0 [13] (🛠️..🛬)    hammer and wrench..airplane arrival
-1F6ED..1F6EF  ; Extended_Pictographic#   NA  [3] (🛭..🛯)    <reserved-1F6ED>..<reserved-1F6EF>
-1F6F0..1F6F3  ; Extended_Pictographic#  7.0  [4] (🛰️..🛳️)    satellite..passenger ship
-1F6F4..1F6F6  ; Extended_Pictographic#  9.0  [3] (🛴..🛶)    kick scooter..canoe
-1F6F7..1F6F8  ; Extended_Pictographic# 10.0  [2] (🛷..🛸)    sled..flying saucer
-1F6F9         ; Extended_Pictographic# 11.0  [1] (🛹)       skateboard
-1F6FA         ; Extended_Pictographic# 12.0  [1] (🛺)       auto rickshaw
-1F6FB..1F6FF  ; Extended_Pictographic#   NA  [5] (🛻..🛿)    <reserved-1F6FB>..<reserved-1F6FF>
-1F774..1F77F  ; Extended_Pictographic#   NA [12] (🝴..🝿)    <reserved-1F774>..<reserved-1F77F>
-1F7D5..1F7D8  ; Extended_Pictographic# 11.0  [4] (🟕..🟘)    CIRCLED TRIANGLE..NEGATIVE CIRCLED SQUARE
-1F7D9..1F7DF  ; Extended_Pictographic#   NA  [7] (🟙..🟟)    <reserved-1F7D9>..<reserved-1F7DF>
-1F7E0..1F7EB  ; Extended_Pictographic# 12.0 [12] (🟠..🟫)    orange circle..brown square
-1F7EC..1F7FF  ; Extended_Pictographic#   NA [20] (🟬..🟿)    <reserved-1F7EC>..<reserved-1F7FF>
-1F80C..1F80F  ; Extended_Pictographic#   NA  [4] (🠌..🠏)    <reserved-1F80C>..<reserved-1F80F>
-1F848..1F84F  ; Extended_Pictographic#   NA  [8] (🡈..🡏)    <reserved-1F848>..<reserved-1F84F>
-1F85A..1F85F  ; Extended_Pictographic#   NA  [6] (🡚..🡟)    <reserved-1F85A>..<reserved-1F85F>
-1F888..1F88F  ; Extended_Pictographic#   NA  [8] (🢈..🢏)    <reserved-1F888>..<reserved-1F88F>
-1F8AE..1F8FF  ; Extended_Pictographic#   NA [82] (🢮..🣿)    <reserved-1F8AE>..<reserved-1F8FF>
-1F90C         ; Extended_Pictographic#   NA  [1] (🤌)       <reserved-1F90C>
-1F90D..1F90F  ; Extended_Pictographic# 12.0  [3] (🤍..🤏)    white heart..pinching hand
-1F910..1F918  ; Extended_Pictographic#  8.0  [9] (🤐..🤘)    zipper-mouth face..sign of the horns
-1F919..1F91E  ; Extended_Pictographic#  9.0  [6] (🤙..🤞)    call me hand..crossed fingers
-1F91F         ; Extended_Pictographic# 10.0  [1] (🤟)       love-you gesture
-1F920..1F927  ; Extended_Pictographic#  9.0  [8] (🤠..🤧)    cowboy hat face..sneezing face
-1F928..1F92F  ; Extended_Pictographic# 10.0  [8] (🤨..🤯)    face with raised eyebrow..exploding head
-1F930         ; Extended_Pictographic#  9.0  [1] (🤰)       pregnant woman
-1F931..1F932  ; Extended_Pictographic# 10.0  [2] (🤱..🤲)    breast-feeding..palms up together
-1F933..1F93A  ; Extended_Pictographic#  9.0  [8] (🤳..🤺)    selfie..person fencing
-1F93C..1F93E  ; Extended_Pictographic#  9.0  [3] (🤼..🤾)    people wrestling..person playing handball
-1F93F         ; Extended_Pictographic# 12.0  [1] (🤿)       diving mask
-1F940..1F945  ; Extended_Pictographic#  9.0  [6] (🥀..🥅)    wilted flower..goal net
-1F947..1F94B  ; Extended_Pictographic#  9.0  [5] (🥇..🥋)    1st place medal..martial arts uniform
-1F94C         ; Extended_Pictographic# 10.0  [1] (🥌)       curling stone
-1F94D..1F94F  ; Extended_Pictographic# 11.0  [3] (🥍..🥏)    lacrosse..flying disc
-1F950..1F95E  ; Extended_Pictographic#  9.0 [15] (🥐..🥞)    croissant..pancakes
-1F95F..1F96B  ; Extended_Pictographic# 10.0 [13] (🥟..🥫)    dumpling..canned food
-1F96C..1F970  ; Extended_Pictographic# 11.0  [5] (🥬..🥰)    leafy green..smiling face with hearts
-1F971         ; Extended_Pictographic# 12.0  [1] (🥱)       yawning face
-1F972         ; Extended_Pictographic#   NA  [1] (🥲)       <reserved-1F972>
-1F973..1F976  ; Extended_Pictographic# 11.0  [4] (🥳..🥶)    partying face..cold face
-1F977..1F979  ; Extended_Pictographic#   NA  [3] (🥷..🥹)    <reserved-1F977>..<reserved-1F979>
-1F97A         ; Extended_Pictographic# 11.0  [1] (🥺)       pleading face
-1F97B         ; Extended_Pictographic# 12.0  [1] (🥻)       sari
-1F97C..1F97F  ; Extended_Pictographic# 11.0  [4] (🥼..🥿)    lab coat..flat shoe
-1F980..1F984  ; Extended_Pictographic#  8.0  [5] (🦀..🦄)    crab..unicorn
-1F985..1F991  ; Extended_Pictographic#  9.0 [13] (🦅..🦑)    eagle..squid
-1F992..1F997  ; Extended_Pictographic# 10.0  [6] (🦒..🦗)    giraffe..cricket
-1F998..1F9A2  ; Extended_Pictographic# 11.0 [11] (🦘..🦢)    kangaroo..swan
-1F9A3..1F9A4  ; Extended_Pictographic#   NA  [2] (🦣..🦤)    <reserved-1F9A3>..<reserved-1F9A4>
-1F9A5..1F9AA  ; Extended_Pictographic# 12.0  [6] (🦥..🦪)    sloth..oyster
-1F9AB..1F9AD  ; Extended_Pictographic#   NA  [3] (🦫..🦭)    <reserved-1F9AB>..<reserved-1F9AD>
-1F9AE..1F9AF  ; Extended_Pictographic# 12.0  [2] (🦮..🦯)    guide dog..probing cane
-1F9B0..1F9B9  ; Extended_Pictographic# 11.0 [10] (🦰..🦹)    red hair..supervillain
-1F9BA..1F9BF  ; Extended_Pictographic# 12.0  [6] (🦺..🦿)    safety vest..mechanical leg
-1F9C0         ; Extended_Pictographic#  8.0  [1] (🧀)       cheese wedge
-1F9C1..1F9C2  ; Extended_Pictographic# 11.0  [2] (🧁..🧂)    cupcake..salt
-1F9C3..1F9CA  ; Extended_Pictographic# 12.0  [8] (🧃..🧊)    beverage box..ice cube
-1F9CB..1F9CC  ; Extended_Pictographic#   NA  [2] (🧋..🧌)    <reserved-1F9CB>..<reserved-1F9CC>
-1F9CD..1F9CF  ; Extended_Pictographic# 12.0  [3] (🧍..🧏)    person standing..deaf person
-1F9D0..1F9E6  ; Extended_Pictographic# 10.0 [23] (🧐..🧦)    face with monocle..socks
-1F9E7..1F9FF  ; Extended_Pictographic# 11.0 [25] (🧧..🧿)    red envelope..nazar amulet
-1FA00..1FA53  ; Extended_Pictographic# 12.0 [84] (🨀..🩓)    NEUTRAL CHESS KING..BLACK CHESS KNIGHT-BISHOP
-1FA54..1FA5F  ; Extended_Pictographic#   NA [12] (🩔..🩟)    <reserved-1FA54>..<reserved-1FA5F>
-1FA60..1FA6D  ; Extended_Pictographic# 11.0 [14] (🩠..🩭)    XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER
-1FA6E..1FA6F  ; Extended_Pictographic#   NA  [2] (🩮..🩯)    <reserved-1FA6E>..<reserved-1FA6F>
-1FA70..1FA73  ; Extended_Pictographic# 12.0  [4] (🩰..🩳)    ballet shoes..shorts
-1FA74..1FA77  ; Extended_Pictographic#   NA  [4] (🩴..🩷)    <reserved-1FA74>..<reserved-1FA77>
-1FA78..1FA7A  ; Extended_Pictographic# 12.0  [3] (🩸..🩺)    drop of blood..stethoscope
-1FA7B..1FA7F  ; Extended_Pictographic#   NA  [5] (🩻..🩿)    <reserved-1FA7B>..<reserved-1FA7F>
-1FA80..1FA82  ; Extended_Pictographic# 12.0  [3] (🪀..🪂)    yo-yo..parachute
-1FA83..1FA8F  ; Extended_Pictographic#   NA [13] (🪃..🪏)    <reserved-1FA83>..<reserved-1FA8F>
-1FA90..1FA95  ; Extended_Pictographic# 12.0  [6] (🪐..🪕)    ringed planet..banjo
-1FA96..1FFFD  ; Extended_Pictographic#   NA[1384] (🪖..🿽)   <reserved-1FA96>..<reserved-1FFFD>
-
-# Total elements: 3793
-
-#EOF

lib/pleroma/emoji.ex

@@ -102,31 +102,36 @@ defmodule Pleroma.Emoji do
     :ets.insert(@ets, emojis)
   end
 
-  @external_resource "lib/pleroma/emoji-data.txt"
+  @external_resource "lib/pleroma/emoji-test.txt"
+
+  regional_indicators =
+    Enum.map(127_462..127_487, fn codepoint ->
+      <<codepoint::utf8>>
+    end)
 
   emojis =
     @external_resource
     |> File.read!()
     |> String.split("\n")
-    |> Enum.filter(fn line -> line != "" and not String.starts_with?(line, "#") end)
+    |> Enum.filter(fn line ->
+      line != "" and not String.starts_with?(line, "#") and
+        String.contains?(line, "fully-qualified")
+    end)
     |> Enum.map(fn line ->
       line
       |> String.split(";", parts: 2)
       |> hd()
       |> String.trim()
-      |> String.split("..")
+      |> String.split()
-      |> case do
+      |> Enum.map(fn codepoint ->
-        [number] ->
+        <<String.to_integer(codepoint, 16)::utf8>>
-          <<String.to_integer(number, 16)::utf8>>
+      end)
-
+      |> Enum.join()
-        [first, last] ->
-          String.to_integer(first, 16)..String.to_integer(last, 16)
-          |> Enum.map(&<<&1::utf8>>)
-      end
     end)
-    |> List.flatten()
     |> Enum.uniq()
 
+  emojis = emojis ++ regional_indicators
+
   for emoji <- emojis do
     def is_unicode_emoji?(unquote(emoji)), do: true
   end

lib/pleroma/emoji/pack.ex

@@ -20,16 +20,18 @@ defmodule Pleroma.Emoji.Pack do
           name: String.t()
         }
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   alias Pleroma.Emoji
   alias Pleroma.Emoji.Pack
+  alias Pleroma.Utils
 
   @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
   def create(name) do
     with :ok <- validate_not_empty([name]),
          dir <- Path.join(emoji_path(), name),
          :ok <- File.mkdir(dir) do
-      %__MODULE__{pack_file: Path.join(dir, "pack.json")}
+      save_pack(%__MODULE__{pack_file: Path.join(dir, "pack.json")})
-      |> save_pack()
     end
   end
 
@@ -62,10 +64,9 @@ defmodule Pleroma.Emoji.Pack do
   @spec delete(String.t()) ::
           {:ok, [binary()]} | {:error, File.posix(), binary()} | {:error, :empty_values}
   def delete(name) do
-    with :ok <- validate_not_empty([name]) do
+    with :ok <- validate_not_empty([name]),
-      emoji_path()
+         pack_path <- Path.join(emoji_path(), name) do
-      |> Path.join(name)
+      File.rm_rf(pack_path)
-      |> File.rm_rf()
     end
   end
 
@@ -94,7 +95,7 @@ defmodule Pleroma.Emoji.Pack do
   def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"} = file) do
     with {:ok, zip_files} <- :zip.table(to_charlist(file.path)),
          [_ | _] = emojies <- unpack_zip_emojies(zip_files),
-         {:ok, tmp_dir} <- Pleroma.Utils.tmp_dir("emoji") do
+         {:ok, tmp_dir} <- Utils.tmp_dir("emoji") do
       try do
         {:ok, _emoji_files} =
           :zip.unzip(
@@ -282,18 +283,21 @@ defmodule Pleroma.Emoji.Pack do
     end
   end
 
-  @spec load_pack(String.t()) :: {:ok, t()} | {:error, :not_found}
+  @spec load_pack(String.t()) :: {:ok, t()} | {:error, :file.posix()}
   def load_pack(name) do
     pack_file = Path.join([emoji_path(), name, "pack.json"])
 
-    if File.exists?(pack_file) do
+    with {:ok, _} <- File.stat(pack_file),
+         {:ok, pack_data} <- File.read(pack_file) do
       pack =
-        pack_file
+        from_json(
-        |> File.read!()
+          pack_data,
-        |> from_json()
+          %{
-        |> Map.put(:pack_file, pack_file)
+            pack_file: pack_file,
-        |> Map.put(:path, Path.dirname(pack_file))
+            path: Path.dirname(pack_file),
-        |> Map.put(:name, name)
+            name: name
+          }
+        )
 
       files_count =
         pack.files
@@ -301,8 +305,6 @@ defmodule Pleroma.Emoji.Pack do
         |> length()
 
       {:ok, Map.put(pack, :files_count, files_count)}
-    else
-      {:error, :not_found}
     end
   end
 
@@ -415,7 +417,7 @@ defmodule Pleroma.Emoji.Pack do
     ttl_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file])
     overall_ttl = :timer.seconds(ttl_per_file * Enum.count(files))
 
-    Cachex.put!(
+    @cachex.put(
       :emoji_packs_cache,
       pack.name,
       # if pack.json MD5 changes, the cache is not valid anymore
@@ -434,10 +436,17 @@ defmodule Pleroma.Emoji.Pack do
     end
   end
 
-  defp from_json(json) do
+  defp from_json(json, attrs) do
     map = Jason.decode!(json)
 
-    struct(__MODULE__, %{files: map["files"], pack: map["pack"]})
+    pack_attrs =
+      attrs
+      |> Map.merge(%{
+        files: map["files"],
+        pack: map["pack"]
+      })
+
+    struct(__MODULE__, pack_attrs)
   end
 
   defp validate_shareable_packs_available(uri) do
@@ -491,10 +500,10 @@ defmodule Pleroma.Emoji.Pack do
   end
 
   defp create_subdirs(file_path) do
-    if String.contains?(file_path, "/") do
+    with true <- String.contains?(file_path, "/"),
-      file_path
+         path <- Path.dirname(file_path),
-      |> Path.dirname()
+         false <- File.exists?(path) do
-      |> File.mkdir_p!()
+      File.mkdir_p!(path)
     end
   end
 
@@ -518,10 +527,15 @@ defmodule Pleroma.Emoji.Pack do
 
   defp get_filename(pack, shortcode) do
     with %{^shortcode => filename} when is_binary(filename) <- pack.files,
-         true <- pack.path |> Path.join(filename) |> File.exists?() do
+         file_path <- Path.join(pack.path, filename),
+         {:ok, _} <- File.stat(file_path) do
       {:ok, filename}
     else
-      _ -> {:error, :doesnt_exist}
+      {:error, _} = error ->
+        error
+
+      _ ->
+        {:error, :doesnt_exist}
     end
   end
 
@@ -594,7 +608,7 @@ defmodule Pleroma.Emoji.Pack do
   end
 
   defp download_archive(url, sha) do
-    with {:ok, %{body: archive}} <- Tesla.get(url) do
+    with {:ok, %{body: archive}} <- Pleroma.HTTP.get(url) do
       if Base.decode16!(sha) == :crypto.hash(:sha256, archive) do
         {:ok, archive}
       else
@@ -606,7 +620,7 @@ defmodule Pleroma.Emoji.Pack do
   defp fetch_archive(pack) do
     hash = :crypto.hash(:md5, File.read!(pack.pack_file))
 
-    case Cachex.get!(:emoji_packs_cache, pack.name) do
+    case @cachex.get!(:emoji_packs_cache, pack.name) do
       %{hash: ^hash, pack_data: archive} -> archive
       _ -> create_archive_and_cache(pack, hash)
     end
@@ -617,7 +631,7 @@ defmodule Pleroma.Emoji.Pack do
   end
 
   defp update_sha_and_save_metadata(pack, data) do
-    with {:ok, %{body: zip}} <- Tesla.get(data[:"fallback-src"]),
+    with {:ok, %{body: zip}} <- Pleroma.HTTP.get(data[:"fallback-src"]),
          :ok <- validate_has_all_files(pack, zip) do
       fallback_sha = :sha256 |> :crypto.hash(zip) |> Base.encode16()
 

lib/pleroma/following_relationship.ex

@@ -62,23 +62,47 @@ defmodule Pleroma.FollowingRelationship do
         follow(follower, following, state)
 
       following_relationship ->
+        with {:ok, _following_relationship} <-
                following_relationship
                |> cast(%{state: state}, [:state])
                |> validate_required([:state])
-        |> Repo.update()
+               |> Repo.update() do
+          after_update(state, follower, following)
+        end
     end
   end
 
   def follow(%User{} = follower, %User{} = following, state \\ :follow_accept) do
+    with {:ok, _following_relationship} <-
            %__MODULE__{}
            |> changeset(%{follower: follower, following: following, state: state})
-    |> Repo.insert(on_conflict: :nothing)
+           |> Repo.insert(on_conflict: :nothing) do
+      after_update(state, follower, following)
+    end
   end
 
   def unfollow(%User{} = follower, %User{} = following) do
     case get(follower, following) do
-      %__MODULE__{} = following_relationship -> Repo.delete(following_relationship)
+      %__MODULE__{} = following_relationship ->
-      _ -> {:ok, nil}
+        with {:ok, _following_relationship} <- Repo.delete(following_relationship) do
+          after_update(:unfollow, follower, following)
+        end
+
+      _ ->
+        {:ok, nil}
+    end
+  end
+
+  defp after_update(state, %User{} = follower, %User{} = following) do
+    with {:ok, following} <- User.update_follower_count(following),
+         {:ok, follower} <- User.update_following_count(follower) do
+      Pleroma.Web.Streamer.stream("follow_relationship", %{
+        state: state,
+        following: following,
+        follower: follower
+      })
+
+      {:ok, follower, following}
     end
   end
 

lib/pleroma/frontend.ex

@@ -0,0 +1,110 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Frontend do
+  alias Pleroma.Config
+
+  require Logger
+
+  def install(name, opts \\ []) do
+    frontend_info = %{
+      "ref" => opts[:ref],
+      "build_url" => opts[:build_url],
+      "build_dir" => opts[:build_dir]
+    }
+
+    frontend_info =
+      [:frontends, :available, name]
+      |> Config.get(%{})
+      |> Map.merge(frontend_info, fn _key, config, cmd ->
+        # This only overrides things that are actually set
+        cmd || config
+      end)
+
+    ref = frontend_info["ref"]
+
+    unless ref do
+      raise "No ref given or configured"
+    end
+
+    dest = Path.join([dir(), name, ref])
+
+    label = "#{name} (#{ref})"
+    tmp_dir = Path.join(dir(), "tmp")
+
+    with {_, :ok} <-
+           {:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, opts[:file])},
+         Logger.info("Installing #{label} to #{dest}"),
+         :ok <- install_frontend(frontend_info, tmp_dir, dest) do
+      File.rm_rf!(tmp_dir)
+      Logger.info("Frontend #{label} installed to #{dest}")
+    else
+      {:download_or_unzip, _} ->
+        Logger.info("Could not download or unzip the frontend")
+        {:error, "Could not download or unzip the frontend"}
+
+      _e ->
+        Logger.info("Could not install the frontend")
+        {:error, "Could not install the frontend"}
+    end
+  end
+
+  def dir(opts \\ []) do
+    if is_nil(opts[:static_dir]) do
+      Pleroma.Config.get!([:instance, :static_dir])
+    else
+      opts[:static_dir]
+    end
+    |> Path.join("frontends")
+  end
+
+  defp download_or_unzip(frontend_info, temp_dir, nil),
+    do: download_build(frontend_info, temp_dir)
+
+  defp download_or_unzip(_frontend_info, temp_dir, file) do
+    with {:ok, zip} <- File.read(Path.expand(file)) do
+      unzip(zip, temp_dir)
+    end
+  end
+
+  def unzip(zip, dest) do
+    with {:ok, unzipped} <- :zip.unzip(zip, [:memory]) do
+      File.rm_rf!(dest)
+      File.mkdir_p!(dest)
+
+      Enum.each(unzipped, fn {filename, data} ->
+        path = filename
+
+        new_file_path = Path.join(dest, path)
+
+        new_file_path
+        |> Path.dirname()
+        |> File.mkdir_p!()
+
+        File.write!(new_file_path, data)
+      end)
+    end
+  end
+
+  defp download_build(frontend_info, dest) do
+    Logger.info("Downloading pre-built bundle for #{frontend_info["name"]}")
+    url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
+
+    with {:ok, %{status: 200, body: zip_body}} <-
+           Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do
+      unzip(zip_body, dest)
+    else
+      {:error, e} -> {:error, e}
+      e -> {:error, e}
+    end
+  end
+
+  defp install_frontend(frontend_info, source, dest) do
+    from = frontend_info["build_dir"] || "dist"
+    File.rm_rf!(dest)
+    File.mkdir_p!(dest)
+    File.cp_r!(Path.join([source, from]), dest)
+    :ok
+  end
+end

lib/pleroma/helpers/auth_helper.ex

@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.AuthHelper do
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
+  alias Plug.Conn
+
+  import Plug.Conn
+
+  @oauth_token_session_key :oauth_token
+
+  @doc """
+  Skips OAuth permissions (scopes) checks, assigns nil `:token`.
+  Intended to be used with explicit authentication and only when OAuth token cannot be determined.
+  """
+  def skip_oauth(conn) do
+    conn
+    |> assign(:token, nil)
+    |> OAuthScopesPlug.skip_plug()
+  end
+
+  @doc "Drops authentication info from connection"
+  def drop_auth_info(conn) do
+    # To simplify debugging, setting a private variable on `conn` if auth info is dropped
+    conn
+    |> assign(:user, nil)
+    |> assign(:token, nil)
+    |> put_private(:authentication_ignored, true)
+  end
+
+  @doc "Gets OAuth token string from session"
+  def get_session_token(%Conn{} = conn) do
+    get_session(conn, @oauth_token_session_key)
+  end
+
+  @doc "Updates OAuth token string in session"
+  def put_session_token(%Conn{} = conn, token) when is_binary(token) do
+    put_session(conn, @oauth_token_session_key, token)
+  end
+
+  @doc "Deletes OAuth token string from session"
+  def delete_session_token(%Conn{} = conn) do
+    delete_session(conn, @oauth_token_session_key)
+  end
+end

lib/pleroma/helpers/inet_helper.ex

@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.InetHelper do
+  def parse_address(ip) when is_tuple(ip) do
+    {:ok, ip}
+  end
+
+  def parse_address(ip) when is_binary(ip) do
+    ip
+    |> String.to_charlist()
+    |> parse_address()
+  end
+
+  def parse_address(ip) do
+    :inet.parse_address(ip)
+  end
+end

lib/pleroma/html.ex

@@ -6,6 +6,8 @@ defmodule Pleroma.HTML do
   # Scrubbers are compiled on boot so they can be configured in OTP releases
   #  @on_load :compile_scrubbers
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   def compile_scrubbers do
     dir = Path.join(:code.priv_dir(:pleroma), "scrubbers")
 
@@ -56,7 +58,7 @@ defmodule Pleroma.HTML do
       ) do
     key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
 
-    Cachex.fetch!(:scrubber_cache, key, fn _key ->
+    @cachex.fetch!(:scrubber_cache, key, fn _key ->
       object = Pleroma.Object.normalize(activity)
       ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
     end)
@@ -105,7 +107,7 @@ defmodule Pleroma.HTML do
     unless object.data["fake"] do
       key = "URL|#{object.id}"
 
-      Cachex.fetch!(:scrubber_cache, key, fn _key ->
+      @cachex.fetch!(:scrubber_cache, key, fn _key ->
         {:commit, {:ok, extract_first_external_url(content)}}
       end)
     else

lib/pleroma/instances.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Instances do
   defdelegate reachable?(url_or_host), to: @adapter
   defdelegate set_reachable(url_or_host), to: @adapter
   defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter
+  defdelegate get_consistently_unreachable(), to: @adapter
 
   def set_consistently_unreachable(url_or_host),
     do: set_unreachable(url_or_host, reachability_datetime_threshold())

lib/pleroma/instances/instance.ex

@@ -77,7 +77,7 @@ defmodule Pleroma.Instances.Instance do
     )
   end
 
-  def reachable?(_), do: true
+  def reachable?(url_or_host) when is_binary(url_or_host), do: true
 
   def set_reachable(url_or_host) when is_binary(url_or_host) do
     with host <- host(url_or_host),
@@ -119,6 +119,17 @@ defmodule Pleroma.Instances.Instance do
 
   def set_unreachable(_, _), do: {:error, nil}
 
+  def get_consistently_unreachable do
+    reachability_datetime_threshold = Instances.reachability_datetime_threshold()
+
+    from(i in Instance,
+      where: ^reachability_datetime_threshold > i.unreachable_since,
+      order_by: i.unreachable_since,
+      select: {i.host, i.unreachable_since}
+    )
+    |> Repo.all()
+  end
+
   defp parse_datetime(datetime) when is_binary(datetime) do
     NaiveDateTime.from_iso8601(datetime)
   end
@@ -155,7 +166,8 @@ defmodule Pleroma.Instances.Instance do
 
   defp scrape_favicon(%URI{} = instance_uri) do
     try do
-      with {:ok, %Tesla.Env{body: html}} <-
+      with {_, true} <- {:reachable, reachable?(instance_uri.host)},
+           {:ok, %Tesla.Env{body: html}} <-
              Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], pool: :media),
            {_, [favicon_rel | _]} when is_binary(favicon_rel) <-
              {:parse,
@@ -164,7 +176,15 @@ defmodule Pleroma.Instances.Instance do
              {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do
         favicon
       else
-        _ -> nil
+        {:reachable, false} ->
+          Logger.debug(
+            "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host"
+          )
+
+          nil
+
+        _ ->
+          nil
       end
     rescue
       e ->

lib/pleroma/moderation_log.ex

@@ -12,6 +12,26 @@ defmodule Pleroma.ModerationLog do
 
   import Ecto.Query
 
+  @type t :: %__MODULE__{}
+  @type log_subject :: Activity.t() | User.t() | list(User.t())
+  @type log_params :: %{
+          required(:actor) => User.t(),
+          required(:action) => String.t(),
+          optional(:subject) => log_subject(),
+          optional(:subject_actor) => User.t(),
+          optional(:subject_id) => String.t(),
+          optional(:subjects) => list(User.t()),
+          optional(:permission) => String.t(),
+          optional(:text) => String.t(),
+          optional(:sensitive) => String.t(),
+          optional(:visibility) => String.t(),
+          optional(:followed) => User.t(),
+          optional(:follower) => User.t(),
+          optional(:nicknames) => list(String.t()),
+          optional(:tags) => list(String.t()),
+          optional(:target) => String.t()
+        }
+
   schema "moderation_log" do
     field(:data, :map)
 
@@ -90,203 +110,105 @@ defmodule Pleroma.ModerationLog do
     parsed_datetime
   end
 
-  @spec insert_log(%{actor: User, subject: [User], action: String.t(), permission: String.t()}) ::
+  defp prepare_log_data(%{actor: actor, action: action} = attrs) do
-          {:ok, ModerationLog} | {:error, any}
+    %{
-  def insert_log(%{
-        actor: %User{} = actor,
-        subject: subjects,
-        action: action,
-        permission: permission
-      }) do
-    %ModerationLog{
-      data: %{
       "actor" => user_to_map(actor),
-        "subject" => user_to_map(subjects),
       "action" => action,
-        "permission" => permission,
       "message" => ""
     }
-    }
+    |> Pleroma.Maps.put_if_present("subject_actor", user_to_map(attrs[:subject_actor]))
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, subject: User, action: String.t()}) ::
+  defp prepare_log_data(attrs), do: attrs
-          {:ok, ModerationLog} | {:error, any}
+
-  def insert_log(%{
+  @spec insert_log(log_params()) :: {:ok, ModerationLog} | {:error, any}
-        actor: %User{} = actor,
+  def insert_log(%{actor: %User{}, subject: subjects, permission: permission} = attrs) do
-        action: "report_update",
+    data =
-        subject: %Activity{data: %{"type" => "Flag"}} = subject
+      attrs
-      }) do
+      |> prepare_log_data
-    %ModerationLog{
+      |> Map.merge(%{"subject" => user_to_map(subjects), "permission" => permission})
-      data: %{
+
-        "actor" => user_to_map(actor),
+    insert_log_entry_with_message(%ModerationLog{data: data})
-        "action" => "report_update",
-        "subject" => report_to_map(subject),
-        "message" => ""
-      }
-    }
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, subject: Activity, action: String.t(), text: String.t()}) ::
+  def insert_log(%{actor: %User{}, action: action, subject: %Activity{} = subject} = attrs)
-          {:ok, ModerationLog} | {:error, any}
+      when action in ["report_note_delete", "report_update", "report_note"] do
-  def insert_log(%{
+    data =
-        actor: %User{} = actor,
+      attrs
-        action: "report_note",
+      |> prepare_log_data
-        subject: %Activity{} = subject,
+      |> Pleroma.Maps.put_if_present("text", attrs[:text])
-        text: text
+      |> Map.merge(%{"subject" => report_to_map(subject)})
-      }) do
+
-    %ModerationLog{
+    insert_log_entry_with_message(%ModerationLog{data: data})
-      data: %{
-        "actor" => user_to_map(actor),
-        "action" => "report_note",
-        "subject" => report_to_map(subject),
-        "text" => text
-      }
-    }
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, subject: Activity, action: String.t(), text: String.t()}) ::
+  def insert_log(
-          {:ok, ModerationLog} | {:error, any}
+        %{
-  def insert_log(%{
+          actor: %User{},
-        actor: %User{} = actor,
+          action: action,
-        action: "report_note_delete",
-        subject: %Activity{} = subject,
-        text: text
-      }) do
-    %ModerationLog{
-      data: %{
-        "actor" => user_to_map(actor),
-        "action" => "report_note_delete",
-        "subject" => report_to_map(subject),
-        "text" => text
-      }
-    }
-    |> insert_log_entry_with_message()
-  end
-
-  @spec insert_log(%{
-          actor: User,
-          subject: Activity,
-          action: String.t(),
-          sensitive: String.t(),
-          visibility: String.t()
-        }) :: {:ok, ModerationLog} | {:error, any}
-  def insert_log(%{
-        actor: %User{} = actor,
-        action: "status_update",
           subject: %Activity{} = subject,
           sensitive: sensitive,
           visibility: visibility
-      }) do
+        } = attrs
-    %ModerationLog{
+      )
-      data: %{
+      when action == "status_update" do
-        "actor" => user_to_map(actor),
+    data =
-        "action" => "status_update",
+      attrs
+      |> prepare_log_data
+      |> Map.merge(%{
         "subject" => status_to_map(subject),
         "sensitive" => sensitive,
-        "visibility" => visibility,
+        "visibility" => visibility
-        "message" => ""
+      })
-      }
+
-    }
+    insert_log_entry_with_message(%ModerationLog{data: data})
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, action: String.t(), subject_id: String.t()}) ::
+  def insert_log(%{actor: %User{}, action: action, subject_id: subject_id} = attrs)
-          {:ok, ModerationLog} | {:error, any}
+      when action == "status_delete" do
-  def insert_log(%{
+    data =
-        actor: %User{} = actor,
+      attrs
-        action: "status_delete",
+      |> prepare_log_data
-        subject_id: subject_id
+      |> Map.merge(%{"subject_id" => subject_id})
-      }) do
+
-    %ModerationLog{
+    insert_log_entry_with_message(%ModerationLog{data: data})
-      data: %{
-        "actor" => user_to_map(actor),
-        "action" => "status_delete",
-        "subject_id" => subject_id,
-        "message" => ""
-      }
-    }
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, subject: User, action: String.t()}) ::
+  def insert_log(%{actor: %User{}, subject: subject, action: _action} = attrs) do
-          {:ok, ModerationLog} | {:error, any}
+    data =
-  def insert_log(%{actor: %User{} = actor, subject: subject, action: action}) do
+      attrs
-    %ModerationLog{
+      |> prepare_log_data
-      data: %{
+      |> Map.merge(%{"subject" => user_to_map(subject)})
-        "actor" => user_to_map(actor),
+
-        "action" => action,
+    insert_log_entry_with_message(%ModerationLog{data: data})
-        "subject" => user_to_map(subject),
-        "message" => ""
-      }
-    }
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, subjects: [User], action: String.t()}) ::
+  def insert_log(%{actor: %User{}, subjects: subjects, action: _action} = attrs) do
-          {:ok, ModerationLog} | {:error, any}
+    data =
-  def insert_log(%{actor: %User{} = actor, subjects: subjects, action: action}) do
+      attrs
-    subjects = Enum.map(subjects, &user_to_map/1)
+      |> prepare_log_data
+      |> Map.merge(%{"subjects" => user_to_map(subjects)})
 
-    %ModerationLog{
+    insert_log_entry_with_message(%ModerationLog{data: data})
-      data: %{
-        "actor" => user_to_map(actor),
-        "action" => action,
-        "subjects" => subjects,
-        "message" => ""
-      }
-    }
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, action: String.t(), followed: User, follower: User}) ::
+  def insert_log(
-          {:ok, ModerationLog} | {:error, any}
+        %{
-  def insert_log(%{
+          actor: %User{},
-        actor: %User{} = actor,
           followed: %User{} = followed,
           follower: %User{} = follower,
-        action: "follow"
+          action: action
-      }) do
+        } = attrs
-    %ModerationLog{
+      )
-      data: %{
+      when action in ["unfollow", "follow"] do
-        "actor" => user_to_map(actor),
+    data =
-        "action" => "follow",
+      attrs
-        "followed" => user_to_map(followed),
+      |> prepare_log_data
-        "follower" => user_to_map(follower),
+      |> Map.merge(%{"followed" => user_to_map(followed), "follower" => user_to_map(follower)})
-        "message" => ""
+
-      }
+    insert_log_entry_with_message(%ModerationLog{data: data})
-    }
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, action: String.t(), followed: User, follower: User}) ::
-          {:ok, ModerationLog} | {:error, any}
-  def insert_log(%{
-        actor: %User{} = actor,
-        followed: %User{} = followed,
-        follower: %User{} = follower,
-        action: "unfollow"
-      }) do
-    %ModerationLog{
-      data: %{
-        "actor" => user_to_map(actor),
-        "action" => "unfollow",
-        "followed" => user_to_map(followed),
-        "follower" => user_to_map(follower),
-        "message" => ""
-      }
-    }
-    |> insert_log_entry_with_message()
-  end
-
-  @spec insert_log(%{
-          actor: User,
-          action: String.t(),
-          nicknames: [String.t()],
-          tags: [String.t()]
-        }) :: {:ok, ModerationLog} | {:error, any}
   def insert_log(%{
         actor: %User{} = actor,
         nicknames: nicknames,
@@ -305,27 +227,16 @@ defmodule Pleroma.ModerationLog do
     |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, action: String.t(), target: String.t()}) ::
+  def insert_log(%{actor: %User{}, action: action, target: target} = attrs)
-          {:ok, ModerationLog} | {:error, any}
-  def insert_log(%{
-        actor: %User{} = actor,
-        action: action,
-        target: target
-      })
       when action in ["relay_follow", "relay_unfollow"] do
-    %ModerationLog{
+    data =
-      data: %{
+      attrs
-        "actor" => user_to_map(actor),
+      |> prepare_log_data
-        "action" => action,
+      |> Map.merge(%{"target" => target})
-        "target" => target,
+
-        "message" => ""
+    insert_log_entry_with_message(%ModerationLog{data: data})
-      }
-    }
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, action: String.t(), subject_id: String.t()}) ::
-          {:ok, ModerationLog} | {:error, any}
   def insert_log(%{actor: %User{} = actor, action: "chat_message_delete", subject_id: subject_id}) do
     %ModerationLog{
       data: %{
@@ -345,32 +256,27 @@ defmodule Pleroma.ModerationLog do
   end
 
   defp user_to_map(users) when is_list(users) do
-    users |> Enum.map(&user_to_map/1)
+    Enum.map(users, &user_to_map/1)
   end
 
   defp user_to_map(%User{} = user) do
     user
-    |> Map.from_struct()
     |> Map.take([:id, :nickname])
     |> Map.new(fn {k, v} -> {Atom.to_string(k), v} end)
     |> Map.put("type", "user")
   end
 
+  defp user_to_map(_), do: nil
+
   defp report_to_map(%Activity{} = report) do
-    %{
+    %{"type" => "report", "id" => report.id, "state" => report.data["state"]}
-      "type" => "report",
-      "id" => report.id,
-      "state" => report.data["state"]
-    }
   end
 
   defp status_to_map(%Activity{} = status) do
-    %{
+    %{"type" => "status", "id" => status.id}
-      "type" => "status",
-      "id" => status.id
-    }
   end
 
+  @spec get_log_entry_message(ModerationLog.t()) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -382,7 +288,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} made @#{follower_nickname} #{action} @#{followed_nickname}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -393,7 +298,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} deleted users: #{users_to_nicknames_string(subjects)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -404,7 +308,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} created users: #{users_to_nicknames_string(subjects)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -415,7 +318,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} activated users: #{users_to_nicknames_string(users)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -426,7 +328,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} deactivated users: #{users_to_nicknames_string(users)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -437,7 +338,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} approved users: #{users_to_nicknames_string(users)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -451,7 +351,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} added tags: #{tags_string} to users: #{nicknames_to_string(nicknames)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -465,7 +364,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} removed tags: #{tags_string} from users: #{nicknames_to_string(nicknames)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -477,7 +375,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} made #{users_to_nicknames_string(users)} #{permission}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -489,7 +386,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} revoked #{permission} role from #{users_to_nicknames_string(users)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -500,7 +396,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} followed relay: #{target}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -511,42 +406,48 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} unfollowed relay: #{target}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
+  def get_log_entry_message(
-  def get_log_entry_message(%ModerationLog{
+        %ModerationLog{
           data: %{
             "actor" => %{"nickname" => actor_nickname},
             "action" => "report_update",
             "subject" => %{"id" => subject_id, "state" => state, "type" => "report"}
           }
-      }) do
+        } = log
-    "@#{actor_nickname} updated report ##{subject_id} with '#{state}' state"
+      ) do
+    "@#{actor_nickname} updated report ##{subject_id}" <>
+      subject_actor_nickname(log, " (on user ", ")") <>
+      " with '#{state}' state"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
+  def get_log_entry_message(
-  def get_log_entry_message(%ModerationLog{
+        %ModerationLog{
           data: %{
             "actor" => %{"nickname" => actor_nickname},
             "action" => "report_note",
             "subject" => %{"id" => subject_id, "type" => "report"},
             "text" => text
           }
-      }) do
+        } = log
-    "@#{actor_nickname} added note '#{text}' to report ##{subject_id}"
+      ) do
+    "@#{actor_nickname} added note '#{text}' to report ##{subject_id}" <>
+      subject_actor_nickname(log, " on user ")
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
+  def get_log_entry_message(
-  def get_log_entry_message(%ModerationLog{
+        %ModerationLog{
           data: %{
             "actor" => %{"nickname" => actor_nickname},
             "action" => "report_note_delete",
             "subject" => %{"id" => subject_id, "type" => "report"},
             "text" => text
           }
-      }) do
+        } = log
-    "@#{actor_nickname} deleted note '#{text}' from report ##{subject_id}"
+      ) do
+    "@#{actor_nickname} deleted note '#{text}' from report ##{subject_id}" <>
+      subject_actor_nickname(log, " on user ")
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -559,7 +460,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} updated status ##{subject_id}, set visibility: '#{visibility}'"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -572,7 +472,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} updated status ##{subject_id}, set sensitive: '#{sensitive}'"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -587,7 +486,6 @@ defmodule Pleroma.ModerationLog do
     }'"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -598,7 +496,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} deleted status ##{subject_id}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -609,7 +506,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} forced password reset for users: #{users_to_nicknames_string(subjects)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -620,7 +516,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} confirmed email for users: #{users_to_nicknames_string(subjects)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -633,7 +528,6 @@ defmodule Pleroma.ModerationLog do
     }"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -644,7 +538,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} updated users: #{users_to_nicknames_string(subjects)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -655,6 +548,16 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} deleted chat message ##{subject_id}"
   end
 
+  def get_log_entry_message(%ModerationLog{
+        data: %{
+          "actor" => %{"nickname" => actor_nickname},
+          "action" => "create_backup",
+          "subject" => %{"nickname" => user_nickname}
+        }
+      }) do
+    "@#{actor_nickname} requested account backup for @#{user_nickname}"
+  end
+
   defp nicknames_to_string(nicknames) do
     nicknames
     |> Enum.map(&"@#{&1}")
@@ -666,4 +569,16 @@ defmodule Pleroma.ModerationLog do
     |> Enum.map(&"@#{&1["nickname"]}")
     |> Enum.join(", ")
   end
+
+  defp subject_actor_nickname(%ModerationLog{data: data}, prefix_msg, postfix_msg \\ "") do
+    case data do
+      %{"subject_actor" => %{"nickname" => subject_actor}} ->
+        [prefix_msg, "@#{subject_actor}", postfix_msg]
+        |> Enum.reject(&(&1 == ""))
+        |> Enum.join()
+
+      _ ->
+        ""
+    end
+  end
 end

lib/pleroma/notification.ex

@@ -70,6 +70,7 @@ defmodule Pleroma.Notification do
     move
     pleroma:chat_mention
     pleroma:emoji_reaction
+    pleroma:report
     reblog
   }
 
@@ -367,7 +368,7 @@ defmodule Pleroma.Notification do
   end
 
   def create_notifications(%Activity{data: %{"type" => type}} = activity, options)
-      when type in ["Follow", "Like", "Announce", "Move", "EmojiReact"] do
+      when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag"] do
     do_create_notifications(activity, options)
   end
 
@@ -410,6 +411,9 @@ defmodule Pleroma.Notification do
       "EmojiReact" ->
         "pleroma:emoji_reaction"
 
+      "Flag" ->
+        "pleroma:report"
+
       # Compatibility with old reactions
       "EmojiReaction" ->
         "pleroma:emoji_reaction"
@@ -467,7 +471,7 @@ defmodule Pleroma.Notification do
   def get_notified_from_activity(activity, local_only \\ true)
 
   def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)
-      when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do
+      when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact", "Flag"] do
     potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity)
 
     potential_receivers =
@@ -503,6 +507,10 @@ defmodule Pleroma.Notification do
     [object_id]
   end
 
+  def get_potential_receiver_ap_ids(%{data: %{"type" => "Flag"}}) do
+    User.all_superusers() |> Enum.map(fn user -> user.ap_id end)
+  end
+
   def get_potential_receiver_ap_ids(activity) do
     []
     |> Utils.maybe_notify_to_recipients(activity)

lib/pleroma/object.ex

@@ -23,6 +23,8 @@ defmodule Pleroma.Object do
 
   @derive {Jason.Encoder, only: [:data]}
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   schema "objects" do
     field(:data, :map)
 
@@ -156,9 +158,9 @@ defmodule Pleroma.Object do
   def get_cached_by_ap_id(ap_id) do
     key = "object:#{ap_id}"
 
-    with {:ok, nil} <- Cachex.get(:object_cache, key),
+    with {:ok, nil} <- @cachex.get(:object_cache, key),
          object when not is_nil(object) <- get_by_ap_id(ap_id),
-         {:ok, true} <- Cachex.put(:object_cache, key, object) do
+         {:ok, true} <- @cachex.put(:object_cache, key, object) do
       object
     else
       {:ok, object} -> object
@@ -216,13 +218,13 @@ defmodule Pleroma.Object do
   end
 
   def invalid_object_cache(%Object{data: %{"id" => id}}) do
-    with {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
+    with {:ok, true} <- @cachex.del(:object_cache, "object:#{id}") do
-      Cachex.del(:web_resp_cache, URI.parse(id).path)
+      @cachex.del(:web_resp_cache, URI.parse(id).path)
     end
   end
 
   def set_cache(%Object{data: %{"id" => ap_id}} = object) do
-    Cachex.put(:object_cache, "object:#{ap_id}", object)
+    @cachex.put(:object_cache, "object:#{ap_id}", object)
     {:ok, object}
   end
 

lib/pleroma/object/fetcher.ex

@@ -12,7 +12,6 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
-  alias Pleroma.Web.FedSockets
 
   require Logger
   require Pleroma.Constants
@@ -183,16 +182,16 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
-  def fetch_and_contain_remote_object_from_id(prm, opts \\ [])
+  def fetch_and_contain_remote_object_from_id(id)
 
-  def fetch_and_contain_remote_object_from_id(%{"id" => id}, opts),
+  def fetch_and_contain_remote_object_from_id(%{"id" => id}),
-    do: fetch_and_contain_remote_object_from_id(id, opts)
+    do: fetch_and_contain_remote_object_from_id(id)
 
-  def fetch_and_contain_remote_object_from_id(id, opts) when is_binary(id) do
+  def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
     Logger.debug("Fetching object #{id} via AP")
 
     with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
-         {:ok, body} <- get_object(id, opts),
+         {:ok, body} <- get_object(id),
          {:ok, data} <- safe_json_decode(body),
          :ok <- Containment.contain_origin_from_id(id, data) do
       {:ok, data}
@@ -208,22 +207,10 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
-  def fetch_and_contain_remote_object_from_id(_id, _opts),
+  def fetch_and_contain_remote_object_from_id(_id),
     do: {:error, "id must be a string"}
 
-  defp get_object(id, opts) do
+  defp get_object(id) do
-    with false <- Keyword.get(opts, :force_http, false),
-         {:ok, fedsocket} <- FedSockets.get_or_create_fed_socket(id) do
-      Logger.debug("fetching via fedsocket - #{inspect(id)}")
-      FedSockets.fetch(fedsocket, id)
-    else
-      _other ->
-        Logger.debug("fetching via http - #{inspect(id)}")
-        get_object_http(id)
-    end
-  end
-
-  defp get_object_http(id) do
     date = Pleroma.Signature.signed_date()
 
     headers =
@@ -232,9 +219,25 @@ defmodule Pleroma.Object.Fetcher do
       |> sign_fetch(id, date)
 
     case HTTP.get(id, headers) do
-      {:ok, %{body: body, status: code}} when code in 200..299 ->
+      {:ok, %{body: body, status: code, headers: headers}} when code in 200..299 ->
+        case List.keyfind(headers, "content-type", 0) do
+          {_, content_type} ->
+            case Plug.Conn.Utils.media_type(content_type) do
+              {:ok, "application", "activity+json", _} ->
                 {:ok, body}
 
+              {:ok, "application", "ld+json",
+               %{"profile" => "https://www.w3.org/ns/activitystreams"}} ->
+                {:ok, body}
+
+              _ ->
+                {:error, {:content_type, content_type}}
+            end
+
+          _ ->
+            {:error, {:content_type, nil}}
+        end
+
       {:ok, %{status: code}} when code in [404, 410] ->
         {:error, "Object has been deleted"}
 

lib/pleroma/password_reset_token.ex

@@ -40,6 +40,7 @@ defmodule Pleroma.PasswordResetToken do
   @spec reset_password(binary(), map()) :: {:ok, User.t()} | {:error, binary()}
   def reset_password(token, data) do
     with %{used: false} = token <- Repo.get_by(PasswordResetToken, %{token: token}),
+         false <- expired?(token),
          %User{} = user <- User.get_cached_by_id(token.user_id),
          {:ok, _user} <- User.reset_password(user, data),
          {:ok, token} <- Repo.update(used_changeset(token)) do
@@ -48,4 +49,14 @@ defmodule Pleroma.PasswordResetToken do
       _e -> {:error, token}
     end
   end
+
+  def expired?(%__MODULE__{inserted_at: inserted_at}) do
+    validity = Pleroma.Config.get([:instance, :password_reset_token_validity], 0)
+
+    now = NaiveDateTime.utc_now()
+
+    difference = NaiveDateTime.diff(now, inserted_at)
+
+    difference > validity
+  end
 end

lib/pleroma/reverse_proxy.ex

@@ -17,6 +17,8 @@ defmodule Pleroma.ReverseProxy do
   @failed_request_ttl :timer.seconds(60)
   @methods ~w(GET HEAD)
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   def max_read_duration_default, do: @max_read_duration
   def default_cache_control_header, do: @default_cache_control_header
 
@@ -107,7 +109,7 @@ defmodule Pleroma.ReverseProxy do
         opts
       end
 
-    with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url),
+    with {:ok, nil} <- @cachex.get(:failed_proxy_url_cache, url),
          {:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
          :ok <-
            header_length_constraint(
@@ -427,6 +429,6 @@ defmodule Pleroma.ReverseProxy do
         nil
       end
 
-    Cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl)
+    @cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl)
   end
 end

lib/pleroma/signature.ex

@@ -39,7 +39,7 @@ defmodule Pleroma.Signature do
   def fetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
          {:ok, actor_id} <- key_id_to_actor_id(kid),
-         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id, force_http: true) do
+         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
     else
       e ->
@@ -50,8 +50,8 @@ defmodule Pleroma.Signature do
   def refetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
          {:ok, actor_id} <- key_id_to_actor_id(kid),
-         {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id, force_http: true),
+         {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
-         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id, force_http: true) do
+         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
     else
       e ->

lib/pleroma/stats.ex

@@ -23,7 +23,6 @@ defmodule Pleroma.Stats do
 
   @impl true
   def init(_args) do
-    if Pleroma.Config.get(:env) == :test, do: :ok = Ecto.Adapters.SQL.Sandbox.checkout(Repo)
     {:ok, nil, {:continue, :calculate_stats}}
   end
 
@@ -32,11 +31,6 @@ defmodule Pleroma.Stats do
     GenServer.call(__MODULE__, :force_update)
   end
 
-  @doc "Performs collect stats"
-  def do_collect do
-    GenServer.cast(__MODULE__, :run_update)
-  end
-
   @doc "Returns stats data"
   @spec get_stats() :: %{
           domain_count: non_neg_integer(),
@@ -111,7 +105,11 @@ defmodule Pleroma.Stats do
   @impl true
   def handle_continue(:calculate_stats, _) do
     stats = calculate_stat_data()
+
+    unless Pleroma.Config.get(:env) == :test do
       Process.send_after(self(), :run_update, @interval)
+    end
+
     {:noreply, stats}
   end
 
@@ -126,13 +124,6 @@ defmodule Pleroma.Stats do
     {:reply, state, state}
   end
 
-  @impl true
-  def handle_cast(:run_update, _state) do
-    new_stats = calculate_stat_data()
-
-    {:noreply, new_stats}
-  end
-
   @impl true
   def handle_info(:run_update, _) do
     new_stats = calculate_stat_data()

lib/pleroma/user.ex

@@ -82,6 +82,8 @@ defmodule Pleroma.User do
     ]
   ]
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   schema "users" do
     field(:bio, :string, default: "")
     field(:raw_bio, :string)
@@ -129,7 +131,6 @@ defmodule Pleroma.User do
     field(:hide_followers, :boolean, default: false)
     field(:hide_follows, :boolean, default: false)
     field(:hide_favorites, :boolean, default: true)
-    field(:unread_conversation_count, :integer, default: 0)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:email_notifications, :map, default: %{"digest" => false})
     field(:mascot, :map, default: nil)
@@ -137,7 +138,7 @@ defmodule Pleroma.User do
     field(:pleroma_settings_store, :map, default: %{})
     field(:fields, {:array, :map}, default: [])
     field(:raw_fields, {:array, :map}, default: [])
-    field(:discoverable, :boolean, default: false)
+    field(:is_discoverable, :boolean, default: false)
     field(:invisible, :boolean, default: false)
     field(:allow_following_move, :boolean, default: true)
     field(:skip_thread_containment, :boolean, default: false)
@@ -247,6 +248,18 @@ defmodule Pleroma.User do
     end
   end
 
+  def cached_blocked_users_ap_ids(user) do
+    @cachex.fetch!(:user_cache, "blocked_users_ap_ids:#{user.ap_id}", fn _ ->
+      blocked_users_ap_ids(user)
+    end)
+  end
+
+  def cached_muted_users_ap_ids(user) do
+    @cachex.fetch!(:user_cache, "muted_users_ap_ids:#{user.ap_id}", fn _ ->
+      muted_users_ap_ids(user)
+    end)
+  end
+
   defdelegate following_count(user), to: FollowingRelationship
   defdelegate following(user), to: FollowingRelationship
   defdelegate following?(follower, followed), to: FollowingRelationship
@@ -427,7 +440,6 @@ defmodule Pleroma.User do
       params,
       [
         :bio,
-        :name,
         :emoji,
         :ap_id,
         :inbox,
@@ -449,19 +461,33 @@ defmodule Pleroma.User do
         :follower_count,
         :fields,
         :following_count,
-        :discoverable,
+        :is_discoverable,
         :invisible,
         :actor_type,
         :also_known_as,
         :accepts_chat_messages
       ]
     )
-    |> validate_required([:name, :ap_id])
+    |> cast(params, [:name], empty_values: [])
+    |> validate_required([:ap_id])
+    |> validate_required([:name], trim: false)
     |> unique_constraint(:nickname)
     |> validate_format(:nickname, @email_regex)
     |> validate_length(:bio, max: bio_limit)
     |> validate_length(:name, max: name_limit)
     |> validate_fields(true)
+    |> validate_non_local()
+  end
+
+  defp validate_non_local(cng) do
+    local? = get_field(cng, :local)
+
+    if local? do
+      cng
+      |> add_error(:local, "User is local, can't update with this changeset.")
+    else
+      cng
+    end
   end
 
   def update_changeset(struct, params \\ %{}) do
@@ -497,7 +523,7 @@ defmodule Pleroma.User do
         :fields,
         :raw_fields,
         :pleroma_settings_store,
-        :discoverable,
+        :is_discoverable,
         :actor_type,
         :accepts_chat_messages
       ]
@@ -767,6 +793,16 @@ defmodule Pleroma.User do
     follow_all(user, autofollowed_users)
   end
 
+  defp autofollowing_users(user) do
+    candidates = Config.get([:instance, :autofollowing_nicknames])
+
+    User.Query.build(%{nickname: candidates, local: true, deactivated: false})
+    |> Repo.all()
+    |> Enum.each(&follow(&1, user, :follow_accept))
+
+    {:ok, :success}
+  end
+
   @doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)"
   def register(%Ecto.Changeset{} = changeset) do
     with {:ok, user} <- Repo.insert(changeset) do
@@ -774,17 +810,50 @@ defmodule Pleroma.User do
     end
   end
 
-  def post_register_action(%User{} = user) do
+  def post_register_action(%User{confirmation_pending: true} = user) do
+    with {:ok, _} <- try_send_confirmation_email(user) do
+      {:ok, user}
+    end
+  end
+
+  def post_register_action(%User{approval_pending: true} = user) do
+    with {:ok, _} <- send_user_approval_email(user),
+         {:ok, _} <- send_admin_approval_emails(user) do
+      {:ok, user}
+    end
+  end
+
+  def post_register_action(%User{approval_pending: false, confirmation_pending: false} = user) do
     with {:ok, user} <- autofollow_users(user),
+         {:ok, _} <- autofollowing_users(user),
          {:ok, user} <- set_cache(user),
          {:ok, _} <- send_welcome_email(user),
          {:ok, _} <- send_welcome_message(user),
-         {:ok, _} <- send_welcome_chat_message(user),
+         {:ok, _} <- send_welcome_chat_message(user) do
-         {:ok, _} <- try_send_confirmation_email(user) do
       {:ok, user}
     end
   end
 
+  defp send_user_approval_email(user) do
+    user
+    |> Pleroma.Emails.UserEmail.approval_pending_email()
+    |> Pleroma.Emails.Mailer.deliver_async()
+
+    {:ok, :enqueued}
+  end
+
+  defp send_admin_approval_emails(user) do
+    all_superusers()
+    |> Enum.filter(fn user -> not is_nil(user.email) end)
+    |> Enum.each(fn superuser ->
+      superuser
+      |> Pleroma.Emails.AdminEmail.new_unapproved_registration(user)
+      |> Pleroma.Emails.Mailer.deliver_async()
+    end)
+
+    {:ok, :enqueued}
+  end
+
   def send_welcome_message(user) do
     if User.WelcomeMessage.enabled?() do
       User.WelcomeMessage.post_message(user)
@@ -861,7 +930,7 @@ defmodule Pleroma.User do
     if not ap_enabled?(followed) do
       follow(follower, followed)
     else
-      {:ok, follower}
+      {:ok, follower, followed}
     end
   end
 
@@ -887,11 +956,6 @@ defmodule Pleroma.User do
 
       true ->
         FollowingRelationship.follow(follower, followed, state)
-
-        {:ok, _} = update_follower_count(followed)
-
-        follower
-        |> update_following_count()
     end
   end
 
@@ -915,11 +979,6 @@ defmodule Pleroma.User do
     case get_follow_state(follower, followed) do
       state when state in [:follow_pending, :follow_accept] ->
         FollowingRelationship.unfollow(follower, followed)
-        {:ok, followed} = update_follower_count(followed)
-
-        {:ok, follower} = update_following_count(follower)
-
-        {:ok, follower, followed}
 
       nil ->
         {:error, "Not subscribed!"}
@@ -993,9 +1052,9 @@ defmodule Pleroma.User do
   def set_cache({:error, err}), do: {:error, err}
 
   def set_cache(%User{} = user) do
-    Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
+    @cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
-    Cachex.put(:user_cache, "nickname:#{user.nickname}", user)
+    @cachex.put(:user_cache, "nickname:#{user.nickname}", user)
-    Cachex.put(:user_cache, "friends_ap_ids:#{user.nickname}", get_user_friends_ap_ids(user))
+    @cachex.put(:user_cache, "friends_ap_ids:#{user.nickname}", get_user_friends_ap_ids(user))
     {:ok, user}
   end
 
@@ -1018,24 +1077,26 @@ defmodule Pleroma.User do
 
   @spec get_cached_user_friends_ap_ids(User.t()) :: [String.t()]
   def get_cached_user_friends_ap_ids(user) do
-    Cachex.fetch!(:user_cache, "friends_ap_ids:#{user.ap_id}", fn _ ->
+    @cachex.fetch!(:user_cache, "friends_ap_ids:#{user.ap_id}", fn _ ->
       get_user_friends_ap_ids(user)
     end)
   end
 
   def invalidate_cache(user) do
-    Cachex.del(:user_cache, "ap_id:#{user.ap_id}")
+    @cachex.del(:user_cache, "ap_id:#{user.ap_id}")
-    Cachex.del(:user_cache, "nickname:#{user.nickname}")
+    @cachex.del(:user_cache, "nickname:#{user.nickname}")
-    Cachex.del(:user_cache, "friends_ap_ids:#{user.ap_id}")
+    @cachex.del(:user_cache, "friends_ap_ids:#{user.ap_id}")
+    @cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
+    @cachex.del(:user_cache, "muted_users_ap_ids:#{user.ap_id}")
   end
 
   @spec get_cached_by_ap_id(String.t()) :: User.t() | nil
   def get_cached_by_ap_id(ap_id) do
     key = "ap_id:#{ap_id}"
 
-    with {:ok, nil} <- Cachex.get(:user_cache, key),
+    with {:ok, nil} <- @cachex.get(:user_cache, key),
          user when not is_nil(user) <- get_by_ap_id(ap_id),
-         {:ok, true} <- Cachex.put(:user_cache, key, user) do
+         {:ok, true} <- @cachex.put(:user_cache, key, user) do
       user
     else
       {:ok, user} -> user
@@ -1047,11 +1108,11 @@ defmodule Pleroma.User do
     key = "id:#{id}"
 
     ap_id =
-      Cachex.fetch!(:user_cache, key, fn _ ->
+      @cachex.fetch!(:user_cache, key, fn _ ->
         user = get_by_id(id)
 
         if user do
-          Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
+          @cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
           {:commit, user.ap_id}
         else
           {:ignore, ""}
@@ -1064,7 +1125,7 @@ defmodule Pleroma.User do
   def get_cached_by_nickname(nickname) do
     key = "nickname:#{nickname}"
 
-    Cachex.fetch!(:user_cache, key, fn ->
+    @cachex.fetch!(:user_cache, key, fn _ ->
       case get_or_fetch_by_nickname(nickname) do
         {:ok, user} -> {:commit, user}
         {:error, _error} -> {:ignore, nil}
@@ -1295,47 +1356,6 @@ defmodule Pleroma.User do
     |> update_and_set_cache()
   end
 
-  def set_unread_conversation_count(%User{local: true} = user) do
-    unread_query = Participation.unread_conversation_count_for_user(user)
-
-    User
-    |> join(:inner, [u], p in subquery(unread_query))
-    |> update([u, p],
-      set: [unread_conversation_count: p.count]
-    )
-    |> where([u], u.id == ^user.id)
-    |> select([u], u)
-    |> Repo.update_all([])
-    |> case do
-      {1, [user]} -> set_cache(user)
-      _ -> {:error, user}
-    end
-  end
-
-  def set_unread_conversation_count(user), do: {:ok, user}
-
-  def increment_unread_conversation_count(conversation, %User{local: true} = user) do
-    unread_query =
-      Participation.unread_conversation_count_for_user(user)
-      |> where([p], p.conversation_id == ^conversation.id)
-
-    User
-    |> join(:inner, [u], p in subquery(unread_query))
-    |> update([u, p],
-      inc: [unread_conversation_count: 1]
-    )
-    |> where([u], u.id == ^user.id)
-    |> where([u, p], p.count == 0)
-    |> select([u], u)
-    |> Repo.update_all([])
-    |> case do
-      {1, [user]} -> set_cache(user)
-      _ -> {:error, user}
-    end
-  end
-
-  def increment_unread_conversation_count(_, user), do: {:ok, user}
-
   @spec get_users_from_set([String.t()], keyword()) :: [User.t()]
   def get_users_from_set(ap_ids, opts \\ []) do
     local_only = Keyword.get(opts, :local_only, true)
@@ -1356,14 +1376,51 @@ defmodule Pleroma.User do
     |> Repo.all()
   end
 
-  @spec mute(User.t(), User.t(), boolean()) ::
+  @spec mute(User.t(), User.t(), map()) ::
           {:ok, list(UserRelationship.t())} | {:error, String.t()}
-  def mute(%User{} = muter, %User{} = mutee, notifications? \\ true) do
+  def mute(%User{} = muter, %User{} = mutee, params \\ %{}) do
-    add_to_mutes(muter, mutee, notifications?)
+    notifications? = Map.get(params, :notifications, true)
+    expires_in = Map.get(params, :expires_in, 0)
+
+    with {:ok, user_mute} <- UserRelationship.create_mute(muter, mutee),
+         {:ok, user_notification_mute} <-
+           (notifications? && UserRelationship.create_notification_mute(muter, mutee)) ||
+             {:ok, nil} do
+      if expires_in > 0 do
+        Pleroma.Workers.MuteExpireWorker.enqueue(
+          "unmute_user",
+          %{"muter_id" => muter.id, "mutee_id" => mutee.id},
+          schedule_in: expires_in
+        )
+      end
+
+      @cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}")
+
+      {:ok, Enum.filter([user_mute, user_notification_mute], & &1)}
+    end
   end
 
   def unmute(%User{} = muter, %User{} = mutee) do
-    remove_from_mutes(muter, mutee)
+    with {:ok, user_mute} <- UserRelationship.delete_mute(muter, mutee),
+         {:ok, user_notification_mute} <-
+           UserRelationship.delete_notification_mute(muter, mutee) do
+      @cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}")
+      {:ok, [user_mute, user_notification_mute]}
+    end
+  end
+
+  def unmute(muter_id, mutee_id) do
+    with {:muter, %User{} = muter} <- {:muter, User.get_by_id(muter_id)},
+         {:mutee, %User{} = mutee} <- {:mutee, User.get_by_id(mutee_id)} do
+      unmute(muter, mutee)
+    else
+      {who, result} = error ->
+        Logger.warn(
+          "User.unmute/2 failed. #{who}: #{result}, muter_id: #{muter_id}, mutee_id: #{mutee_id}"
+        )
+
+        {:error, error}
+    end
   end
 
   def subscribe(%User{} = subscriber, %User{} = target) do
@@ -1569,10 +1626,33 @@ defmodule Pleroma.User do
     end)
   end
 
-  def approve(%User{} = user) do
+  def approve(%User{approval_pending: true} = user) do
-    change(user, approval_pending: false)
+    with chg <- change(user, approval_pending: false),
-    |> update_and_set_cache()
+         {:ok, user} <- update_and_set_cache(chg) do
+      post_register_action(user)
+      {:ok, user}
     end
+  end
+
+  def approve(%User{} = user), do: {:ok, user}
+
+  def confirm(users) when is_list(users) do
+    Repo.transaction(fn ->
+      Enum.map(users, fn user ->
+        with {:ok, user} <- confirm(user), do: user
+      end)
+    end)
+  end
+
+  def confirm(%User{confirmation_pending: true} = user) do
+    with chg <- confirmation_changeset(user, need_confirmation: false),
+         {:ok, user} <- update_and_set_cache(chg) do
+      post_register_action(user)
+      {:ok, user}
+    end
+  end
+
+  def confirm(%User{} = user), do: {:ok, user}
 
   def update_notification_settings(%User{} = user, settings) do
     user
@@ -1620,7 +1700,7 @@ defmodule Pleroma.User do
       pleroma_settings_store: %{},
       fields: [],
       raw_fields: [],
-      discoverable: false,
+      is_discoverable: false,
       also_known_as: []
     })
   end
@@ -1770,12 +1850,12 @@ defmodule Pleroma.User do
 
   def html_filter_policy(_), do: Config.get([:markup, :scrub_policy])
 
-  def fetch_by_ap_id(ap_id, opts \\ []), do: ActivityPub.make_user_from_ap_id(ap_id, opts)
+  def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id)
 
-  def get_or_fetch_by_ap_id(ap_id, opts \\ []) do
+  def get_or_fetch_by_ap_id(ap_id) do
     cached_user = get_cached_by_ap_id(ap_id)
 
-    maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id, opts)
+    maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id)
 
     case {cached_user, maybe_fetched_user} do
       {_, {:ok, %User{} = user}} ->
@@ -1848,8 +1928,8 @@ defmodule Pleroma.User do
 
   def public_key(_), do: {:error, "key not found"}
 
-  def get_public_key_for_ap_id(ap_id, opts \\ []) do
+  def get_public_key_for_ap_id(ap_id) do
-    with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id, opts),
+    with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id),
          {:ok, public_key} <- public_key(user) do
       {:ok, public_key}
     else
@@ -2060,18 +2140,6 @@ defmodule Pleroma.User do
     updated_user
   end
 
-  @spec toggle_confirmation(User.t()) :: {:ok, User.t()} | {:error, Changeset.t()}
-  def toggle_confirmation(%User{} = user) do
-    user
-    |> confirmation_changeset(need_confirmation: !user.confirmation_pending)
-    |> update_and_set_cache()
-  end
-
-  @spec toggle_confirmation([User.t()]) :: [{:ok, User.t()} | {:error, Changeset.t()}]
-  def toggle_confirmation(users) do
-    Enum.map(users, &toggle_confirmation/1)
-  end
-
   @spec need_confirmation(User.t(), boolean()) :: {:ok, User.t()} | {:error, Changeset.t()}
   def need_confirmation(%User{} = user, bool) do
     user
@@ -2343,29 +2411,18 @@ defmodule Pleroma.User do
   @spec add_to_block(User.t(), User.t()) ::
           {:ok, UserRelationship.t()} | {:error, Ecto.Changeset.t()}
   defp add_to_block(%User{} = user, %User{} = blocked) do
-    UserRelationship.create_block(user, blocked)
+    with {:ok, relationship} <- UserRelationship.create_block(user, blocked) do
+      @cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
+      {:ok, relationship}
+    end
   end
 
   @spec add_to_block(User.t(), User.t()) ::
           {:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()}
   defp remove_from_block(%User{} = user, %User{} = blocked) do
-    UserRelationship.delete_block(user, blocked)
+    with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do
-  end
+      @cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
-
+      {:ok, relationship}
-  defp add_to_mutes(%User{} = user, %User{} = muted_user, notifications?) do
-    with {:ok, user_mute} <- UserRelationship.create_mute(user, muted_user),
-         {:ok, user_notification_mute} <-
-           (notifications? && UserRelationship.create_notification_mute(user, muted_user)) ||
-             {:ok, nil} do
-      {:ok, Enum.filter([user_mute, user_notification_mute], & &1)}
-    end
-  end
-
-  defp remove_from_mutes(user, %User{} = muted_user) do
-    with {:ok, user_mute} <- UserRelationship.delete_mute(user, muted_user),
-         {:ok, user_notification_mute} <-
-           UserRelationship.delete_notification_mute(user, muted_user) do
-      {:ok, [user_mute, user_notification_mute]}
     end
   end
 
@@ -2407,4 +2464,8 @@ defmodule Pleroma.User do
       end
     end)
   end
+
+  def get_host(%User{ap_id: ap_id} = _user) do
+    URI.parse(ap_id).host
+  end
 end

lib/pleroma/user/backup.ex

@@ -0,0 +1,258 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.Backup do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+  import Ecto.Query
+  import Pleroma.Web.Gettext
+
+  require Pleroma.Constants
+
+  alias Pleroma.Activity
+  alias Pleroma.Bookmark
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.ActivityPub.UserView
+  alias Pleroma.Workers.BackupWorker
+
+  schema "backups" do
+    field(:content_type, :string)
+    field(:file_name, :string)
+    field(:file_size, :integer, default: 0)
+    field(:processed, :boolean, default: false)
+
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
+
+    timestamps()
+  end
+
+  def create(user, admin_id \\ nil) do
+    with :ok <- validate_email_enabled(),
+         :ok <- validate_user_email(user),
+         :ok <- validate_limit(user, admin_id),
+         {:ok, backup} <- user |> new() |> Repo.insert() do
+      BackupWorker.process(backup, admin_id)
+    end
+  end
+
+  def new(user) do
+    rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+    datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
+    name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
+
+    %__MODULE__{
+      user_id: user.id,
+      content_type: "application/zip",
+      file_name: name
+    }
+  end
+
+  def delete(backup) do
+    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+
+    with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do
+      Repo.delete(backup)
+    end
+  end
+
+  defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok
+
+  defp validate_limit(user, nil) do
+    case get_last(user.id) do
+      %__MODULE__{inserted_at: inserted_at} ->
+        days = Pleroma.Config.get([__MODULE__, :limit_days])
+        diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
+
+        if diff > days do
+          :ok
+        else
+          {:error,
+           dngettext(
+             "errors",
+             "Last export was less than a day ago",
+             "Last export was less than %{days} days ago",
+             days,
+             days: days
+           )}
+        end
+
+      nil ->
+        :ok
+    end
+  end
+
+  defp validate_email_enabled do
+    if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
+      :ok
+    else
+      {:error, dgettext("errors", "Backups require enabled email")}
+    end
+  end
+
+  defp validate_user_email(%User{email: nil}) do
+    {:error, dgettext("errors", "Email is required")}
+  end
+
+  defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok
+
+  def get_last(user_id) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> order_by(desc: :id)
+    |> limit(1)
+    |> Repo.one()
+  end
+
+  def list(%User{id: user_id}) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> order_by(desc: :id)
+    |> Repo.all()
+  end
+
+  def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> where([b], b.id != ^latest_id)
+    |> Repo.all()
+    |> Enum.each(&BackupWorker.delete/1)
+  end
+
+  def get(id), do: Repo.get(__MODULE__, id)
+
+  def process(%__MODULE__{} = backup) do
+    with {:ok, zip_file} <- export(backup),
+         {:ok, %{size: size}} <- File.stat(zip_file),
+         {:ok, _upload} <- upload(backup, zip_file) do
+      backup
+      |> cast(%{file_size: size, processed: true}, [:file_size, :processed])
+      |> Repo.update()
+    end
+  end
+
+  @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
+  def export(%__MODULE__{} = backup) do
+    backup = Repo.preload(backup, :user)
+    name = String.trim_trailing(backup.file_name, ".zip")
+    dir = dir(name)
+
+    with :ok <- File.mkdir(dir),
+         :ok <- actor(dir, backup.user),
+         :ok <- statuses(dir, backup.user),
+         :ok <- likes(dir, backup.user),
+         :ok <- bookmarks(dir, backup.user),
+         {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
+         {:ok, _} <- File.rm_rf(dir) do
+      {:ok, to_string(zip_path)}
+    end
+  end
+
+  def dir(name) do
+    dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!()
+    Path.join(dir, name)
+  end
+
+  def upload(%__MODULE__{} = backup, zip_path) do
+    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+
+    upload = %Pleroma.Upload{
+      name: backup.file_name,
+      tempfile: zip_path,
+      content_type: backup.content_type,
+      path: Path.join("backups", backup.file_name)
+    }
+
+    with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
+         :ok <- File.rm(zip_path) do
+      {:ok, upload}
+    end
+  end
+
+  defp actor(dir, user) do
+    with {:ok, json} <-
+           UserView.render("user.json", %{user: user})
+           |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
+           |> Jason.encode() do
+      File.write(Path.join(dir, "actor.json"), json)
+    end
+  end
+
+  defp write_header(file, name) do
+    IO.write(
+      file,
+      """
+      {
+        "@context": "https://www.w3.org/ns/activitystreams",
+        "id": "#{name}.json",
+        "type": "OrderedCollection",
+        "orderedItems": [
+
+      """
+    )
+  end
+
+  defp write(query, dir, name, fun) do
+    path = Path.join(dir, "#{name}.json")
+
+    with {:ok, file} <- File.open(path, [:write, :utf8]),
+         :ok <- write_header(file, name) do
+      total =
+        query
+        |> Pleroma.Repo.chunk_stream(100)
+        |> Enum.reduce(0, fn i, acc ->
+          with {:ok, data} <- fun.(i),
+               {:ok, str} <- Jason.encode(data),
+               :ok <- IO.write(file, str <> ",\n") do
+            acc + 1
+          else
+            _ -> acc
+          end
+        end)
+
+      with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n  \"totalItems\": #{total}}") do
+        File.close(file)
+      end
+    end
+  end
+
+  defp bookmarks(dir, %{id: user_id} = _user) do
+    Bookmark
+    |> where(user_id: ^user_id)
+    |> join(:inner, [b], activity in assoc(b, :activity))
+    |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
+    |> write(dir, "bookmarks", fn a -> {:ok, a.object} end)
+  end
+
+  defp likes(dir, user) do
+    user.ap_id
+    |> Activity.Queries.by_actor()
+    |> Activity.Queries.by_type("Like")
+    |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
+    |> write(dir, "likes", fn a -> {:ok, a.object} end)
+  end
+
+  defp statuses(dir, user) do
+    opts =
+      %{}
+      |> Map.put(:type, ["Create", "Announce"])
+      |> Map.put(:actor_id, user.ap_id)
+
+    [
+      [Pleroma.Constants.as_public(), user.ap_id],
+      User.following(user),
+      Pleroma.List.memberships(user)
+    ]
+    |> Enum.concat()
+    |> ActivityPub.fetch_activities_query(opts)
+    |> write(dir, "outbox", fn a ->
+      with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
+        {:ok, Map.delete(activity, "@context")}
+      end
+    end)
+  end
+end

lib/pleroma/user/import.ex

@@ -45,7 +45,7 @@ defmodule Pleroma.User.Import do
       identifiers,
       fn identifier ->
         with {:ok, %User{} = followed} <- User.get_or_fetch(identifier),
-             {:ok, follower} <- User.maybe_direct_follow(follower, followed),
+             {:ok, follower, followed} <- User.maybe_direct_follow(follower, followed),
              {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do
           followed
         else

lib/pleroma/user/query.ex

@@ -43,6 +43,7 @@ defmodule Pleroma.User.Query do
             active: boolean(),
             deactivated: boolean(),
             need_approval: boolean(),
+            unconfirmed: boolean(),
             is_admin: boolean(),
             is_moderator: boolean(),
             super_users: boolean(),
@@ -55,7 +56,8 @@ defmodule Pleroma.User.Query do
             ap_id: [String.t()],
             order_by: term(),
             select: term(),
-            limit: pos_integer()
+            limit: pos_integer(),
+            actor_types: [String.t()]
           }
           | map()
 
@@ -114,6 +116,10 @@ defmodule Pleroma.User.Query do
     where(query, [u], u.is_admin == ^bool)
   end
 
+  defp compose_query({:actor_types, actor_types}, query) when is_list(actor_types) do
+    where(query, [u], u.actor_type in ^actor_types)
+  end
+
   defp compose_query({:is_moderator, bool}, query) do
     where(query, [u], u.is_moderator == ^bool)
   end
@@ -156,6 +162,10 @@ defmodule Pleroma.User.Query do
     where(query, [u], u.approval_pending)
   end
 
+  defp compose_query({:unconfirmed, _}, query) do
+    where(query, [u], u.confirmation_pending)
+  end
+
   defp compose_query({:followers, %User{id: id}}, query) do
     query
     |> where([u], u.id != ^id)

lib/pleroma/user/search.ex

@@ -85,7 +85,6 @@ defmodule Pleroma.User.Search do
     |> base_query(following)
     |> filter_blocked_user(for_user)
     |> filter_invisible_users()
-    |> filter_discoverable_users()
     |> filter_internal_users()
     |> filter_blocked_domains(for_user)
     |> fts_search(query_string)
@@ -163,10 +162,6 @@ defmodule Pleroma.User.Search do
     from(q in query, where: q.invisible == false)
   end
 
-  defp filter_discoverable_users(query) do
-    from(q in query, where: q.discoverable == true)
-  end
-
   defp filter_internal_users(query) do
     from(q in query, where: q.actor_type != "Application")
   end

lib/pleroma/utils.ex

@@ -3,6 +3,14 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Utils do
+  @posix_error_codes ~w(
+    eacces eagain ebadf ebadmsg ebusy edeadlk edeadlock edquot eexist efault
+    efbig eftype eintr einval eio eisdir eloop emfile emlink emultihop
+    enametoolong enfile enobufs enodev enolck enolink enoent enomem enospc
+    enosr enostr enosys enotblk enotdir enotsup enxio eopnotsupp eoverflow
+    eperm epipe erange erofs espipe esrch estale etxtbsy exdev
+  )a
+
   def compile_dir(dir) when is_binary(dir) do
     dir
     |> File.ls!()
@@ -44,4 +52,12 @@ defmodule Pleroma.Utils do
       error -> error
     end
   end
+
+  @spec posix_error_message(atom()) :: binary()
+  def posix_error_message(code) when code in @posix_error_codes do
+    error_message = Gettext.dgettext(Pleroma.Web.Gettext, "posix_errors", "#{code}")
+    "(POSIX error: #{error_message})"
+  end
+
+  def posix_error_message(_), do: ""
 end

lib/pleroma/web.ex

@@ -20,6 +20,7 @@ defmodule Pleroma.Web do
   below.
   """
 
+  alias Pleroma.Helpers.AuthHelper
   alias Pleroma.Web.Plugs.EnsureAuthenticatedPlug
   alias Pleroma.Web.Plugs.EnsurePublicOrAuthenticatedPlug
   alias Pleroma.Web.Plugs.ExpectAuthenticatedCheckPlug
@@ -75,7 +76,7 @@ defmodule Pleroma.Web do
       defp maybe_drop_authentication_if_oauth_check_ignored(conn) do
         if PlugHelper.plug_called?(conn, ExpectPublicOrAuthenticatedCheckPlug) and
              not PlugHelper.plug_called_or_skipped?(conn, OAuthScopesPlug) do
-          OAuthScopesPlug.drop_auth_info(conn)
+          AuthHelper.drop_auth_info(conn)
         else
           conn
         end
@@ -172,7 +173,7 @@ defmodule Pleroma.Web do
   def channel do
     quote do
       # credo:disable-for-next-line Credo.Check.Consistency.MultiAliasImportRequireUse
-      use Phoenix.Channel
+      import Phoenix.Channel
       import Pleroma.Web.Gettext
     end
   end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -32,6 +32,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   require Logger
   require Pleroma.Constants
 
+  @behaviour Pleroma.Web.ActivityPub.ActivityPub.Persisting
+
   defp get_recipients(%{"type" => "Create"} = data) do
     to = Map.get(data, "to", [])
     cc = Map.get(data, "cc", [])
@@ -85,13 +87,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
   @object_types ~w[ChatMessage Question Answer Audio Video Event Article]
-  @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
+  @impl true
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do
       {:ok, object, meta}
     end
   end
 
+  @impl true
   def persist(object, meta) do
     with local <- Keyword.fetch!(meta, :local),
          {recipients, _, _} <- get_recipients(object),
@@ -123,7 +126,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       # Splice in the child object if we have one.
       activity = Maps.put_if_present(activity, :object, object)
 
-      BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
+      ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
+        Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
+      end)
 
       {:ok, activity}
     else
@@ -332,7 +337,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   @spec flag(map()) :: {:ok, Activity.t()} | {:error, any()}
-  def flag(
+  def flag(params) do
+    with {:ok, result} <- Repo.transaction(fn -> do_flag(params) end) do
+      result
+    end
+  end
+
+  defp do_flag(
          %{
            actor: actor,
            context: _context,
@@ -358,7 +369,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          {:ok, activity} <- insert(flag_data, local),
          {:ok, stripped_activity} <- strip_report_status_data(activity),
          _ <- notify_and_stream(activity),
-         :ok <- maybe_federate(stripped_activity) do
+         :ok <-
+           maybe_federate(stripped_activity) do
       User.all_superusers()
       |> Enum.filter(fn user -> not is_nil(user.email) end)
       |> Enum.each(fn superuser ->
@@ -368,6 +380,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       end)
 
       {:ok, activity}
+    else
+      {:error, error} -> Repo.rollback(error)
     end
   end
 
@@ -827,7 +841,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     query =
       from([activity] in query,
         where: fragment("not (? = ANY(?))", activity.actor, ^mutes),
-        where: fragment("not (?->'to' \\?| ?)", activity.data, ^mutes)
+        where:
+          fragment(
+            "not (?->'to' \\?| ?) or ? = ?",
+            activity.data,
+            ^mutes,
+            activity.actor,
+            ^user.ap_id
+          )
       )
 
     unless opts[:skip_preload] do
@@ -930,16 +951,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_muted_reblogs(query, _), do: query
 
-  defp restrict_instance(query, %{instance: instance}) do
+  defp restrict_instance(query, %{instance: instance}) when is_binary(instance) do
-    users =
     from(
-        u in User,
+      activity in query,
-        select: u.ap_id,
+      where: fragment("split_part(actor::text, '/'::text, 3) = ?", ^instance)
-        where: fragment("? LIKE ?", u.nickname, ^"%@#{instance}")
     )
-      |> Repo.all()
-
-    from(activity in query, where: activity.actor in ^users)
   end
 
   defp restrict_instance(query, _), do: query
@@ -1232,7 +1248,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     capabilities = data["capabilities"] || %{}
     accepts_chat_messages = capabilities["acceptsChatMessages"]
     data = Transmogrifier.maybe_fix_user_object(data)
-    discoverable = data["discoverable"] || false
+    is_discoverable = data["discoverable"] || false
     invisible = data["invisible"] || false
     actor_type = data["type"] || "Person"
 
@@ -1258,7 +1274,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       fields: fields,
       emoji: emojis,
       is_locked: is_locked,
-      discoverable: discoverable,
+      is_discoverable: is_discoverable,
       invisible: invisible,
       avatar: avatar,
       name: data["name"],
@@ -1287,12 +1303,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def fetch_follow_information_for_user(user) do
     with {:ok, following_data} <-
-           Fetcher.fetch_and_contain_remote_object_from_id(user.following_address,
+           Fetcher.fetch_and_contain_remote_object_from_id(user.following_address),
-             force_http: true
-           ),
          {:ok, hide_follows} <- collection_private(following_data),
          {:ok, followers_data} <-
-           Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address, force_http: true),
+           Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address),
          {:ok, hide_followers} <- collection_private(followers_data) do
       {:ok,
        %{
@@ -1366,11 +1380,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def fetch_and_prepare_user_from_ap_id(ap_id, opts \\ []) do
+  def fetch_and_prepare_user_from_ap_id(ap_id) do
-    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id, opts),
+    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
          {:ok, data} <- user_data_from_user_object(data) do
       {:ok, maybe_update_follow_information(data)}
     else
+      # If this has been deleted, only log a debug and not an error
       {:error, "Object has been deleted" = e} ->
         Logger.debug("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
         {:error, e}
@@ -1409,13 +1424,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def make_user_from_ap_id(ap_id, opts \\ []) do
+  def make_user_from_ap_id(ap_id) do
     user = User.get_cached_by_ap_id(ap_id)
 
     if user && !User.ap_enabled?(user) do
       Transmogrifier.upgrade_user_from_ap_id(ap_id)
     else
-      with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, opts) do
+      with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do
         if user do
           user
           |> User.remote_user_changeset(data)

lib/pleroma/web/activity_pub/activity_pub/persisting.ex

@@ -0,0 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ActivityPub.Persisting do
+  @callback persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
+end

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -82,7 +82,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   def object(conn, _) do
     with ap_id <- Endpoint.url() <> conn.request_path,
          %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
-         {_, true} <- {:public?, Visibility.is_public?(object)} do
+         {_, true} <- {:public?, Visibility.is_public?(object)},
+         {_, false} <- {:local?, Visibility.is_local_public?(object)} do
       conn
       |> assign(:tracking_fun_data, object.id)
       |> set_cache_ttl_for(object)
@@ -92,6 +93,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     else
       {:public?, false} ->
         {:error, :not_found}
+
+      {:local?, true} ->
+        {:error, :not_found}
     end
   end
 
@@ -108,7 +112,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   def activity(conn, _params) do
     with ap_id <- Endpoint.url() <> conn.request_path,
          %Activity{} = activity <- Activity.normalize(ap_id),
-         {_, true} <- {:public?, Visibility.is_public?(activity)} do
+         {_, true} <- {:public?, Visibility.is_public?(activity)},
+         {_, false} <- {:local?, Visibility.is_local_public?(activity)} do
       conn
       |> maybe_set_tracking_data(activity)
       |> set_cache_ttl_for(activity)
@@ -117,6 +122,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
       |> render("object.json", object: activity)
     else
       {:public?, false} -> {:error, :not_found}
+      {:local?, true} -> {:error, :not_found}
       nil -> {:error, :not_found}
     end
   end
@@ -414,7 +420,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
       object =
         object
         |> Map.merge(Map.take(params, ["to", "cc"]))
-        |> Map.put("attributedTo", user.ap_id())
+        |> Map.put("attributedTo", user.ap_id)
         |> Transmogrifier.fix_object()
 
       ActivityPub.create(%{
@@ -458,7 +464,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
         %{assigns: %{user: %User{nickname: nickname} = user}} = conn,
         %{"nickname" => nickname} = params
       ) do
-    actor = user.ap_id()
+    actor = user.ap_id
 
     params =
       params
@@ -525,19 +531,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     {new_user, for_user}
   end
 
-  @doc """
-  Endpoint based on <https://www.w3.org/wiki/SocialCG/ActivityPub/MediaUpload>
-
-  Parameters:
-  - (required) `file`: data of the media
-  - (optionnal) `description`: description of the media, intended for accessibility
-
-  Response:
-  - HTTP Code: 201 Created
-  - HTTP Body: ActivityPub object to be inserted into another's `attachment` field
-
-  Note: Will not point to a URL with a `Location` header because no standalone Activity has been created.
-  """
   def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do
     with {:ok, object} <-
            ActivityPub.upload(

lib/pleroma/web/activity_pub/builder.ex

@@ -222,6 +222,9 @@ defmodule Pleroma.Web.ActivityPub.Builder do
         actor.ap_id == Relay.ap_id() ->
           [actor.follower_address]
 
+        public? and Visibility.is_local_public?(object) ->
+          [actor.follower_address, object.data["actor"], Pleroma.Constants.as_local_public()]
+
         public? ->
           [actor.follower_address, object.data["actor"], Pleroma.Constants.as_public()]
 

lib/pleroma/web/activity_pub/mrf.ex

@@ -3,7 +3,64 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF do
+  require Logger
+
+  @behaviour Pleroma.Web.ActivityPub.MRF.PipelineFiltering
+
+  @mrf_config_descriptions [
+    %{
+      group: :pleroma,
+      key: :mrf,
+      tab: :mrf,
+      label: "MRF",
+      type: :group,
+      description: "General MRF settings",
+      children: [
+        %{
+          key: :policies,
+          type: [:module, {:list, :module}],
+          description:
+            "A list of MRF policies enabled. Module names are shortened (removed leading `Pleroma.Web.ActivityPub.MRF.` part), but on adding custom module you need to use full name.",
+          suggestions: {:list_behaviour_implementations, Pleroma.Web.ActivityPub.MRF}
+        },
+        %{
+          key: :transparency,
+          label: "MRF transparency",
+          type: :boolean,
+          description:
+            "Make the content of your Message Rewrite Facility settings public (via nodeinfo)"
+        },
+        %{
+          key: :transparency_exclusions,
+          label: "MRF transparency exclusions",
+          type: {:list, :string},
+          description:
+            "Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.",
+          suggestions: [
+            "exclusion.com"
+          ]
+        }
+      ]
+    }
+  ]
+
+  @default_description %{
+    label: "",
+    description: ""
+  }
+
+  @required_description_keys [:key, :related_policy]
+
   @callback filter(Map.t()) :: {:ok | :reject, Map.t()}
+  @callback describe() :: {:ok | :error, Map.t()}
+  @callback config_description() :: %{
+              optional(:children) => [map()],
+              key: atom(),
+              related_policy: String.t(),
+              label: String.t(),
+              description: String.t()
+            }
+  @optional_callbacks config_description: 0
 
   def filter(policies, %{} = message) do
     policies
@@ -15,6 +72,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do
 
   def filter(%{} = object), do: get_policies() |> filter(object)
 
+  @impl true
   def pipeline_filter(%{} = message, meta) do
     object = meta[:object_data]
     ap_id = message["object"]
@@ -51,8 +109,6 @@ defmodule Pleroma.Web.ActivityPub.MRF do
     Enum.any?(domains, fn domain -> Regex.match?(domain, host) end)
   end
 
-  @callback describe() :: {:ok | :error, Map.t()}
-
   def describe(policies) do
     {:ok, policy_configs} =
       policies
@@ -82,4 +138,41 @@ defmodule Pleroma.Web.ActivityPub.MRF do
   end
 
   def describe, do: get_policies() |> describe()
+
+  def config_descriptions do
+    Pleroma.Web.ActivityPub.MRF
+    |> Pleroma.Docs.Generator.list_behaviour_implementations()
+    |> config_descriptions()
+  end
+
+  def config_descriptions(policies) do
+    Enum.reduce(policies, @mrf_config_descriptions, fn policy, acc ->
+      if function_exported?(policy, :config_description, 0) do
+        description =
+          @default_description
+          |> Map.merge(policy.config_description)
+          |> Map.put(:group, :pleroma)
+          |> Map.put(:tab, :mrf)
+          |> Map.put(:type, :group)
+
+        if Enum.all?(@required_description_keys, &Map.has_key?(description, &1)) do
+          [description | acc]
+        else
+          Logger.warn(
+            "#{policy} config description doesn't have one or all required keys #{
+              inspect(@required_description_keys)
+            }"
+          )
+
+          acc
+        end
+      else
+        Logger.debug(
+          "#{policy} is excluded from config descriptions, because does not implement `config_description/0` method."
+        )
+
+        acc
+      end
+    end)
+  end
 end

lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex

@@ -40,4 +40,22 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do
       _ -> Map.put(activity, "expires_at", expires_at)
     end
   end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_activity_expiration,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy",
+      label: "MRF Activity Expiration Policy",
+      description: "Adds automatic expiration to all local activities",
+      children: [
+        %{
+          key: :days,
+          type: :integer,
+          description: "Default global expiration time for all local activities (in days)",
+          suggestions: [90, 365]
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex

@@ -97,4 +97,31 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
   @impl true
   def describe,
     do: {:ok, %{mrf_hellthread: Pleroma.Config.get(:mrf_hellthread) |> Enum.into(%{})}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_hellthread,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.HellthreadPolicy",
+      label: "MRF Hellthread",
+      description: "Block messages with excessive user mentions",
+      children: [
+        %{
+          key: :delist_threshold,
+          type: :integer,
+          description:
+            "Number of mentioned users after which the message gets removed from timelines and" <>
+              "disables notifications. Set to 0 to disable.",
+          suggestions: [10]
+        },
+        %{
+          key: :reject_threshold,
+          type: :integer,
+          description:
+            "Number of mentioned users after which the messaged gets rejected. Set to 0 to disable.",
+          suggestions: [20]
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/keyword_policy.ex

@@ -126,4 +126,46 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
 
     {:ok, %{mrf_keyword: mrf_keyword}}
   end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_keyword,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy",
+      label: "MRF Keyword",
+      description:
+        "Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).",
+      children: [
+        %{
+          key: :reject,
+          type: {:list, :string},
+          description: """
+            A list of patterns which result in message being rejected.
+
+            Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+          """,
+          suggestions: ["foo", ~r/foo/iu]
+        },
+        %{
+          key: :federated_timeline_removal,
+          type: {:list, :string},
+          description: """
+            A list of patterns which result in message being removed from federated timelines (a.k.a unlisted).
+
+            Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+          """,
+          suggestions: ["foo", ~r/foo/iu]
+        },
+        %{
+          key: :replace,
+          type: {:list, :tuple},
+          description: """
+            **Pattern**: a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+
+            **Replacement**: a string. Leaving the field empty is permitted.
+          """
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex

@@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
   alias Pleroma.HTTP
   alias Pleroma.Web.MediaProxy
-  alias Pleroma.Workers.BackgroundWorker
 
   require Logger
 
@@ -17,7 +16,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
     recv_timeout: 10_000
   ]
 
-  def perform(:prefetch, url) do
+  defp prefetch(url) do
     # Fetching only proxiable resources
     if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do
       # If preview proxy is enabled, it'll also hit media proxy (so we're caching both requests)
@@ -25,17 +24,25 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
       Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
 
-      HTTP.get(prefetch_url, [], @adapter_options)
+      if Pleroma.Config.get(:env) == :test do
+        fetch(prefetch_url)
+      else
+        ConcurrentLimiter.limit(MediaProxy, fn ->
+          Task.start(fn -> fetch(prefetch_url) end)
+        end)
+      end
     end
   end
 
-  def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do
+  defp fetch(url), do: HTTP.get(url, [], @adapter_options)
+
+  defp preload(%{"object" => %{"attachment" => attachments}} = _message) do
     Enum.each(attachments, fn
       %{"url" => url} when is_list(url) ->
         url
         |> Enum.each(fn
           %{"href" => href} ->
-            BackgroundWorker.enqueue("media_proxy_prefetch", %{"url" => href})
+            prefetch(href)
 
           x ->
             Logger.debug("Unhandled attachment URL object #{inspect(x)}")
@@ -51,7 +58,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
         %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message
       )
       when is_list(attachments) and length(attachments) > 0 do
-    BackgroundWorker.enqueue("media_proxy_preload", %{"message" => message})
+    preload(message)
 
     {:ok, message}
   end

lib/pleroma/web/activity_pub/mrf/mention_policy.ex

@@ -25,4 +25,22 @@ defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do
 
   @impl true
   def describe, do: {:ok, %{}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_mention,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.MentionPolicy",
+      label: "MRF Mention",
+      description: "Block messages which mention a specific user",
+      children: [
+        %{
+          key: :actors,
+          type: {:list, :string},
+          description: "A list of actors for which any post mentioning them will be dropped",
+          suggestions: ["actor1", "actor2"]
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/normalize_markup.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do
 
   @behaviour Pleroma.Web.ActivityPub.MRF
 
+  @impl true
   def filter(%{"type" => "Create", "object" => child_object} = object) do
     scrub_policy = Pleroma.Config.get([:mrf_normalize_markup, :scrub_policy])
 
@@ -22,5 +23,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do
 
   def filter(object), do: {:ok, object}
 
+  @impl true
   def describe, do: {:ok, %{}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_normalize_markup,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.NormalizeMarkup",
+      label: "MRF Normalize Markup",
+      description: "MRF NormalizeMarkup settings. Scrub configured hypertext markup.",
+      children: [
+        %{
+          key: :scrub_policy,
+          type: :module,
+          suggestions: [Pleroma.HTML.Scrubber.Default]
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/object_age_policy.ex

@@ -106,4 +106,32 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
 
     {:ok, %{mrf_object_age: mrf_object_age}}
   end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_object_age,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy",
+      label: "MRF Object Age",
+      description:
+        "Rejects or delists posts based on their timestamp deviance from your server's clock.",
+      children: [
+        %{
+          key: :threshold,
+          type: :integer,
+          description: "Required age (in seconds) of a post before actions are taken.",
+          suggestions: [172_800]
+        },
+        %{
+          key: :actions,
+          type: {:list, :atom},
+          description:
+            "A list of actions to apply to the post. `:delist` removes the post from public timelines; " <>
+              "`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines; " <>
+              "`:reject` rejects the message entirely",
+          suggestions: [:delist, :strip_followers, :reject]
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/pipeline_filtering.ex

@@ -0,0 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.PipelineFiltering do
+  @callback pipeline_filter(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
+end