Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into feature/expire-mutes

.credo.exs

@@ -25,7 +25,7 @@
       #
       # If you create your own checks, you must specify the source files for
       # them here, so they can be loaded by Credo before running the analysis.
-      requires: [],
+      requires: ["./test/credo/check/consistency/file_location.ex"],
       #
       # Credo automatically checks for updates, like e.g. Hex does.
       # You can disable this behaviour below:
@@ -71,7 +71,6 @@
         # set this value to 0 (zero).
         {Credo.Check.Design.TagTODO, exit_status: 0},
         {Credo.Check.Design.TagFIXME, exit_status: 0},
-
         {Credo.Check.Readability.FunctionNames},
         {Credo.Check.Readability.LargeNumbers},
         {Credo.Check.Readability.MaxLineLength, priority: :low, max_length: 100},
@@ -91,7 +90,6 @@
         {Credo.Check.Readability.VariableNames},
         {Credo.Check.Readability.Semicolons},
         {Credo.Check.Readability.SpaceAfterCommas},
-
         {Credo.Check.Refactor.DoubleBooleanNegation},
         {Credo.Check.Refactor.CondStatements},
         {Credo.Check.Refactor.CyclomaticComplexity},
@@ -102,7 +100,6 @@
         {Credo.Check.Refactor.Nesting},
         {Credo.Check.Refactor.PipeChainStart},
         {Credo.Check.Refactor.UnlessWithElse},
-
         {Credo.Check.Warning.BoolOperationOnSameValues},
         {Credo.Check.Warning.IExPry},
         {Credo.Check.Warning.IoInspect},
@@ -131,6 +128,7 @@
 
         # Custom checks can be created using `mix credo.gen.check`.
         #
+        {Credo.Check.Consistency.FileLocation}
       ]
     }
   ]

.gitignore

@@ -27,6 +27,8 @@ erl_crash.dump
 # variables.
 /config/*.secret.exs
 /config/generated_config.exs
+/config/*.env
+
 
 # Database setup file, some may forget to delete it
 /config/setup_db.psql

.gitlab-ci.yml

@@ -25,6 +25,8 @@ before_script:
   - apt-get update && apt-get install -y cmake
   - mix local.hex --force
   - mix local.rebar --force
+  - apt-get -qq update
+  - apt-get install -y libmagic-dev
 
 build:
   stage: build
@@ -59,7 +61,7 @@ unit-testing:
     alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
-    - apt-get update && apt-get install -y libimage-exiftool-perl
+    - apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg
     - mix deps.get
     - mix ecto.create
     - mix ecto.migrate
@@ -93,7 +95,7 @@ unit-testing-rum:
     <<: *global_variables
     RUM_ENABLED: "true"
   script:
-    - apt-get update && apt-get install -y libimage-exiftool-perl
+    - apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg
     - mix deps.get
     - mix ecto.create
     - mix ecto.migrate
@@ -196,7 +198,7 @@ amd64:
   variables: &release-variables
     MIX_ENV: prod
   before_script: &before-release
-  - apt-get update && apt-get install -y cmake
+  - apt-get update && apt-get install -y cmake libmagic-dev
   - echo "import Mix.Config" > config/prod.secret.exs
   - mix local.hex --force
   - mix local.rebar --force
@@ -215,7 +217,7 @@ amd64-musl:
   cache: *release-cache
   variables: *release-variables
   before_script: &before-release-musl
-  - apk add git gcc g++ musl-dev make cmake
+  - apk add git gcc g++ musl-dev make cmake file-dev
   - echo "import Mix.Config" > config/prod.secret.exs
   - mix local.hex --force
   - mix local.rebar --force

CHANGELOG.md

@@ -5,20 +5,47 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+### Added
+- Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
+- Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
+- Mix task option for force-unfollowing relays
+- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
+- Pleroma API: Importing the mutes users from CSV files.
+- Experimental websocket-based federation between Pleroma instances.
+- Support pagination of blocks and mutes
+- App metrics: ability to restrict access to specified IP whitelist.
+- Account backup
+- Configuration: Add `:instance, autofollowing_nicknames` setting to provide a way to make accounts automatically follow new users that register on the local Pleroma instance.
+- Ability to view remote timelines, with ex. `/api/v1/timelines/public?instance=lain.com` and streams `public:remote` and `public:remote:media`
+
 ### Changed
 
+- **Breaking** Requires `libmagic` (or `file`) to guess file types.
+- **Breaking:** Pleroma Admin API: emoji packs and files routes changed.
+- **Breaking:** Sensitive/NSFW statuses no longer disable link previews.
+- **Breaking:** App metrics endpoint (`/api/pleroma/app_metrics`) is disabled by default, check `docs/API/prometheus.md` on enabling and configuring. 
+- Search: Users are now findable by their urls.
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
 - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.
 - The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false.
 - Users with the `discoverable` field set to false will not show up in searches.
 - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
+- Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`.
+- Polls now always return a `voters_count`, even if they are single-choice
+
+<details>
+  <summary>API Changes</summary>
 
-### Added
-- Media preview proxy (requires media proxy be enabled; see `:media_preview_proxy` config for more details).
 - Pleroma API: Importing the mutes users from CSV files.
-- Experimental websocket-based federation between Pleroma instances.
 - Admin API: Importing emoji from a zip file
-- User and conversation mutes can now auto-expire if `expires_in` parameter was given while adding the mute.
+- Pleroma API: Pagination for remote/local packs and emoji.
+- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status
+- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type`
+- Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
+- Pleroma API: (`GET /api/v1/pleroma/federation_status`) Add a way to get a list of unreachable instances.
+- Mastodon API: User and conversation mutes can now auto-expire if `expires_in` parameter was given while adding the mute.
+
+</details>
 
 ### Removed
 
@@ -29,7 +56,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 switched to a new configuration mechanism, however it was not officially removed until now.
 
 ### Fixed
+
+- Add documented-but-missing chat pagination.
 - Allow sending out emails again.
+- Allow sending chat messages to yourself.
+- Fix remote users with a whitespace name.
+- OStatus / static FE endpoints: fixed inaccessibility for anonymous users on non-federating instances, switched to handling per `:restrict_unauthenticated` setting.
+- Mastodon API: Current user is now included in conversation if it's the only participant
+- Mastodon API: Fixed last_status.account being not filled with account data
+
+## Unreleased (Patch)
+
+### Changed
+- API: Empty parameter values for integer parameters are now ignored in non-strict validaton mode.
 
 ## [2.1.2] - 2020-09-17
 

config/benchmark.exs

@@ -59,8 +59,6 @@ config :web_push_encryption, :vapid_details,
     "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4",
   private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA"
 
-config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock
-
 config :pleroma, Pleroma.ScheduledActivity,
   daily_user_limit: 2,
   total_user_limit: 3,

config/config.exs

@@ -123,7 +123,6 @@ websocket_config = [
 
 # Configures the endpoint
 config :pleroma, Pleroma.Web.Endpoint,
-  instrumenters: [Pleroma.Web.Endpoint.Instrumenter],
   url: [host: "localhost"],
   http: [
     ip: {127, 0, 0, 1},
@@ -143,7 +142,7 @@ config :pleroma, Pleroma.Web.Endpoint,
   secret_key_base: "aK4Abxf29xU9TTDKre9coZPUgevcVCFQJe/5xP/7Lt4BEif6idBIbjupVbOrbKxl",
   signing_salt: "CqaoopA2",
   render_errors: [view: Pleroma.Web.ErrorView, accepts: ~w(json)],
-  pubsub: [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2],
+  pubsub_server: Pleroma.PubSub,
   secure_cookie_flag: true,
   extra_cookie_attrs: [
     "SameSite=Lax"
@@ -235,6 +234,7 @@ config :pleroma, :instance,
     "text/bbcode"
   ],
   autofollowed_nicknames: [],
+  autofollowing_nicknames: [],
   max_pinned_statuses: 1,
   attachment_links: false,
   max_report_comment_size: 1000,
@@ -551,6 +551,7 @@ config :pleroma, Oban,
   queues: [
     activity_expiration: 10,
     token_expiration: 5,
+    backup: 1,
     federator_incoming: 50,
     federator_outgoing: 50,
     ingestion_queue: 50,
@@ -637,7 +638,12 @@ config :pleroma, Pleroma.Emails.UserEmail,
 
 config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: false
 
-config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics"
+config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
+  enabled: false,
+  auth: false,
+  ip_whitelist: [],
+  path: "/api/pleroma/app_metrics",
+  format: :text
 
 config :pleroma, Pleroma.ScheduledActivity,
   daily_user_limit: 25,
@@ -678,7 +684,18 @@ config :pleroma, :rate_limit,
 
 config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600
 
-config :pleroma, Pleroma.Plugs.RemoteIp, enabled: true
+config :pleroma, Pleroma.Web.Plugs.RemoteIp,
+  enabled: true,
+  headers: ["x-forwarded-for"],
+  proxies: [],
+  reserved: [
+    "127.0.0.0/8",
+    "::1/128",
+    "fc00::/7",
+    "10.0.0.0/8",
+    "172.16.0.0/12",
+    "192.168.0.0/16"
+  ]
 
 config :pleroma, :static_fe, enabled: false
 
@@ -792,6 +809,8 @@ config :pleroma, :hackney_pools,
     timeout: 300_000
   ]
 
+config :pleroma, :majic_pool, size: 2
+
 private_instance? = :if_instance_is_private
 
 config :pleroma, :restrict_unauthenticated,
@@ -810,7 +829,7 @@ config :tzdata, :http_client, Pleroma.HTTP.Tzdata
 
 config :ex_aws, http_client: Pleroma.HTTP.ExAws
 
-config :web_push_encryption, http_client: Pleroma.HTTP
+config :web_push_encryption, http_client: Pleroma.HTTP.WebPush
 
 config :pleroma, :instances_favicons, enabled: false
 
@@ -818,6 +837,11 @@ config :floki, :html_parser, Floki.HTMLParser.FastHtml
 
 config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
 
+config :pleroma, Pleroma.User.Backup,
+  purge_after_days: 30,
+  limit_days: 7,
+  dir: nil
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/description.exs

@@ -44,11 +44,13 @@ frontend_options = [
   },
   %{
     key: "git",
+    label: "Git Repository URL",
     type: :string,
     description: "URL of the git repository of the frontend"
   },
   %{
     key: "build_url",
+    label: "Build URL",
     type: :string,
     description:
       "Either an url to a zip file containing the frontend or a template to build it by inserting the `ref`. The string `${ref}` will be replaced by the configured `ref`.",
@@ -56,6 +58,7 @@ frontend_options = [
   },
   %{
     key: "build_dir",
+    label: "Build directory",
     type: :string,
     description: "The directory inside the zip file "
   }
@@ -826,13 +829,13 @@ config :pleroma, :config_description, [
         key: :autofollowed_nicknames,
         type: {:list, :string},
         description:
-          "Set to nicknames of (local) users that every new user should automatically follow",
+          "Set to nicknames of (local) users that every new user should automatically follow"
-        suggestions: [
+      },
-          "lain",
+      %{
-          "kaniini",
+        key: :autofollowing_nicknames,
-          "lanodan",
+        type: {:list, :string},
-          "rinpatch"
+        description:
-        ]
+          "Set to nicknames of (local) users that automatically follows every newly registered user"
       },
       %{
         key: :attachment_links,
@@ -1754,28 +1757,37 @@ config :pleroma, :config_description, [
     related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy",
     label: "MRF Keyword",
     type: :group,
-    description: "Reject or Word-Replace messages with a keyword or regex",
+    description:
+      "Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).",
     children: [
       %{
         key: :reject,
         type: {:list, :string},
-        description:
+        description: """
-          "A list of patterns which result in message being rejected. Each pattern can be a string or a regular expression.",
+          A list of patterns which result in message being rejected.
+
+          Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+        """,
         suggestions: ["foo", ~r/foo/iu]
       },
       %{
         key: :federated_timeline_removal,
         type: {:list, :string},
-        description:
+        description: """
-          "A list of patterns which result in message being removed from federated timelines (a.k.a unlisted). Each pattern can be a string or a regular expression.",
+          A list of patterns which result in message being removed from federated timelines (a.k.a unlisted).
+
+          Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+        """,
         suggestions: ["foo", ~r/foo/iu]
       },
       %{
         key: :replace,
         type: {:list, :tuple},
-        description:
+        description: """
-          "A list of tuples containing {pattern, replacement}. Each pattern can be a string or a regular expression.",
+          **Pattern**: a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
-        suggestions: [{"foo", "bar"}, {~r/foo/iu, "bar"}]
+
+          **Replacement**: a string. Leaving the field empty is permitted.
+        """
       }
     ]
   },
@@ -2285,6 +2297,12 @@ config :pleroma, :config_description, [
             description: "Activity expiration queue",
             suggestions: [10]
           },
+          %{
+            key: :backup,
+            type: :integer,
+            description: "Backup queue",
+            suggestions: [1]
+          },
           %{
             key: :attachments_cleanup,
             type: :integer,
@@ -3247,10 +3265,10 @@ config :pleroma, :config_description, [
   },
   %{
     group: :pleroma,
-    key: Pleroma.Plugs.RemoteIp,
+    key: Pleroma.Web.Plugs.RemoteIp,
     type: :group,
     description: """
-    `Pleroma.Plugs.RemoteIp` is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
+    `Pleroma.Web.Plugs.RemoteIp` is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
     **If your instance is not behind at least one reverse proxy, you should not enable this plug.**
     """,
     children: [
@@ -3262,20 +3280,22 @@ config :pleroma, :config_description, [
       %{
         key: :headers,
         type: {:list, :string},
-        description:
+        description: """
-          "A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Default: `~w[forwarded x-forwarded-for x-client-ip x-real-ip]`."
+          A list of strings naming the HTTP headers to use when deriving the true client IP. Default: `["x-forwarded-for"]`.
+        """
       },
       %{
         key: :proxies,
         type: {:list, :string},
         description:
-          "A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Default: `[]`."
+          "A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128."
       },
       %{
         key: :reserved,
         type: {:list, :string},
-        description:
+        description: """
-          "Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network)."
+          A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`
+        """
       }
     ]
   },
@@ -3681,9 +3701,7 @@ config :pleroma, :config_description, [
         type: :map,
         description:
           "A map containing available frontends and parameters for their installation.",
-        children: [
+        children: frontend_options
-          frontend_options
-        ]
       }
     ]
   },
@@ -3705,5 +3723,76 @@ config :pleroma, :config_description, [
         ]
       }
     ]
+  },
+  %{
+    group: :pleroma,
+    key: :majic_pool,
+    type: :group,
+    description: "Majic/libmagic configuration",
+    children: [
+      %{
+        key: :size,
+        type: :integer,
+        description: "Number of majic workers to start.",
+        suggestions: [2]
+      }
+    ]
+  },
+  %{
+    group: :pleroma,
+    key: Pleroma.User.Backup,
+    type: :group,
+    description: "Account Backup",
+    children: [
+      %{
+        key: :purge_after_days,
+        type: :integer,
+        description: "Remove backup achives after N days",
+        suggestions: [30]
+      },
+      %{
+        key: :limit_days,
+        type: :integer,
+        description: "Limit user to export not more often than once per N days",
+        suggestions: [7]
+      }
+    ]
+  },
+  %{
+    group: :prometheus,
+    key: Pleroma.Web.Endpoint.MetricsExporter,
+    type: :group,
+    description: "Prometheus app metrics endpoint configuration",
+    children: [
+      %{
+        key: :enabled,
+        type: :boolean,
+        description: "[Pleroma extension] Enables app metrics endpoint."
+      },
+      %{
+        key: :ip_whitelist,
+        type: [{:list, :string}, {:list, :charlist}, {:list, :tuple}],
+        description:
+          "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses."
+      },
+      %{
+        key: :auth,
+        type: [:boolean, :tuple],
+        description: "Enables HTTP Basic Auth for app metrics endpoint.",
+        suggestion: [false, {:basic, "myusername", "mypassword"}]
+      },
+      %{
+        key: :path,
+        type: :string,
+        description: "App metrics endpoint URI path.",
+        suggestions: ["/api/pleroma/app_metrics"]
+      },
+      %{
+        key: :format,
+        type: :atom,
+        description: "App metrics endpoint output format.",
+        suggestions: [:text, :protobuf]
+      }
+    ]
   }
 ]

config/test.exs

@@ -83,8 +83,6 @@ config :web_push_encryption, :vapid_details,
     "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4",
   private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA"
 
-config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock
-
 config :pleroma, Oban,
   queues: false,
   crontab: false,
@@ -115,7 +113,7 @@ config :pleroma, Pleroma.Gun, Pleroma.GunMock
 
 config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: true
 
-config :pleroma, Pleroma.Plugs.RemoteIp, enabled: false
+config :pleroma, Pleroma.Web.Plugs.RemoteIp, enabled: false
 
 config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true
 

coveralls.json

@@ -1,6 +1,7 @@
 {
   "skip_files": [
     "test/support",
-    "lib/mix/tasks/pleroma/benchmark.ex"
+    "lib/mix/tasks/pleroma/benchmark.ex",
+    "lib/credo/check/consistency/file_location.ex"
   ]
 }

docs/API/admin_api.md

@@ -20,12 +20,14 @@ Configuration options:
     - `external`: only external users
     - `active`: only active users
     - `need_approval`: only unapproved users
+    - `unconfirmed`: only unconfirmed users
     - `deactivated`: only deactivated users
     - `is_admin`: users with admin role
     - `is_moderator`: users with moderator role
   - *optional* `page`: **integer** page number
   - *optional* `page_size`: **integer** number of users per page (default is `50`)
   - *optional* `tags`: **[string]** tags list
+  - *optional* `actor_types`: **[string]** actor type list (`Person`, `Service`, `Application`)
   - *optional* `name`: **string** user display name
   - *optional* `email`: **string** user email
 - Example: `https://mypleroma.org/api/pleroma/admin/users?query=john&filters=local,active&page=1&page_size=10&tags[]=some_tag&tags[]=another_tag&name=display_name&email=email@example.com`
@@ -349,9 +351,9 @@ Response:
 
 ### Unfollow a Relay
 
-Params:
+- Params:
-
+  - `relay_url`
-* `relay_url`
+  - *optional* `force`: forcefully unfollow a relay even when the relay is not available. (default is `false`)
 
 Response:
 

docs/API/chats.md

@@ -116,6 +116,10 @@ The modified chat message
 This will return a list of chats that you have been involved in, sorted by their
 last update (so new chats will be at the top).
 
+Parameters:
+
+- with_muted: Include chats from muted users (boolean).
+
 Returned data:
 
 ```json
@@ -173,11 +177,14 @@ Returned data:
     "created_at": "2020-04-21T15:06:45.000Z",
     "emojis": [],
     "id": "12",
-    "unread": false
+    "unread": false,
+    "idempotency_key": "75442486-0874-440c-9db1-a7006c25a31f"
   }
 ]
 ```
 
+- idempotency_key: The copy of the `idempotency-key` HTTP request header that can be used for optimistic message sending. Included only during the first few minutes after the message creation.
+
 ### Posting a chat message
 
 Posting a chat message for given Chat id works like this:

docs/API/differences_in_mastoapi_responses.md

@@ -9,9 +9,13 @@ Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mas
 ## Timelines
 
 Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users.
+
 Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`.
+
 Adding the parameter `reply_visibility` to the public and home timelines queries will filter replies. Possible values: without parameter (default) shows all replies, `following` - replies directed to you or users you follow, `self` - replies directed to you.
 
+Adding the parameter `instance=lain.com` to the public timeline will show only statuses originating from `lain.com` (or any remote instance).
+
 ## Statuses
 
 - `visibility`: has an additional possible value `list`
@@ -249,6 +253,8 @@ Has these additional fields under the `pleroma` object:
 
 There is an additional `user:pleroma_chat` stream. Incoming chat messages will make the current chat be sent to this `user` stream. The `event` of an incoming chat message is `pleroma:chat_update`. The payload is the updated chat with the incoming chat message in the `last_message` field.
 
+For viewing remote server timelines, there are `public:remote` and `public:remote:media` streams. Each of these accept a parameter like `?instance=lain.com`.
+
 ## Not implemented
 
 Pleroma is generally compatible with the Mastodon 2.7.2 API, but some newer features and non-essential features are omitted. These features usually return an HTTP 200 status code, but with an empty response. While they may be added in the future, they are considered low priority.

docs/API/pleroma_api.md

@@ -378,44 +378,43 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Params: None
 * Response: JSON, returns a list of Mastodon Conversation entities that were marked as read (200 - healthy, 503 unhealthy).
 
-## `GET /api/pleroma/emoji/packs/import`
+## `GET /api/pleroma/emoji/pack?name=:name`
-### Imports packs from filesystem
+
+### Get pack.json for the pack
+
 * Method `GET`
-* Authentication: required
+* Authentication: not required
-* Params: None
-* Response: JSON, returns a list of imported packs.
-
-## `GET /api/pleroma/emoji/packs/remote`
-### Make request to another instance for packs list
-* Method `GET`
-* Authentication: required
 * Params:
-  * `url`: url of the instance to get packs from
+  * `page`: page number for files (default 1)
-* Response: JSON with the pack list, hashmap with pack name and pack contents
+  * `page_size`: page size for files (default 30)
+* Response: JSON, pack json with `files`, `files_count` and `pack` keys with 200 status or 404 if the pack does not exist.
 
-## `POST /api/pleroma/emoji/packs/download`
+```json
-### Download pack from another instance
+{
-* Method `POST`
+  "files": {...},
-* Authentication: required
+  "files_count": 0, // emoji count in pack
-* Params:
+  "pack": {...}
-  * `url`: url of the instance to download from
+}
-  * `name`: pack to download from that instance
+```
-  * `as`: (*optional*) name how to save pack
+
-* Response: JSON, "ok" with 200 status if the pack was downloaded, or 500 if there were
+## `POST /api/pleroma/emoji/pack?name=:name`
-  errors downloading the pack
 
-## `POST /api/pleroma/emoji/packs/:name`
 ### Creates an empty pack
+
 * Method `POST`
-* Authentication: required
+* Authentication: required (admin)
-* Params: None
+* Params:
+  * `name`: pack name
 * Response: JSON, "ok" and 200 status or 409 if the pack with that name already exists
 
-## `PATCH /api/pleroma/emoji/packs/:name`
+## `PATCH /api/pleroma/emoji/pack?name=:name`
+
 ### Updates (replaces) pack metadata
+
 * Method `PATCH`
-* Authentication: required
+* Authentication: required (admin)
 * Params:
+  * `name`: pack name
   * `metadata`: metadata to replace the old one
     * `license`: Pack license
     * `homepage`: Pack home page url
@@ -426,39 +425,85 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Response: JSON, updated "metadata" section of the pack and 200 status or 400 if there was a
   problem with the new metadata (the error is specified in the "error" part of the response JSON)
 
-## `DELETE /api/pleroma/emoji/packs/:name`
+## `DELETE /api/pleroma/emoji/pack?name=:name`
+
 ### Delete a custom emoji pack
+
 * Method `DELETE`
-* Authentication: required
+* Authentication: required (admin)
-* Params: None
+* Params:
+  * `name`: pack name
 * Response: JSON, "ok" and 200 status or 500 if there was an error deleting the pack
 
-## `POST /api/pleroma/emoji/packs/:name/files`
+## `GET /api/pleroma/emoji/packs/import`
-### Add new file to the pack
+
-* Method `POST`
+### Imports packs from filesystem
-* Authentication: required
+
+* Method `GET`
+* Authentication: required (admin)
+* Params: None
+* Response: JSON, returns a list of imported packs.
+
+## `GET /api/pleroma/emoji/packs/remote`
+
+### Make request to another instance for packs list
+
+* Method `GET`
+* Authentication: required (admin)
 * Params:
+  * `url`: url of the instance to get packs from
+  * `page`: page number for packs (default 1)
+  * `page_size`: page size for packs (default 50)
+* Response: JSON with the pack list, hashmap with pack name and pack contents
+
+## `POST /api/pleroma/emoji/packs/download`
+
+### Download pack from another instance
+
+* Method `POST`
+* Authentication: required (admin)
+* Params:
+  * `url`: url of the instance to download from
+  * `name`: pack to download from that instance
+  * `as`: (*optional*) name how to save pack
+* Response: JSON, "ok" with 200 status if the pack was downloaded, or 500 if there were
+  errors downloading the pack
+
+## `POST /api/pleroma/emoji/packs/files?name=:name`
+
+### Add new file to the pack
+
+* Method `POST`
+* Authentication: required (admin)
+* Params:
+  * `name`: pack name
   * `file`: file needs to be uploaded with the multipart request or link to remote file.
   * `shortcode`: (*optional*) shortcode for new emoji, must be unique for all emoji. If not sended, shortcode will be taken from original filename.
   * `filename`: (*optional*) new emoji file name. If not specified will be taken from original filename.
 * Response: JSON, list of files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
-## `PATCH /api/pleroma/emoji/packs/:name/files`
+## `PATCH /api/pleroma/emoji/packs/files?name=:name`
+
 ### Update emoji file from pack
+
 * Method `PATCH`
-* Authentication: required
+* Authentication: required (admin)
 * Params:
+  * `name`: pack name
   * `shortcode`: emoji file shortcode
   * `new_shortcode`: new emoji file shortcode
   * `new_filename`: new filename for emoji file
   * `force`: (*optional*) with true value to overwrite existing emoji with new shortcode
 * Response: JSON, list with updated files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
-## `DELETE /api/pleroma/emoji/packs/:name/files`
+## `DELETE /api/pleroma/emoji/packs/files?name=:name`
+
 ### Delete emoji file from pack
+
 * Method `DELETE`
-* Authentication: required
+* Authentication: required (admin)
 * Params:
+  * `name`: pack name
   * `shortcode`: emoji file shortcode
 * Response: JSON, list with updated files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
@@ -483,30 +528,14 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 }
 ```
 
-## `GET /api/pleroma/emoji/packs/:name`
+## `GET /api/pleroma/emoji/packs/archive?name=:name`
 
-### Get pack.json for the pack
+### Requests a local pack archive from the instance
 
 * Method `GET`
 * Authentication: not required
 * Params:
-  * `page`: page number for files (default 1)
+  * `name`: pack name
-  * `page_size`: page size for files (default 30)
-* Response: JSON, pack json with `files`, `files_count` and `pack` keys with 200 status or 404 if the pack does not exist.
-
-```json
-{
-  "files": {...},
-  "files_count": 0, // emoji count in pack
-  "pack": {...}
-}
-```
-
-## `GET /api/pleroma/emoji/packs/:name/archive`
-### Requests a local pack archive from the instance
-* Method `GET`
-* Authentication: not required
-* Params: None
 * Response: the archive of the pack with a 200 status code, 403 if the pack is not set as shared,
   404 if the pack does not exist
 
@@ -586,3 +615,41 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
   {"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]}
 ]
 ```
+
+## `POST /api/v1/pleroma/backups`
+### Create a user backup archive
+
+* Method: `POST`
+* Authentication: required
+* Params: none
+* Response: JSON
+* Example response:
+
+```json
+[{
+    "content_type": "application/zip",
+    "file_size": 0,
+    "inserted_at": "2020-09-10T16:18:03.000Z",
+    "processed": false,
+    "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
+}]
+```
+
+## `GET /api/v1/pleroma/backups`
+### Lists user backups
+
+* Method: `GET`
+* Authentication: not required
+* Params: none
+* Response: JSON
+* Example response:
+
+```json
+[{
+    "content_type": "application/zip",
+    "file_size": 55457,
+    "inserted_at": "2020-09-10T16:18:03.000Z",
+    "processed": true,
+    "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
+}]
+```

docs/API/prometheus.md

@@ -2,15 +2,37 @@
 
 Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library.
 
+Config example:
+
+```
+config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
+  enabled: true,
+  auth: {:basic, "myusername", "mypassword"},
+  ip_whitelist: ["127.0.0.1"],
+  path: "/api/pleroma/app_metrics",
+  format: :text
+```
+
+* `enabled` (Pleroma extension) enables the endpoint
+* `ip_whitelist` (Pleroma extension) could be used to restrict access only to specified IPs
+* `auth` sets the authentication (`false` for no auth; configurable to HTTP Basic Auth, see [prometheus-plugs](https://github.com/deadtrickster/prometheus-plugs#exporting) documentation)
+* `format` sets the output format (`:text` or `:protobuf`)
+* `path` sets the path to app metrics page 
+
+
 ## `/api/pleroma/app_metrics`
+
 ### Exports Prometheus application metrics
+
 * Method: `GET`
-* Authentication: not required
+* Authentication: not required by default (see configuration options above)
 * Params: none
-* Response: JSON
+* Response: text
 
 ## Grafana
+
 ### Config example
+
 The following is a config example to use with [Grafana](https://grafana.com)
 
 ```

docs/administration/CLI_tasks/email.md

@@ -1,4 +1,4 @@
-# Managing emails
+# EMail administration tasks
 
 {! backend/administration/CLI_tasks/general_cli_task_info.include !}
 
@@ -30,3 +30,17 @@ Example:
     ```sh
     mix pleroma.email test --to root@example.org
     ```
+
+## Send confirmation emails to all unconfirmed user accounts
+
+=== "OTP"
+
+    ```sh
+     ./bin/pleroma_ctl email send_confirmation_mails
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.email send_confirmation_mails
+    ```

docs/administration/CLI_tasks/frontend.md

@@ -1,12 +1,23 @@
 # Managing frontends
 
-`mix pleroma.frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]`
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]
+    ```
 
 Frontend can be installed either from local zip file, or automatically downloaded from the web.
 
-You can give all the options directly on the command like, but missing information will be filled out by looking at the data configured under `frontends.available` in the config files.
+You can give all the options directly on the command line, but missing information will be filled out by looking at the data configured under `frontends.available` in the config files.
+
+Currently, known `<frontend>` values are:
 
-Currently known `<frontend>` values are:
 - [admin-fe](https://git.pleroma.social/pleroma/admin-fe)
 - [kenoma](http://git.pleroma.social/lambadalambda/kenoma)
 - [pleroma-fe](http://git.pleroma.social/pleroma/pleroma-fe)
@@ -19,51 +30,67 @@ You can still install frontends that are not configured, see below.
 
 For a frontend configured under the `available` key, it's enough to install it by name.
 
-```sh tab="OTP"
+=== "OTP"
+
+    ```sh
     ./bin/pleroma_ctl frontend install pleroma
     ```
 
-```sh tab="From Source"
+=== "From Source"
+
+    ```sh
     mix pleroma.frontend install pleroma
     ```
 
-This will download the latest build for the the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`).
+This will download the latest build for the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`).
 
-You can override any of the details. To install a pleroma build from a different url, you could do this:
+You can override any of the details. To install a pleroma build from a different URL, you could do this:
 
-```sh tab="OPT"
+=== "OTP"
+
+    ```sh
     ./bin/pleroma_ctl frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
     ```
 
-```sh tab="From Source"
+=== "From Source"
+
+    ```sh
     mix pleroma.frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
     ```
 
 Similarly, you can also install from a local zip file.
 
-```sh tab="OTP"
+=== "OTP"
+
+    ```sh
     ./bin/pleroma_ctl frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
     ```
 
-```sh tab="From Source"
+=== "From Source"
+
+    ```sh
     mix pleroma.frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
     ```
 
-The resulting frontend will always be installed into a folder of this template: `${instance_static}/frontends/${name}/${ref}`
+The resulting frontend will always be installed into a folder of this template: `${instance_static}/frontends/${name}/${ref}`.
 
-Careful: This folder will be completely replaced on installation
+Careful: This folder will be completely replaced on installation.
 
 ## Example installation for an unknown frontend
 
-The installation process is the same, but you will have to give all the needed options on the commond line. For example:
+The installation process is the same, but you will have to give all the needed options on the command line. For example:
 
-```sh tab="OTP"
+=== "OTP"
+
+    ```sh
     ./bin/pleroma_ctl frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
     ```
 
-```sh tab="From Source"
+=== "From Source"
+
+    ```sh
     mix pleroma.frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
     ```
 
-If you don't have a zip file but just want to install a frontend from a local path, you can simply copy the files over a folder of this template: `${instance_static}/frontends/${name}/${ref}`
+If you don't have a zip file but just want to install a frontend from a local path, you can simply copy the files over a folder of this template: `${instance_static}/frontends/${name}/${ref}`.
 

docs/administration/CLI_tasks/instance.md

@@ -37,3 +37,8 @@ If any of the options are left unspecified, you will be prompted interactively.
 - `--static-dir <path>` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
 - `--listen-ip <ip>` - the ip the app should listen to, defaults to 127.0.0.1
 - `--listen-port <port>` - the port the app should listen to, defaults to 4000
+- `--strip-uploads <Y|N>` - use ExifTool to strip uploads of sensitive location data
+- `--anonymize-uploads <Y|N>` - randomize uploaded filenames
+- `--dedupe-uploads <Y|N>` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames
+- `--skip-release-env` - skip generation the release environment file
+- `--release-env-file` - release environment file path

docs/administration/CLI_tasks/release_environments.md

@@ -0,0 +1,9 @@
+# Generate release environment file
+
+```sh tab="OTP"
+ ./bin/pleroma_ctl release_env gen
+```
+
+```sh tab="From Source"
+mix pleroma.release_env gen
+```

docs/administration/CLI_tasks/user.md

@@ -224,9 +224,10 @@
     ```
 
 ### Options
+- `--admin`/`--no-admin` - whether the user should be an admin
+- `--confirmed`/`--no-confirmed` - whether the user account is confirmed
 - `--locked`/`--no-locked` - whether the user should be locked
 - `--moderator`/`--no-moderator` - whether the user should be a moderator
-- `--admin`/`--no-admin` - whether the user should be an admin
 
 ## Add tags to a user
 
@@ -271,3 +272,33 @@
     ```sh
     mix pleroma.user toggle_confirmed <nickname>
     ```
+
+## Set confirmation status for all regular active users
+*Admins and moderators are excluded*
+
+=== "OTP"
+
+    ```sh
+     ./bin/pleroma_ctl user confirm_all
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user confirm_all
+    ```
+
+## Revoke confirmation status for all regular active users
+*Admins and moderators are excluded*
+
+=== "OTP"
+
+    ```sh
+     ./bin/pleroma_ctl user unconfirm_all
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user unconfirm_all
+    ```

docs/administration/backup.md

@@ -5,20 +5,25 @@
 1. Stop the Pleroma service.
 2. Go to the working directory of Pleroma (default is `/opt/pleroma`)
 3. Run `sudo -Hu postgres pg_dump -d <pleroma_db> --format=custom -f </path/to/backup_location/pleroma.pgdump>` (make sure the postgres user has write access to the destination file)
-4. Copy `pleroma.pgdump`, `config/prod.secret.exs` and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too.
+4. Copy `pleroma.pgdump`, `config/prod.secret.exs`, `config/setup_db.psql` (if still available) and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too.
 5. Restart the Pleroma service.
 
 ## Restore/Move
 
-1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers). Try to use the same database name.
+1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers).
 2. Stop the Pleroma service.
 3. Go to the working directory of Pleroma (default is `/opt/pleroma`)
 4. Copy the above mentioned files back to their original position.
-5. Drop the existing database and recreate an empty one `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;';` `sudo -Hu postgres psql -c 'CREATE DATABASE <pleroma_db>;';`
+5. Drop the existing database and user if restoring in-place. `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;';` `sudo -Hu postgres psql -c 'DROP USER <pleroma_db>;'`
-6. Run `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>`
+6. Restore the database schema and pleroma postgres role the with the original `setup_db.psql` if you have it: `sudo -Hu postgres psql -f config/setup_db.psql`. 
-7. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
+
-8. Restart the Pleroma service.
+  Alternatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the database user, name, and password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed.
-9. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries.
+
+7. Now restore the Pleroma instance's data into the empty database schema: `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>`
+8. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
+9. Restart the Pleroma service.
+10. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries.
+11. If setting up on a new server configure Nginx by using the `installation/pleroma.nginx` config sample or reference the Pleroma installation guide for your OS which contains the Nginx configuration instructions.
 
 [^1]: Prefix with `MIX_ENV=prod` to run it using the production config file.
 
@@ -31,6 +36,6 @@
 3. Disable pleroma from systemd `systemctl disable pleroma`
 4. Remove the files and folders you created during installation (see installation guide). This includes the pleroma, nginx and systemd files and folders.
 5. Reload nginx now that the configuration is removed `systemctl reload nginx`
-6. Remove the database and database user `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;';` `sudo -Hu postgres psql -c 'DROP USER <pleroma_db>;';`
+6. Remove the database and database user `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;';` `sudo -Hu postgres psql -c 'DROP USER <pleroma_db>;'`
 7. Remove the system user `userdel pleroma`
 8. Remove the dependencies that you don't need anymore (see installation guide). Make sure you don't remove packages that are still needed for other software that you have running!

docs/ap_extensions.md

@@ -1,11 +1,41 @@
-# ChatMessages
+# AP Extensions
+## Actor endpoints
 
-ChatMessages are the messages sent in 1-on-1 chats. They are similar to
+The following endpoints are additionally present into our actors.
+
+- `oauthRegistrationEndpoint` (`http://litepub.social/ns#oauthRegistrationEndpoint`)
+- `uploadMedia` (`https://www.w3.org/ns/activitystreams#uploadMedia`)
+
+### oauthRegistrationEndpoint
+
+Points to MastodonAPI `/api/v1/apps` for now.
+
+See <https://docs.joinmastodon.org/methods/apps/>
+
+### uploadMedia
+
+Inspired by <https://www.w3.org/wiki/SocialCG/ActivityPub/MediaUpload>, it is part of the ActivityStreams namespace because it used to be part of the ActivityPub specification and got removed from it.
+
+Content-Type: multipart/form-data
+
+Parameters:
+- (required) `file`: The file being uploaded
+- (optionnal) `description`: A plain-text description of the media, for accessibility purposes.
+
+Response: HTTP 201 Created with the object into the body, no `Location` header provided as it doesn't have an `id`
+
+The object given in the reponse should then be inserted into an Object's `attachment` field.
+
+## ChatMessages
+
+`ChatMessage`s are the messages sent in 1-on-1 chats. They are similar to
 `Note`s, but the addresing is done by having a single AP actor in the `to`
 field. Addressing multiple actors is not allowed. These messages are always
 private, there is no public version of them. They are created with a `Create`
 activity.
 
+They are part of the `litepub` namespace as `http://litepub.social/ns#ChatMessage`.
+
 Example:
 
 ```json

docs/clients.md

@@ -7,97 +7,105 @@ Feel free to contact us to be added to this list!
 - Homepage: <https://www.pleroma.com/#desktopApp>
 - Source Code: <https://github.com/roma-apps/roma-desktop>
 - Platforms: Windows, Mac, Linux
-- Features: Streaming Ready
+- Features: MastoAPI, Streaming Ready
 
 ### Social
 - Source Code: <https://gitlab.gnome.org/World/Social>
 - Contact: [@brainblasted@social.libre.fi](https://social.libre.fi/users/brainblasted)
 - Platforms: Linux (GNOME)
 - Note(2019-01-28): Not at a pre-alpha stage yet
+- Features: MastoAPI
 
 ### Whalebird
 - Homepage: <https://whalebird.org/>
 - Source Code: <https://github.com/h3poteto/whalebird-desktop>
 - Contact: [@h3poteto@pleroma.io](https://pleroma.io/users/h3poteto)
 - Platforms: Windows, Mac, Linux
-- Features: Streaming Ready
+- Features: MastoAPI, Streaming Ready
 
 ## Handheld
+### AndStatus
+- Homepage: <http://andstatus.org/>
+- Source Code: <https://github.com/andstatus/andstatus/>
+- Platforms: Android
+- Features: MastoAPI, ActivityPub (Client-to-Server)
+
 ### Amaroq
 - Homepage: <https://itunes.apple.com/us/app/amaroq-for-mastodon/id1214116200>
 - Source Code: <https://github.com/ReticentJohn/Amaroq>
 - Contact: [@eurasierboy@mastodon.social](https://mastodon.social/users/eurasierboy)
 - Platforms: iOS
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Fedilab
 - Homepage: <https://fedilab.app/>
 - Source Code: <https://framagit.org/tom79/fedilab/>
 - Contact: [@fedilab@framapiaf.org](https://framapiaf.org/users/fedilab)
 - Platforms: Android
-- Features: Streaming Ready, Moderation, Text Formatting
+- Features: MastoAPI, Streaming Ready, Moderation, Text Formatting
 
 ### Kyclos
 - Source Code: <https://git.pleroma.social/pleroma/harbour-kyclos>
 - Platforms: SailfishOS
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Husky
 - Source code: <https://git.mentality.rip/FWGS/Husky>
 - Contact: [@Husky@enigmatic.observer](https://enigmatic.observer/users/Husky)
 - Platforms: Android
-- Features: No Streaming, Emoji Reactions, Text Formatting, FE Stickers
+- Features: MastoAPI, No Streaming, Emoji Reactions, Text Formatting, FE Stickers
 
 ### Fedi
 - Homepage: <https://www.fediapp.com/>
 - Source Code: Proprietary, but gratis
 - Platforms: iOS, Android
-- Features: Pleroma-specific features like Reactions
+- Features: MastoAPI, Pleroma-specific features like Reactions
 
 ### Tusky
 - Homepage: <https://tuskyapp.github.io/>
 - Source Code: <https://github.com/tuskyapp/Tusky>
 - Contact: [@ConnyDuck@mastodon.social](https://mastodon.social/users/ConnyDuck)
 - Platforms: Android
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Twidere
 - Homepage: <https://twidere.mariotaku.org/>
 - Source Code: <https://github.com/TwidereProject/Twidere-Android/>
 - Contact: <me@mariotaku.org>
 - Platform: Android
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Indigenous
 - Homepage: <https://indigenous.realize.be/>
 - Source Code: <https://github.com/swentel/indigenous-android/>
-- Contact: [@realize.be@realize.be](@realize.be@realize.be)
+- Contact: [@swentel@realize.be](https://realize.be)
 - Platforms: Android
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ## Alternative Web Interfaces
 ### Brutaldon
 - Homepage: <https://jfm.carcosa.net/projects/software/brutaldon/>
 - Source Code: <https://git.carcosa.net/jmcbray/brutaldon>
 - Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc)
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Halcyon
 - Source Code: <https://notabug.org/halcyon-suite/halcyon>
 - Contact: [@halcyon@social.csswg.org](https://social.csswg.org/users/halcyon)
-- Features: Streaming Ready
+- Features: MastoAPI, Streaming Ready
 
 ### Pinafore
 - Homepage: <https://pinafore.social/>
 - Source Code: <https://github.com/nolanlawson/pinafore>
 - Contact: [@pinafore@mastodon.technology](https://mastodon.technology/users/pinafore)
 - Note: Pleroma support is a secondary goal
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Sengi
 - Homepage: <https://nicolasconstant.github.io/sengi/>
 - Source Code: <https://github.com/NicolasConstant/sengi>
 - Contact: [@sengi_app@mastodon.social](https://mastodon.social/users/sengi_app)
+- Features: MastoAPI
 
 ### DashFE
 - Source Code: <https://notabug.org/daisuke/DashboardFE>
@@ -107,3 +115,4 @@ Feel free to contact us to be added to this list!
 - Source Code: <https://git.freesoftwareextremist.com/bloat/>
 - Contact: [@r@freesoftwareextremist.com](https://freesoftwareextremist.com/users/r)
 - Features: Does not requires JavaScript
+- Features: MastoAPI

docs/configuration/cheatsheet.md

@@ -45,6 +45,7 @@ To add configuration to your config file, you can copy it from the base config.
     older software for theses nicknames.
 * `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature.
 * `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow.
+* `autofollowing_nicknames`: Set to nicknames of (local) users that automatically follows every newly registered user.
 * `attachment_links`: Set to true to enable automatically adding attachment link text to statuses.
 * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`).
 * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). Default: `false`.
@@ -113,7 +114,7 @@ To add configuration to your config file, you can copy it from the base config.
     * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (See [`:mrf_mention`](#mrf_mention)).
     * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (See [`:mrf_vocabulary`](#mrf_vocabulary)).
     * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)).
-    * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.ActivityExpiration` to be enabled for processing the scheduled delections.
+    * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.Workers.PurgeExpiredActivity` to be enabled for processing the scheduled delections.
     * `Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy`: Makes all bot posts to disappear from public timelines.
 * `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
 * `transparency_exclusions`: Exclude specific instance names from MRF transparency.  The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.
@@ -219,12 +220,6 @@ config :pleroma, :mrf_user_allowlist, %{
 * `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`)
 * `enabled`: whether scheduled activities are sent to the job queue to be executed
 
-## Pleroma.ActivityExpiration
-
-Enables the worker which processes posts scheduled for deletion. Pinned posts are exempt from expiration.
-
-* `enabled`: whether expired activities will be sent to the job queue to be deleted
-
 ## FedSockets
 FedSockets is an experimental feature allowing for Pleroma backends to federate using a persistant websocket connection as opposed to making each federation a seperate http connection. This feature is currently off by default. It is configurable throught he following options.
 
@@ -416,25 +411,25 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
 * ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`.
 * ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header.
 
-### Pleroma.Plugs.RemoteIp
+### Pleroma.Web.Plugs.RemoteIp
 
 !!! warning
     If your instance is not behind at least one reverse proxy, you should not enable this plug.
 
-`Pleroma.Plugs.RemoteIp` is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
+`Pleroma.Web.Plugs.RemoteIp` is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
 
 Available options:
 
 * `enabled` - Enable/disable the plug. Defaults to `false`.
-* `headers` - A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Defaults to `["x-forwarded-for"]`.
+* `headers` - A list of strings naming the HTTP headers to use when deriving the true client IP address. Defaults to `["x-forwarded-for"]`.
-* `proxies` - A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Defaults to `[]`.
+* `proxies` - A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128.
-* `reserved` - Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network).
+* `reserved` - A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`.
 
 
 ### :rate_limit
 
 !!! note
-   If your instance is behind a reverse proxy ensure [`Pleroma.Plugs.RemoteIp`](#pleroma-plugs-remoteip) is enabled (it is enabled by default).
+   If your instance is behind a reverse proxy ensure [`Pleroma.Web.Plugs.RemoteIp`](#pleroma-plugs-remoteip) is enabled (it is enabled by default).
 
 A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where:
 
@@ -1083,6 +1078,20 @@ Control favicons for instances.
 
 * `enabled`: Allow/disallow displaying and getting instances favicons
 
+## Pleroma.User.Backup
+
+!!! note
+    Requires enabled email
+
+* `:purge_after_days` an integer, remove backup achives after N days.
+* `:limit_days` an integer, limit user to export not more often than once per N days.
+* `:dir` a string with a path to backup temporary directory or `nil` to let Pleroma choose temporary directory in the following order:
+    1. the directory named by the TMPDIR environment variable
+    2. the directory named by the TEMP environment variable
+    3. the directory named by the TMP environment variable
+    4. C:\TMP on Windows or /tmp on Unix-like operating systems
+    5. as a last resort, the current working directory
+
 ## Frontend management
 
 Frontends in Pleroma are swappable - you can specify which one to use here.

docs/configuration/howto_ejabberd.md

@@ -0,0 +1,136 @@
+# Configuring Ejabberd (XMPP Server) to use Pleroma for authentication
+
+If you want to give your Pleroma users an XMPP (chat) account, you can configure [Ejabberd](https://github.com/processone/ejabberd) to use your Pleroma server for user authentication, automatically giving every local user an XMPP account.
+
+In general, you just have to follow the configuration described at [https://docs.ejabberd.im/admin/configuration/authentication/#external-script](https://docs.ejabberd.im/admin/configuration/authentication/#external-script). Please read this section carefully. 
+
+Copy the script below to suitable path on your system and set owner and permissions. Also do not forget adjusting `PLEROMA_HOST` and `PLEROMA_PORT`, if necessary.
+
+```bash
+cp pleroma_ejabberd_auth.py /etc/ejabberd/pleroma_ejabberd_auth.py
+chown ejabberd /etc/ejabberd/pleroma_ejabberd_auth.py
+chmod 700 /etc/ejabberd/pleroma_ejabberd_auth.py
+```
+
+Set external auth params in ejabberd.yaml file:
+
+```bash
+auth_method: [external]
+extauth_program: "python3 /etc/ejabberd/pleroma_ejabberd_auth.py"
+extauth_instances: 3
+auth_use_cache: false
+```
+
+Restart / reload your ejabberd service.
+
+After restarting your Ejabberd server, your users should now be able to connect with their Pleroma credentials.
+
+
+```python
+import sys
+import struct
+import http.client
+from base64 import b64encode
+import logging
+
+
+PLEROMA_HOST = "127.0.0.1"
+PLEROMA_PORT = "4000"
+AUTH_ENDPOINT = "/api/v1/accounts/verify_credentials"
+USER_ENDPOINT = "/api/v1/accounts"
+LOGFILE = "/var/log/ejabberd/pleroma_auth.log"
+
+logging.basicConfig(filename=LOGFILE, level=logging.INFO)
+
+
+# Pleroma functions
+def create_connection():
+    return http.client.HTTPConnection(PLEROMA_HOST, PLEROMA_PORT)
+
+
+def verify_credentials(user: str, password: str) -> bool:
+    user_pass_b64 = b64encode("{}:{}".format(
+        user, password).encode('utf-8')).decode("ascii")
+    params = {}
+    headers = {
+        "Authorization": "Basic {}".format(user_pass_b64)
+    }
+
+    try:
+        conn = create_connection()
+        conn.request("GET", AUTH_ENDPOINT, params, headers)
+
+        response = conn.getresponse()
+        if response.status == 200:
+            return True
+
+        return False
+    except Exception as e:
+        logging.info("Can not connect: %s", str(e))
+        return False
+
+
+def does_user_exist(user: str) -> bool:
+    conn = create_connection()
+    conn.request("GET", "{}/{}".format(USER_ENDPOINT, user))
+
+    response = conn.getresponse()
+    if response.status == 200:
+        return True
+
+    return False
+
+
+def auth(username: str, server: str, password: str) -> bool:
+    return verify_credentials(username, password)
+
+
+def isuser(username, server):
+    return does_user_exist(username)
+
+
+def read():
+    (pkt_size,) = struct.unpack('>H', bytes(sys.stdin.read(2), encoding='utf8'))
+    pkt = sys.stdin.read(pkt_size)
+    cmd = pkt.split(':')[0]
+    if cmd == 'auth':
+        username, server, password = pkt.split(':', 3)[1:]
+        write(auth(username, server, password))
+    elif cmd == 'isuser':
+        username, server = pkt.split(':', 2)[1:]
+        write(isuser(username, server))
+    elif cmd == 'setpass':
+        # u, s, p = pkt.split(':', 3)[1:]
+        write(False)
+    elif cmd == 'tryregister':
+        # u, s, p = pkt.split(':', 3)[1:]
+        write(False)
+    elif cmd == 'removeuser':
+        # u, s = pkt.split(':', 2)[1:]
+        write(False)
+    elif cmd == 'removeuser3':
+        # u, s, p = pkt.split(':', 3)[1:]
+        write(False)
+    else:
+        write(False)
+
+
+def write(result):
+    if result:
+        sys.stdout.write('\x00\x02\x00\x01')
+    else:
+        sys.stdout.write('\x00\x02\x00\x00')
+    sys.stdout.flush()
+
+
+if __name__ == "__main__":
+    logging.info("Starting pleroma ejabberd auth daemon...")
+    while True:
+        try:
+            read()
+        except Exception as e:
+            logging.info(
+                "Error while processing data from ejabberd %s", str(e))
+            pass
+
+```

docs/dev.md

@@ -6,7 +6,7 @@ This document contains notes and guidelines for Pleroma developers.
 
 * Pleroma supports hierarchical OAuth scopes, just like Mastodon but with added granularity of admin scopes. For a reference, see [Mastodon OAuth scopes](https://docs.joinmastodon.org/api/oauth-scopes/).
 
-* It is important to either define OAuth scope restrictions or explicitly mark OAuth scope check as skipped, for every controller action. To define scopes, call `plug(Pleroma.Plugs.OAuthScopesPlug, %{scopes: [...]})`. To explicitly set OAuth scopes check skipped, call `plug(:skip_plug, Pleroma.Plugs.OAuthScopesPlug <when ...>)`.
+* It is important to either define OAuth scope restrictions or explicitly mark OAuth scope check as skipped, for every controller action. To define scopes, call `plug(Pleroma.Web.Plugs.OAuthScopesPlug, %{scopes: [...]})`. To explicitly set OAuth scopes check skipped, call `plug(:skip_plug, Pleroma.Web.Plugs.OAuthScopesPlug <when ...>)`.
 
 * In controllers, `use Pleroma.Web, :controller` will result in `action/2` (see `Pleroma.Web.controller/0` for definition) be called prior to actual controller action, and it'll perform security / privacy checks before passing control to actual controller action.
 
@@ -16,7 +16,7 @@ This document contains notes and guidelines for Pleroma developers.
 
 ## [HTTP Basic Authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization)
 
-* With HTTP Basic Auth, OAuth scopes check is _not_ performed for any action (since password is provided during the auth, requester is able to obtain a token with full permissions anyways). `Pleroma.Plugs.AuthenticationPlug` and `Pleroma.Plugs.LegacyAuthenticationPlug` both call `Pleroma.Plugs.OAuthScopesPlug.skip_plug(conn)` when password is provided.
+* With HTTP Basic Auth, OAuth scopes check is _not_ performed for any action (since password is provided during the auth, requester is able to obtain a token with full permissions anyways). `Pleroma.Web.Plugs.AuthenticationPlug` and `Pleroma.Web.Plugs.LegacyAuthenticationPlug` both call `Pleroma.Web.Plugs.OAuthScopesPlug.skip_plug(conn)` when password is provided.
 
 ## Auth-related configuration, OAuth consumer mode etc.
 

docs/installation/alpine_linux_en.md

@@ -13,6 +13,7 @@ It assumes that you have administrative rights, either as root or a user with [s
 * `erlang-parsetools`
 * `erlang-xmerl`
 * `git`
+* `file-dev`
 * Development Tools
 * `cmake`
 
@@ -20,6 +21,9 @@ It assumes that you have administrative rights, either as root or a user with [s
 
 * `nginx` (preferred, example configs for other reverse proxies can be found in the repo)
 * `certbot` (or any other ACME client for Let’s Encrypt certificates)
+* `ImageMagick`
+* `ffmpeg`
+* `exiftool`
 
 ### Prepare the system
 
@@ -29,7 +33,6 @@ It assumes that you have administrative rights, either as root or a user with [s
 awk 'NR==2' /etc/apk/repositories | sed 's/main/community/' | tee -a /etc/apk/repositories
 ```
 
-
 * Then update the system, if not already done:
 
 ```shell
@@ -40,7 +43,7 @@ sudo apk upgrade
 * Install some tools, which are needed later:
 
 ```shell
-sudo apk add git build-base cmake
+sudo apk add git build-base cmake file-dev
 ```
 
 ### Install Elixir and Erlang
@@ -56,6 +59,7 @@ sudo apk add erlang erlang-runtime-tools erlang-xmerl elixir
 ```shell
 sudo apk add erlang-eldap
 ```
+
 ### Install PostgreSQL
 
 * Install Postgresql server:
@@ -76,6 +80,12 @@ sudo /etc/init.d/postgresql start
 sudo rc-update add postgresql
 ```
 
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+```shell
+sudo apk add ffmpeg imagemagick exiftool
+```
+
 ### Install PleromaBE
 
 * Add a new system user for the Pleroma service:

docs/installation/arch_linux_en.md

@@ -10,11 +10,15 @@ This guide will assume that you have administrative rights, either as root or a
 * `git`
 * `base-devel`
 * `cmake`
+* `file`
 
 #### Optional packages used in this guide
 
 * `nginx` (preferred, example configs for other reverse proxies can be found in the repo)
 * `certbot` (or any other ACME client for Let’s Encrypt certificates)
+* `ImageMagick`
+* `ffmpeg`
+* `exiftool`
 
 ### Prepare the system
 
@@ -27,7 +31,7 @@ sudo pacman -Syu
 * Install some of the above mentioned programs:
 
 ```shell
-sudo pacman -S git base-devel elixir cmake
+sudo pacman -S git base-devel elixir cmake file
 ```
 
 ### Install PostgreSQL
@@ -52,6 +56,12 @@ sudo -iu postgres initdb -D /var/lib/postgres/data
 sudo systemctl enable --now postgresql.service
 ```
 
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+```shell
+sudo pacman -S ffmpeg imagemagick perl-image-exiftool
+```
+
 ### Install PleromaBE
 
 * Add a new system user for the Pleroma service:

docs/installation/debian_based_en.md

@@ -10,6 +10,7 @@ This guide will assume you are on Debian Stretch. This guide should also work wi
 * `elixir` (1.8+, Follow the guide to install from the Erlang Solutions repo or use [asdf](https://github.com/asdf-vm/asdf) as the pleroma user)
 * `erlang-dev`
 * `erlang-nox`
+* `libmagic-dev`
 * `git`
 * `build-essential`
 * `cmake`
@@ -18,6 +19,9 @@ This guide will assume you are on Debian Stretch. This guide should also work wi
 
 * `nginx` (preferred, example configs for other reverse proxies can be found in the repo)
 * `certbot` (or any other ACME client for Let’s Encrypt certificates)
+* `ImageMagick`
+* `ffmpeg`
+* `exiftool`
 
 ### Prepare the system
 
@@ -31,7 +35,7 @@ sudo apt full-upgrade
 * Install some of the above mentioned programs:
 
 ```shell
-sudo apt install git build-essential postgresql postgresql-contrib cmake
+sudo apt install git build-essential postgresql postgresql-contrib cmake libmagic-devel
 ```
 
 ### Install Elixir and Erlang
@@ -50,6 +54,12 @@ sudo apt update
 sudo apt install elixir erlang-dev erlang-nox
 ```
 
+### Optional packages: [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)
+
+```shell
+sudo apt install imagemagick ffmpeg libimage-exiftool-perl
+```
+
 ### Install PleromaBE
 
 * Add a new system user for the Pleroma service:
@@ -91,6 +101,7 @@ sudo -Hu pleroma mix deps.get
 mv config/{generated_config.exs,prod.secret.exs}
 ```
 
+
 * The previous command creates also the file `config/setup_db.psql`, with which you can create the database:
 
 ```shell
@@ -171,6 +182,7 @@ sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.se
 ```
 
 * Edit the service file and make sure that all paths fit your installation
+* Check that `EnvironmentFile` contains the correct path to the env file. Or generate the env file: `sudo -Hu pleroma mix pleroma.release_env gen`
 * Enable and start `pleroma.service`:
 
 ```shell

docs/installation/debian_based_jp.md

@@ -17,11 +17,15 @@
 - `git`
 - `build-essential`
 - `cmake`
+- `libmagic-dev`
 
 #### このガイドで利用している追加パッケージ
 
 - `nginx` (おすすめです。他のリバースプロキシを使う場合は、参考となる設定をこのリポジトリから探してください)
 - `certbot` (または何らかのLet's Encrypt向けACMEクライアント)
+- `ImageMagick`
+- `ffmpeg`
+- `exiftool`
 
 ### システムを準備する
 
@@ -33,10 +37,9 @@ sudo apt full-upgrade
 
 * 上記に挙げたパッケージをインストールしておきます。
 ```
-sudo apt install git build-essential postgresql postgresql-contrib cmake
+sudo apt install git build-essential postgresql postgresql-contrib cmake ffmpeg imagemagick libmagic-dev
 ```
 
-
 ### ElixirとErlangをインストールします
 
 * Erlangのリポジトリをダウンロードおよびインストールします。
@@ -51,6 +54,12 @@ sudo apt update
 sudo apt install elixir erlang-dev erlang-nox
 ```
 
+### オプションパッケージ: [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)
+
+```shell
+sudo apt install imagemagick ffmpeg libimage-exiftool-perl
+```
+
 ### Pleroma BE (バックエンド) をインストールします
 
 *  Pleroma用に新しいユーザーを作ります。

docs/installation/freebsd_en.md

@@ -26,6 +26,12 @@ Setup the required services to automatically start at boot, using `sysrc(8)`.
 # service postgresql start
 ```
 
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+```shell
+# pkg install imagemagick ffmpeg p5-Image-ExifTool
+```
+
 ## Configuring Pleroma
 
 Create a user for Pleroma:

docs/installation/gentoo_en.md

@@ -29,12 +29,16 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * `dev-lang/elixir`
 * `dev-vcs/git`
 * `dev-util/cmake`
+* `sys-apps/file`
 
 #### Optional ebuilds used in this guide
 
 * `www-servers/nginx` (preferred, example configs for other reverse proxies can be found in the repo)
 * `app-crypt/certbot` (or any other ACME client for Let’s Encrypt certificates)
 * `app-crypt/certbot-nginx` (nginx certbot plugin that allows use of the all-powerful `--nginx` flag on certbot)
+* `media-gfx/imagemagick`
+* `media-video/ffmpeg`
+* `media-libs/exiftool`
 
 ### Prepare the system
 
@@ -47,7 +51,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * Emerge all required the required and suggested software in one go:
 
 ```shell
- # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake
+ # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake sys-apps/file
 ```
 
 If you would not like to install the optional packages, remove them from this line. 
@@ -87,6 +91,12 @@ If you do not plan to make any modifications to your Pleroma instance, cloning d
 
 Not only does this make it much easier to deploy changes you make, as you can commit and pull from upstream and all that good stuff from the comfort of your local machine then simply `git pull` on your instance server when you're ready to deploy, it also ensures you are compliant with the Affero General Public Licence that Pleroma is licenced under, which stipulates that all network services provided with modified AGPL code must publish their changes on a publicly available internet service and for free. It also makes it much easier to ask for help from and provide help to your fellow Pleroma admins if your public repo always reflects what you are running because it is part of your deployment procedure.
 
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+```shell
+# emerge --ask media-video/ffmpeg media-gfx/imagemagick media-libs/exiftool
+```
+
 ### Install PleromaBE
 
 * Add a new system user for the Pleroma service and set up default directories:

docs/installation/netbsd_en.md

@@ -10,7 +10,7 @@ Pleroma uses.
 
 The `mksh` shell is needed to run the Elixir `mix` script.
 
-`# pkgin install acmesh elixir git-base git-docs mksh nginx postgresql11-server postgresql11-client postgresql11-contrib sudo`
+`# pkgin install acmesh elixir git-base git-docs mksh nginx postgresql11-server postgresql11-client postgresql11-contrib sudo ffmpeg4 ImageMagick`
 
 You can also build these packages using pkgsrc:
 ```
@@ -44,6 +44,10 @@ pgsql=YES
 
 First, run `# /etc/rc.d/pgsql start`. Then, `$ sudo -Hu pgsql -g pgsql createdb`.
 
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+`# pkgin install ImageMagick ffmpeg4 p5-Image-ExifTool`
+
 ## Configuring Pleroma
 
 Create a user for Pleroma:

docs/installation/openbsd_en.md

@@ -10,20 +10,34 @@ The following packages need to be installed:
 
   * elixir
   * gmake
-  * ImageMagick
   * git
   * postgresql-server
   * postgresql-contrib
   * cmake
+  * ffmpeg
+  * ImageMagick
 
 To install them, run the following command (with doas or as root):
 
 ```
-pkg_add elixir gmake ImageMagick git postgresql-server postgresql-contrib cmake
+pkg_add elixir gmake git postgresql-server postgresql-contrib cmake ffmpeg ImageMagick
 ```
 
 Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt.
 
+#### Optional software
+
+Per [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md):
+  * ImageMagick
+  * ffmpeg
+  * exiftool
+
+To install the above:
+
+```
+pkg_add ImageMagick ffmpeg p5-Image-ExifTool
+```
+
 #### Creating the pleroma user
 Pleroma will be run by a dedicated user, \_pleroma. Before creating it, insert the following lines in login.conf:
 ```

docs/installation/openbsd_fi.md

@@ -16,7 +16,18 @@ Matrix-kanava #freenode_#pleroma:matrix.org ovat hyviä paikkoja löytää apua
 
 Asenna tarvittava ohjelmisto:
 
-`# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3 cmake`
+`# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3 cmake ffmpeg ImageMagick`
+
+#### Optional software
+
+[`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md):
+  * ImageMagick
+  * ffmpeg
+  * exiftool
+
+Asenna tarvittava ohjelmisto:
+
+`# pkg_add ImageMagick ffmpeg p5-Image-ExifTool`
 
 Luo postgresql-tietokanta:
 

docs/installation/optional/media_graphics_packages.md

@@ -0,0 +1,32 @@
+# Optional software packages needed for specific functionality
+
+For specific Pleroma functionality (which is disabled by default) some or all of the below packages are required:
+  * `ImageMagic`
+  * `ffmpeg`
+  * `exiftool`
+  
+Please refer to documentation in `docs/installation` on how to install them on specific OS.
+  
+Note: the packages are not required with the current default settings of Pleroma.
+
+## `ImageMagick`
+
+`ImageMagick` is a set of tools to create, edit, compose, or convert bitmap images.
+
+It is required for the following Pleroma features:
+  * `Pleroma.Upload.Filters.Mogrify`, `Pleroma.Upload.Filters.Mogrifun` upload filters (related config: `Plaroma.Upload/filters` in `config/config.exs`)
+  * Media preview proxy for still images (related config: `media_preview_proxy/enabled` in `config/config.exs`)
+  
+## `ffmpeg`
+
+`ffmpeg` is software to record, convert and stream audio and video.
+
+It is required for the following Pleroma features:
+  * Media preview proxy for videos (related config: `media_preview_proxy/enabled` in `config/config.exs`)
+
+## `exiftool`
+
+`exiftool` is media files metadata reader/writer.
+
+It is required for the following Pleroma features:
+  * `Pleroma.Upload.Filters.Exiftool` upload filter (related config: `Plaroma.Upload/filters` in `config/config.exs`)

docs/installation/otp_en.md

@@ -27,17 +27,37 @@ Other than things bundled in the OTP release Pleroma depends on:
 * PostgreSQL (also utilizes extensions in postgresql-contrib)
 * nginx (could be swapped with another reverse proxy but this guide covers only it)
 * certbot (for Let's Encrypt certificates, could be swapped with another ACME client, but this guide covers only it)
+* libmagic/file
 
 === "Alpine"
     ```
     echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories
     apk update
-    apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot
+    apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot file-dev
     ```
 
 === "Debian/Ubuntu"
     ```
-    apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot
+    apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot libmagic-dev
+    ```
+
+### Installing optional packages
+
+Per [`docs/installation/optional/media_graphics_packages.md`](optional/media_graphics_packages.md):
+  * ImageMagick
+  * ffmpeg
+  * exiftool
+
+=== "Alpine"
+    ```
+    echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories
+    apk update
+    apk add imagemagick ffmpeg exiftool
+    ```
+
+=== "Debian/Ubuntu"
+    ```
+    apt install imagemagick ffmpeg libimage-exiftool-perl
     ```
 
 ## Setup
@@ -82,6 +102,8 @@ It is encouraged to check [Optimizing your PostgreSQL performance](../configurat
 If you are using PostgreSQL 12 or higher, add this to your Ecto database configuration
 
 ```elixir
+#
+config :pleroma, Pleroma.Repo,
 prepare: :named,
 parameters: [
   plan_cache_mode: "force_custom_plan"
@@ -127,6 +149,9 @@ chown -R pleroma /etc/pleroma
 # Run the config generator
 su pleroma -s $SHELL -lc "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql"
 
+# Run the environment file generator.
+su pleroma -s $SHELL -lc "./bin/pleroma_ctl release_env gen"
+
 # Create the postgres database
 su postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql"
 
@@ -137,7 +162,7 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
 # su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
 
 # Start the instance to verify that everything is working as expected
-su pleroma -s $SHELL -lc "./bin/pleroma daemon"
+su pleroma -s $SHELL -lc "export $(cat /opt/pleroma/config/pleroma.env); ./bin/pleroma daemon"
 
 # Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly
 sleep 20 && curl http://localhost:4000/api/v1/instance
@@ -289,4 +314,3 @@ This will create an account withe the username of 'joeuser' with the email addre
 ## Questions
 
 Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
-

installation/init.d/pleroma

@@ -8,6 +8,7 @@ pidfile="/var/run/pleroma.pid"
 directory=/opt/pleroma
 healthcheck_delay=60
 healthcheck_timer=30
+export $(cat /opt/pleroma/config/pleroma.env)
 
 : ${pleroma_port:-4000}
 

installation/pleroma.service

@@ -17,6 +17,8 @@ Environment="MIX_ENV=prod"
 Environment="HOME=/var/lib/pleroma"
 ; Path to the folder containing the Pleroma installation.
 WorkingDirectory=/opt/pleroma
+; Path to the environment file. the file contains RELEASE_COOKIE and etc 
+EnvironmentFile=/opt/pleroma/config/pleroma.env
 ; Path to the Mix binary.
 ExecStart=/usr/bin/mix phx.server
 
@@ -29,8 +31,6 @@ ProtectHome=true
 ProtectSystem=full
 ; Sets up a new /dev mount for the process and only adds API pseudo devices like /dev/null, /dev/zero or /dev/random but not physical devices. Disabled by default because it may not work on devices like the Raspberry Pi.
 PrivateDevices=false
-; Ensures that the service process and all its children can never gain new privileges through execve().
-NoNewPrivileges=true
 ; Drops the sysadmin capability from the daemon.
 CapabilityBoundingSet=~CAP_SYS_ADMIN
 

installation/pleroma.vcl

@@ -1,3 +1,4 @@
+# Recommended varnishncsa logging format: '%h %l %u %t "%m %{X-Forwarded-Proto}i://%{Host}i%U%q %H" %s %b "%{Referer}i" "%{User-agent}i"'
 vcl 4.1;
 import std;
 
@@ -14,8 +15,11 @@ acl purge {
 sub vcl_recv {
     # Redirect HTTP to HTTPS
     if (std.port(server.ip) != 443) {
+      set req.http.X-Forwarded-Proto = "http";
       set req.http.x-redir = "https://" + req.http.host + req.url;
       return (synth(750, ""));
+    } else {
+      set req.http.X-Forwarded-Proto = "https";
     }
 
     # CHUNKED SUPPORT
@@ -105,7 +109,7 @@ sub vcl_hash {
 
 sub vcl_backend_fetch {
     # Be more lenient for slow servers on the fediverse
-    if bereq.url ~ "^/proxy/" {
+    if (bereq.url ~ "^/proxy/") {
       set bereq.first_byte_timeout = 300s;
     }
 

lib/mix/tasks/pleroma/count_statuses.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.CountStatuses do
   @shortdoc "Re-counts statuses for all users"
 

lib/mix/tasks/pleroma/digest.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.Digest do
   use Mix.Task
   import Mix.Pleroma

lib/mix/tasks/pleroma/docs.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.Docs do
   use Mix.Task
   import Mix.Pleroma

lib/mix/tasks/pleroma/email.ex

@@ -1,12 +1,16 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.Email do
   use Mix.Task
   import Mix.Pleroma
 
-  @shortdoc "Simple Email test"
+  @shortdoc "Email administrative tasks"
   @moduledoc File.read!("docs/administration/CLI_tasks/email.md")
 
   def run(["test" | args]) do
-    Mix.Pleroma.start_pleroma()
+    start_pleroma()
 
     {options, [], []} =
       OptionParser.parse(
@@ -21,4 +25,20 @@ defmodule Mix.Tasks.Pleroma.Email do
 
     shell_info("Test email has been sent to #{inspect(email.to)} from #{inspect(email.from)}")
   end
+
+  def run(["resend_confirmation_emails"]) do
+    start_pleroma()
+
+    shell_info("Sending emails to all unconfirmed users")
+
+    Pleroma.User.Query.build(%{
+      local: true,
+      deactivated: false,
+      confirmation_pending: true,
+      invisible: false
+    })
+    |> Pleroma.Repo.chunk_stream(500)
+    |> Stream.each(&Pleroma.User.try_send_confirmation_email(&1))
+    |> Stream.run()
+  end
 end

lib/mix/tasks/pleroma/instance.ex

@@ -33,7 +33,12 @@ defmodule Mix.Tasks.Pleroma.Instance do
           uploads_dir: :string,
           static_dir: :string,
           listen_ip: :string,
-          listen_port: :string
+          listen_port: :string,
+          strip_uploads: :string,
+          anonymize_uploads: :string,
+          dedupe_uploads: :string,
+          skip_release_env: :boolean,
+          release_env_file: :string
         ],
         aliases: [
           o: :output,
@@ -158,6 +163,30 @@ defmodule Mix.Tasks.Pleroma.Instance do
         )
         |> Path.expand()
 
+      strip_uploads =
+        get_option(
+          options,
+          :strip_uploads,
+          "Do you want to strip location (GPS) data from uploaded images? (y/n)",
+          "y"
+        ) === "y"
+
+      anonymize_uploads =
+        get_option(
+          options,
+          :anonymize_uploads,
+          "Do you want to anonymize the filenames of uploads? (y/n)",
+          "n"
+        ) === "y"
+
+      dedupe_uploads =
+        get_option(
+          options,
+          :dedupe_uploads,
+          "Do you want to deduplicate uploaded files? (y/n)",
+          "n"
+        ) === "y"
+
       Config.put([:instance, :static_dir], static_dir)
 
       secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
@@ -188,7 +217,13 @@ defmodule Mix.Tasks.Pleroma.Instance do
           uploads_dir: uploads_dir,
           rum_enabled: rum_enabled,
           listen_ip: listen_ip,
-          listen_port: listen_port
+          listen_port: listen_port,
+          upload_filters:
+            upload_filters(%{
+              strip: strip_uploads,
+              anonymize: anonymize_uploads,
+              dedupe: dedupe_uploads
+            })
         )
 
       result_psql =
@@ -208,6 +243,24 @@ defmodule Mix.Tasks.Pleroma.Instance do
 
       write_robots_txt(static_dir, indexable, template_dir)
 
+      if Keyword.get(options, :skip_release_env, false) do
+        shell_info("""
+        Release environment file is skip. Please generate the release env file before start.
+        `MIX_ENV=#{Mix.env()} mix pleroma.release_env gen`
+        """)
+      else
+        shell_info("Generation the environment file:")
+
+        release_env_args =
+          with path when not is_nil(path) <- Keyword.get(options, :release_env_file) do
+            ["gen", "--path", path]
+          else
+            _ -> ["gen"]
+          end
+
+        Mix.Tasks.Pleroma.ReleaseEnv.run(release_env_args)
+      end
+
       shell_info(
         "\n All files successfully written! Refer to the installation instructions for your platform for next steps."
       )
@@ -247,4 +300,31 @@ defmodule Mix.Tasks.Pleroma.Instance do
     File.write(robots_txt_path, robots_txt)
     shell_info("Writing #{robots_txt_path}.")
   end
+
+  defp upload_filters(filters) when is_map(filters) do
+    enabled_filters =
+      if filters.strip do
+        [Pleroma.Upload.Filter.ExifTool]
+      else
+        []
+      end
+
+    enabled_filters =
+      if filters.anonymize do
+        enabled_filters ++ [Pleroma.Upload.Filter.AnonymizeFilename]
+      else
+        enabled_filters
+      end
+
+    enabled_filters =
+      if filters.dedupe do
+        enabled_filters ++ [Pleroma.Upload.Filter.Dedupe]
+      else
+        enabled_filters
+      end
+
+    enabled_filters
+  end
+
+  defp upload_filters(_), do: []
 end

lib/mix/tasks/pleroma/notification_settings.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.NotificationSettings do
   @shortdoc "Enable&Disable privacy option for push notifications"
   @moduledoc """

lib/mix/tasks/pleroma/relay.ex

@@ -21,10 +21,19 @@ defmodule Mix.Tasks.Pleroma.Relay do
     end
   end
 
-  def run(["unfollow", target]) do
+  def run(["unfollow", target | rest]) do
     start_pleroma()
 
-    with {:ok, _activity} <- Relay.unfollow(target) do
+    {options, [], []} =
+      OptionParser.parse(
+        rest,
+        strict: [force: :boolean],
+        aliases: [f: :force]
+      )
+
+    force = Keyword.get(options, :force, false)
+
+    with {:ok, _activity} <- Relay.unfollow(target, %{force: force}) do
       # put this task to sleep to allow the genserver to push out the messages
       :timer.sleep(500)
     else

lib/mix/tasks/pleroma/release_env.ex

@@ -0,0 +1,76 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.ReleaseEnv do
+  use Mix.Task
+  import Mix.Pleroma
+
+  @shortdoc "Generate Pleroma environment file."
+  @moduledoc File.read!("docs/administration/CLI_tasks/release_environments.md")
+
+  def run(["gen" | rest]) do
+    {options, [], []} =
+      OptionParser.parse(
+        rest,
+        strict: [
+          force: :boolean,
+          path: :string
+        ],
+        aliases: [
+          p: :path,
+          f: :force
+        ]
+      )
+
+    file_path =
+      get_option(
+        options,
+        :path,
+        "Environment file path",
+        "./config/pleroma.env"
+      )
+
+    env_path = Path.expand(file_path)
+
+    proceed? =
+      if File.exists?(env_path) do
+        get_option(
+          options,
+          :force,
+          "Environment file already exists. Do you want to overwrite the #{env_path} file? (y/n)",
+          "n"
+        ) === "y"
+      else
+        true
+      end
+
+    if proceed? do
+      case do_generate(env_path) do
+        {:error, reason} ->
+          shell_error(
+            File.Error.message(%{action: "write to file", reason: reason, path: env_path})
+          )
+
+        _ ->
+          shell_info("\nThe file generated: #{env_path}.\n")
+
+          shell_info("""
+          WARNING: before start pleroma app please make sure to make the file read-only and non-modifiable.
+            Example:
+              chmod 0444 #{file_path}
+              chattr +i #{file_path}
+          """)
+      end
+    else
+      shell_info("\nThe file is exist. #{env_path}.\n")
+    end
+  end
+
+  def do_generate(path) do
+    content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}"
+
+    File.mkdir_p!(Path.dirname(path))
+    File.write(path, content)
+  end
+end

lib/mix/tasks/pleroma/user.ex

@@ -196,17 +196,24 @@ defmodule Mix.Tasks.Pleroma.User do
       OptionParser.parse(
         rest,
         strict: [
-          moderator: :boolean,
           admin: :boolean,
-          locked: :boolean
+          confirmed: :boolean,
+          locked: :boolean,
+          moderator: :boolean
         ]
       )
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
       user =
-        case Keyword.get(options, :moderator) do
+        case Keyword.get(options, :admin) do
           nil -> user
-          value -> set_moderator(user, value)
+          value -> set_admin(user, value)
+        end
+
+      user =
+        case Keyword.get(options, :confirmed) do
+          nil -> user
+          value -> set_confirmed(user, value)
         end
 
       user =
@@ -216,9 +223,9 @@ defmodule Mix.Tasks.Pleroma.User do
         end
 
       _user =
-        case Keyword.get(options, :admin) do
+        case Keyword.get(options, :moderator) do
           nil -> user
-          value -> set_admin(user, value)
+          value -> set_moderator(user, value)
         end
     else
       _ ->
@@ -353,6 +360,42 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
+  def run(["confirm_all"]) do
+    start_pleroma()
+
+    Pleroma.User.Query.build(%{
+      local: true,
+      deactivated: false,
+      is_moderator: false,
+      is_admin: false,
+      invisible: false
+    })
+    |> Pleroma.Repo.chunk_stream(500, :batches)
+    |> Stream.each(fn users ->
+      users
+      |> Enum.each(fn user -> User.need_confirmation(user, false) end)
+    end)
+    |> Stream.run()
+  end
+
+  def run(["unconfirm_all"]) do
+    start_pleroma()
+
+    Pleroma.User.Query.build(%{
+      local: true,
+      deactivated: false,
+      is_moderator: false,
+      is_admin: false,
+      invisible: false
+    })
+    |> Pleroma.Repo.chunk_stream(500, :batches)
+    |> Stream.each(fn users ->
+      users
+      |> Enum.each(fn user -> User.need_confirmation(user, true) end)
+    end)
+    |> Stream.run()
+  end
+
   def run(["sign_out", nickname]) do
     start_pleroma()
 
@@ -376,7 +419,7 @@ defmodule Mix.Tasks.Pleroma.User do
       |> Enum.each(fn user ->
         shell_info(
           "#{user.nickname} moderator: #{user.is_moderator}, admin: #{user.is_admin}, locked: #{
-            user.locked
+            user.is_locked
           }, deactivated: #{user.deactivated}"
         )
       end)
@@ -404,10 +447,17 @@ defmodule Mix.Tasks.Pleroma.User do
   defp set_locked(user, value) do
     {:ok, user} =
       user
-      |> Changeset.change(%{locked: value})
+      |> Changeset.change(%{is_locked: value})
       |> User.update_and_set_cache()
 
-    shell_info("Locked status of #{user.nickname}: #{user.locked}")
+    shell_info("Locked status of #{user.nickname}: #{user.is_locked}")
+    user
+  end
+
+  defp set_confirmed(user, value) do
+    {:ok, user} = User.need_confirmation(user, !value)
+
+    shell_info("Confirmation pending status of #{user.nickname}: #{user.confirmation_pending}")
     user
   end
 end

lib/phoenix/transports/web_socket/raw.ex

@@ -31,7 +31,12 @@ defmodule Phoenix.Transports.WebSocket.Raw do
 
     case conn do
       %{halted: false} = conn ->
-        case Transport.connect(endpoint, handler, transport, __MODULE__, nil, conn.params) do
+        case handler.connect(%{
+               endpoint: endpoint,
+               transport: transport,
+               options: [serializer: nil],
+               params: conn.params
+             }) do
           {:ok, socket} ->
             {:ok, conn, {__MODULE__, {socket, opts}}}
 

lib/pleroma/activity.ex

@@ -14,6 +14,7 @@ defmodule Pleroma.Activity do
   alias Pleroma.ReportNote
   alias Pleroma.ThreadMute
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
 
   import Ecto.Changeset
   import Ecto.Query
@@ -153,6 +154,18 @@ defmodule Pleroma.Activity do
 
   def get_bookmark(_, _), do: nil
 
+  def get_report(activity_id) do
+    opts = %{
+      type: "Flag",
+      skip_preload: true,
+      preload_report_notes: true
+    }
+
+    ActivityPub.fetch_activities_query([], opts)
+    |> where(id: ^activity_id)
+    |> Repo.one()
+  end
+
   def change(struct, params \\ %{}) do
     struct
     |> cast(params, [:data, :recipients])

lib/pleroma/activity/ir/topics.ex

@@ -40,7 +40,8 @@ defmodule Pleroma.Activity.Ir.Topics do
   end
 
   defp item_creation_tags(tags, object, %{data: %{"type" => "Create"}} = activity) do
-    tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
+    tags ++
+      remote_topics(activity) ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
   end
 
   defp item_creation_tags(tags, _, _) do
@@ -55,9 +56,19 @@ defmodule Pleroma.Activity.Ir.Topics do
 
   defp hashtags_to_topics(_), do: []
 
+  defp remote_topics(%{local: true}), do: []
+
+  defp remote_topics(%{actor: actor}) when is_binary(actor),
+    do: ["public:remote:" <> URI.parse(actor).host]
+
+  defp remote_topics(_), do: []
+
   defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: []
 
   defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"]
 
+  defp attachment_topics(_object, %{actor: actor}) when is_binary(actor),
+    do: ["public:media", "public:remote:media:" <> URI.parse(actor).host]
+
   defp attachment_topics(_object, _act), do: ["public:media"]
 end

lib/pleroma/application.ex

@@ -52,11 +52,10 @@ defmodule Pleroma.Application do
     Pleroma.HTML.compile_scrubbers()
     Pleroma.Config.Oban.warn()
     Config.DeprecationWarnings.warn()
-    Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
+    Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
     Pleroma.ApplicationRequirements.verify!()
     setup_instrumenters()
     load_custom_modules()
-    check_system_commands()
     Pleroma.Docs.JSON.compile()
 
     adapter = Application.get_env(:tesla, :adapter)
@@ -89,18 +88,19 @@ defmodule Pleroma.Application do
         Pleroma.Repo,
         Config.TransferTask,
         Pleroma.Emoji,
-        Pleroma.Plugs.RateLimiter.Supervisor
+        Pleroma.Web.Plugs.RateLimiter.Supervisor
       ] ++
         cachex_children() ++
         http_children(adapter, @env) ++
         [
           Pleroma.Stats,
           Pleroma.JobQueueMonitor,
+          {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]},
           {Oban, Config.get(Oban)}
         ] ++
         task_children(@env) ++
         dont_run_in_test(@env) ++
-        chat_child(@env, chat_enabled?()) ++
+        chat_child(chat_enabled?()) ++
         [
           Pleroma.Web.Endpoint,
           Pleroma.Gopher.Server
@@ -151,7 +151,10 @@ defmodule Pleroma.Application do
 
     Pleroma.Web.Endpoint.MetricsExporter.setup()
     Pleroma.Web.Endpoint.PipelineInstrumenter.setup()
-    Pleroma.Web.Endpoint.Instrumenter.setup()
+
+    # Note: disabled until prometheus-phx is integrated into prometheus-phoenix:
+    # Pleroma.Web.Endpoint.Instrumenter.setup()
+    PrometheusPhx.setup()
   end
 
   defp cachex_children do
@@ -165,7 +168,11 @@ defmodule Pleroma.Application do
       build_cachex("web_resp", limit: 2500),
       build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
       build_cachex("failed_proxy_url", limit: 2500),
-      build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
+      build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
+      build_cachex("chat_message_id_idempotency_key",
+        expiration: chat_message_id_idempotency_key_expiration(),
+        limit: 500_000
+      )
     ]
   end
 
@@ -175,6 +182,9 @@ defmodule Pleroma.Application do
   defp idempotency_expiration,
     do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
 
+  defp chat_message_id_idempotency_key_expiration,
+    do: expiration(default: :timer.minutes(2), interval: :timer.seconds(60))
+
   defp seconds_valid_interval,
     do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
 
@@ -202,11 +212,14 @@ defmodule Pleroma.Application do
     ]
   end
 
-  defp chat_child(_env, true) do
+  defp chat_child(true) do
-    [Pleroma.Web.ChatChannel.ChatChannelState]
+    [
+      Pleroma.Web.ChatChannel.ChatChannelState,
+      {Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]}
+    ]
   end
 
-  defp chat_child(_, _), do: []
+  defp chat_child(_), do: []
 
   defp task_children(:test) do
     [
@@ -260,21 +273,4 @@ defmodule Pleroma.Application do
   end
 
   defp http_children(_, _), do: []
-
-  defp check_system_commands do
-    filters = Config.get([Pleroma.Upload, :filters])
-
-    check_filter = fn filter, command_required ->
-      with true <- filter in filters,
-           false <- Pleroma.Utils.command_available?(command_required) do
-        Logger.error(
-          "#{filter} is specified in list of Pleroma.Upload filters, but the #{command_required} command is not found"
-        )
-      end
-    end
-
-    check_filter.(Pleroma.Upload.Filters.Exiftool, "exiftool")
-    check_filter.(Pleroma.Upload.Filters.Mogrify, "mogrify")
-    check_filter.(Pleroma.Upload.Filters.Mogrifun, "mogrify")
-  end
 end

lib/pleroma/application_requirements.ex

@@ -9,6 +9,9 @@ defmodule Pleroma.ApplicationRequirements do
 
   defmodule VerifyError, do: defexception([:message])
 
+  alias Pleroma.Config
+  alias Pleroma.Helpers.MediaHelper
+
   import Ecto.Query
 
   require Logger
@@ -16,7 +19,8 @@ defmodule Pleroma.ApplicationRequirements do
   @spec verify!() :: :ok | VerifyError.t()
   def verify! do
     :ok
-    |> check_confirmation_accounts!
+    |> check_system_commands!()
+    |> check_confirmation_accounts!()
     |> check_migrations_applied!()
     |> check_welcome_message_config!()
     |> check_rum!()
@@ -48,7 +52,9 @@ defmodule Pleroma.ApplicationRequirements do
     if Pleroma.Config.get([:instance, :account_activation_required]) &&
          not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
       Logger.error(
-        "Account activation enabled, but no Mailer settings enabled.\nPlease set config :pleroma, :instance, account_activation_required: false\nOtherwise setup and enable Mailer."
+        "Account activation enabled, but no Mailer settings enabled.\n" <>
+          "Please set config :pleroma, :instance, account_activation_required: false\n" <>
+          "Otherwise setup and enable Mailer."
       )
 
       {:error,
@@ -81,7 +87,9 @@ defmodule Pleroma.ApplicationRequirements do
               Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end)
 
             Logger.error(
-              "The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true"
+              "The following migrations were not applied:\n#{down_migrations_text}" <>
+                "If you want to start Pleroma anyway, set\n" <>
+                "config :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true"
             )
 
             {:error, "Unapplied Migrations detected"}
@@ -124,14 +132,22 @@ defmodule Pleroma.ApplicationRequirements do
     case {setting, migrate} do
       {true, false} ->
         Logger.error(
-          "Use `RUM` index is enabled, but were not applied migrations for it.\nIf you want to start Pleroma anyway, set\nconfig :pleroma, :database, rum_enabled: false\nOtherwise apply the following migrations:\n`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`"
+          "Use `RUM` index is enabled, but were not applied migrations for it.\n" <>
+            "If you want to start Pleroma anyway, set\n" <>
+            "config :pleroma, :database, rum_enabled: false\n" <>
+            "Otherwise apply the following migrations:\n" <>
+            "`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`"
         )
 
         {:error, "Unapplied RUM Migrations detected"}
 
       {false, true} ->
         Logger.error(
-          "Detected applied migrations to use `RUM` index, but `RUM` isn't enable in settings.\nIf you want to use `RUM`, set\nconfig :pleroma, :database, rum_enabled: true\nOtherwise roll `RUM` migrations back.\n`mix ecto.rollback --migrations-path priv/repo/optional_migrations/rum_indexing/`"
+          "Detected applied migrations to use `RUM` index, but `RUM` isn't enable in settings.\n" <>
+            "If you want to use `RUM`, set\n" <>
+            "config :pleroma, :database, rum_enabled: true\n" <>
+            "Otherwise roll `RUM` migrations back.\n" <>
+            "`mix ecto.rollback --migrations-path priv/repo/optional_migrations/rum_indexing/`"
         )
 
         {:error, "RUM Migrations detected"}
@@ -140,4 +156,50 @@ defmodule Pleroma.ApplicationRequirements do
         :ok
     end
   end
+
+  defp check_system_commands!(:ok) do
+    filter_commands_statuses = [
+      check_filter(Pleroma.Upload.Filters.Exiftool, "exiftool"),
+      check_filter(Pleroma.Upload.Filters.Mogrify, "mogrify"),
+      check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify")
+    ]
+
+    preview_proxy_commands_status =
+      if !Config.get([:media_preview_proxy, :enabled]) or
+           MediaHelper.missing_dependencies() == [] do
+        true
+      else
+        Logger.error(
+          "The following dependencies required by Media preview proxy " <>
+            "(which is currently enabled) are not installed: " <>
+            inspect(MediaHelper.missing_dependencies())
+        )
+
+        false
+      end
+
+    if Enum.all?([preview_proxy_commands_status | filter_commands_statuses], & &1) do
+      :ok
+    else
+      {:error,
+       "System commands missing. Check logs and see `docs/installation` for more details."}
+    end
+  end
+
+  defp check_system_commands!(result), do: result
+
+  defp check_filter(filter, command_required) do
+    filters = Config.get([Pleroma.Upload, :filters])
+
+    if filter in filters and not Pleroma.Utils.command_available?(command_required) do
+      Logger.error(
+        "#{filter} is specified in list of Pleroma.Upload filters, but the " <>
+          "#{command_required} command is not found"
+      )
+
+      false
+    else
+      true
+    end
+  end
 end

lib/pleroma/bbs/authenticator.ex

@@ -4,8 +4,8 @@
 
 defmodule Pleroma.BBS.Authenticator do
   use Sshd.PasswordAuthenticator
-  alias Pleroma.Plugs.AuthenticationPlug
   alias Pleroma.User
+  alias Pleroma.Web.Plugs.AuthenticationPlug
 
   def authenticate(username, password) do
     username = to_string(username)

lib/pleroma/captcha/kocaptcha.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.Captcha.Kocaptcha do
   def new do
     endpoint = Pleroma.Config.get!([__MODULE__, :endpoint])
 
-    case Tesla.get(endpoint <> "/new") do
+    case Pleroma.HTTP.get(endpoint <> "/new") do
       {:error, _} ->
         %{error: :kocaptcha_service_unavailable}
 

lib/pleroma/chat.ex

@@ -18,6 +18,7 @@ defmodule Pleroma.Chat do
   It is a helper only, to make it easy to display a list of chats with other people, ordered by last bump. The actual messages are retrieved by querying the recipients of the ChatMessages.
   """
 
+  @type t :: %__MODULE__{}
   @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
 
   schema "chats" do
@@ -41,16 +42,28 @@ defmodule Pleroma.Chat do
     |> unique_constraint(:user_id, name: :chats_user_id_recipient_index)
   end
 
+  @spec get_by_user_and_id(User.t(), FlakeId.Ecto.CompatType.t()) ::
+          {:ok, t()} | {:error, :not_found}
+  def get_by_user_and_id(%User{id: user_id}, id) do
+    from(c in __MODULE__,
+      where: c.id == ^id,
+      where: c.user_id == ^user_id
+    )
+    |> Repo.find_resource()
+  end
+
+  @spec get_by_id(FlakeId.Ecto.CompatType.t()) :: t() | nil
   def get_by_id(id) do
-    __MODULE__
+    Repo.get(__MODULE__, id)
-    |> Repo.get(id)
   end
 
+  @spec get(FlakeId.Ecto.CompatType.t(), String.t()) :: t() | nil
   def get(user_id, recipient) do
-    __MODULE__
+    Repo.get_by(__MODULE__, user_id: user_id, recipient: recipient)
-    |> Repo.get_by(user_id: user_id, recipient: recipient)
   end
 
+  @spec get_or_create(FlakeId.Ecto.CompatType.t(), String.t()) ::
+          {:ok, t()} | {:error, Ecto.Changeset.t()}
   def get_or_create(user_id, recipient) do
     %__MODULE__{}
     |> changeset(%{user_id: user_id, recipient: recipient})
@@ -62,6 +75,8 @@ defmodule Pleroma.Chat do
     )
   end
 
+  @spec bump_or_create(FlakeId.Ecto.CompatType.t(), String.t()) ::
+          {:ok, t()} | {:error, Ecto.Changeset.t()}
   def bump_or_create(user_id, recipient) do
     %__MODULE__{}
     |> changeset(%{user_id: user_id, recipient: recipient})

lib/pleroma/config/deprecation_warnings.ex

@@ -33,39 +33,14 @@ defmodule Pleroma.Config.DeprecationWarnings do
     end
   end
 
-  def mrf_user_allowlist do
-    config = Config.get(:mrf_user_allowlist)
-
-    if config && Enum.any?(config, fn {k, _} -> is_atom(k) end) do
-      rewritten =
-        Enum.reduce(Config.get(:mrf_user_allowlist), Map.new(), fn {k, v}, acc ->
-          Map.put(acc, to_string(k), v)
-        end)
-
-      Config.put(:mrf_user_allowlist, rewritten)
-
-      Logger.error("""
-      !!!DEPRECATION WARNING!!!
-      As of Pleroma 2.0.7, the `mrf_user_allowlist` setting changed of format.
-      Pleroma 2.1 will remove support for the old format. Please change your configuration to match this:
-
-      config :pleroma, :mrf_user_allowlist, #{inspect(rewritten, pretty: true)}
-      """)
-
-      :error
-    else
-      :ok
-    end
-  end
-
   def warn do
     with :ok <- check_hellthread_threshold(),
-         :ok <- mrf_user_allowlist(),
          :ok <- check_old_mrf_config(),
          :ok <- check_media_proxy_whitelist_config(),
          :ok <- check_welcome_message_config(),
          :ok <- check_gun_pool_options(),
-         :ok <- check_activity_expiration_config() do
+         :ok <- check_activity_expiration_config(),
+         :ok <- check_remote_ip_plug_name() do
       :ok
     else
       _ ->
@@ -83,9 +58,9 @@ defmodule Pleroma.Config.DeprecationWarnings do
     if use_old_config do
       Logger.error("""
       !!!DEPRECATION WARNING!!!
-      Your config is using the old namespace for Welcome messages configuration. You need to change to the new namespace:
+      Your config is using the old namespace for Welcome messages configuration. You need to convert to the new namespace. e.g.,
-      \n* `config :pleroma, :instance, welcome_user_nickname` is now `config :pleroma, :welcome, :direct_message, :sender_nickname`
+      \n* `config :pleroma, :instance, welcome_user_nickname` and `config :pleroma, :instance, welcome_message` are now equal to:
-      \n* `config :pleroma, :instance, welcome_message` is now `config :pleroma, :welcome, :direct_message, :message`
+      \n* `config :pleroma, :welcome, direct_message: [enabled: true, sender_nickname: "NICKNAME", message: "Your welcome message"]`"
       """)
 
       :error
@@ -148,7 +123,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
     if timeout = pool_config[:await_up_timeout] do
       Logger.warn("""
       !!!DEPRECATION WARNING!!!
-      Your config is using old setting name `await_up_timeout` instead of `connect_timeout`. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
+      Your config is using old setting `config :pleroma, :connections_pool, await_up_timeout`. Please change to `config :pleroma, :connections_pool, connect_timeout` to ensure compatibility with future releases.
       """)
 
       Config.put(:connections_pool, Keyword.put_new(pool_config, :connect_timeout, timeout))
@@ -202,4 +177,20 @@ defmodule Pleroma.Config.DeprecationWarnings do
       warning_preface
     )
   end
+
+  @spec check_remote_ip_plug_name() :: :ok | nil
+  def check_remote_ip_plug_name do
+    warning_preface = """
+    !!!DEPRECATION WARNING!!!
+    Your config is using old namespace for RemoteIp Plug. Setting should work for now, but you are advised to change to new namespace to prevent possible issues later:
+    """
+
+    move_namespace_and_warn(
+      [
+        {Pleroma.Plugs.RemoteIp, Pleroma.Web.Plugs.RemoteIp,
+         "\n* `config :pleroma, Pleroma.Plugs.RemoteIp` is now `config :pleroma, Pleroma.Web.Plugs.RemoteIp`"}
+      ],
+      warning_preface
+    )
+  end
 end

lib/pleroma/config/oban.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Config.Oban do
   require Logger
 

lib/pleroma/conversation.ex

@@ -43,7 +43,7 @@ defmodule Pleroma.Conversation do
   def maybe_create_recipientships(participation, activity) do
     participation = Repo.preload(participation, :recipients)
 
-    if participation.recipients |> Enum.empty?() do
+    if Enum.empty?(participation.recipients) do
       recipients = User.get_all_by_ap_id(activity.recipients)
       RecipientShip.create(recipients, participation)
     end
@@ -69,10 +69,6 @@ defmodule Pleroma.Conversation do
         Enum.map(users, fn user ->
           invisible_conversation = Enum.any?(users, &User.blocks?(user, &1))
 
-          unless invisible_conversation do
-            User.increment_unread_conversation_count(conversation, user)
-          end
-
           opts = Keyword.put(opts, :invisible_conversation, invisible_conversation)
 
           {:ok, participation} =

lib/pleroma/conversation/participation.ex

@@ -63,21 +63,10 @@ defmodule Pleroma.Conversation.Participation do
     end
   end
 
-  def mark_as_read(participation) do
+  def mark_as_read(%__MODULE__{} = participation) do
-    __MODULE__
+    participation
-    |> where(id: ^participation.id)
+    |> change(read: true)
-    |> update(set: [read: true])
+    |> Repo.update()
-    |> select([p], p)
-    |> Repo.update_all([])
-    |> case do
-      {1, [participation]} ->
-        participation = Repo.preload(participation, :user)
-        User.set_unread_conversation_count(participation.user)
-        {:ok, participation}
-
-      error ->
-        error
-    end
   end
 
   def mark_all_as_read(%User{local: true} = user, %User{} = target_user) do
@@ -93,7 +82,6 @@ defmodule Pleroma.Conversation.Participation do
     |> update([p], set: [read: true])
     |> Repo.update_all([])
 
-    {:ok, user} = User.set_unread_conversation_count(user)
     {:ok, user, []}
   end
 
@@ -108,7 +96,6 @@ defmodule Pleroma.Conversation.Participation do
       |> select([p], p)
       |> Repo.update_all([])
 
-    {:ok, user} = User.set_unread_conversation_count(user)
     {:ok, user, participations}
   end
 
@@ -220,6 +207,12 @@ defmodule Pleroma.Conversation.Participation do
     {:ok, Repo.preload(participation, :recipients, force: true)}
   end
 
+  @spec unread_count(User.t()) :: integer()
+  def unread_count(%User{id: user_id}) do
+    from(q in __MODULE__, where: q.user_id == ^user_id and q.read == false)
+    |> Repo.aggregate(:count, :id)
+  end
+
   def unread_conversation_count_for_user(user) do
     from(p in __MODULE__,
       where: p.user_id == ^user.id,

lib/pleroma/docs/generator.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Docs.Generator do
   @callback process(keyword()) :: {:ok, String.t()}
 

lib/pleroma/docs/json.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Docs.JSON do
   @behaviour Pleroma.Docs.Generator
   @external_resource "config/description.exs"

lib/pleroma/docs/markdown.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Docs.Markdown do
   @behaviour Pleroma.Docs.Generator
 

lib/pleroma/emails/admin_email.ex

@@ -88,7 +88,7 @@ defmodule Pleroma.Emails.AdminEmail do
     html_body = """
     <p>New account for review: <a href="#{user_url(account)}">@#{account.nickname}</a></p>
     <blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote>
-    <a href="#{Pleroma.Web.base_url()}/pleroma/admin">Visit AdminFE</a>
+    <a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
     """
 
     new()

lib/pleroma/emails/user_email.ex

@@ -189,4 +189,30 @@ defmodule Pleroma.Emails.UserEmail do
 
     Router.Helpers.subscription_url(Endpoint, :unsubscribe, token)
   end
+
+  def backup_is_ready_email(backup, admin_user_id \\ nil) do
+    %{user: user} = Pleroma.Repo.preload(backup, :user)
+    download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
+
+    html_body =
+      if is_nil(admin_user_id) do
+        """
+        <p>You requested a full backup of your Pleroma account. It's ready for download:</p>
+        <p><a href="#{download_url}">#{download_url}</a></p>
+        """
+      else
+        admin = Pleroma.Repo.get(User, admin_user_id)
+
+        """
+        <p>Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:</p>
+        <p><a href="#{download_url}">#{download_url}</a></p>
+        """
+      end
+
+    new()
+    |> to(recipient(user))
+    |> from(sender())
+    |> subject("Your account archive is ready")
+    |> html_body(html_body)
+  end
 end

lib/pleroma/emoji/pack.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Emoji.Pack do
   @derive {Jason.Encoder, only: [:files, :pack, :files_count]}
   defstruct files: %{},
@@ -198,13 +202,13 @@ defmodule Pleroma.Emoji.Pack do
     end
   end
 
-  @spec list_remote(String.t()) :: {:ok, map()} | {:error, atom()}
+  @spec list_remote(keyword()) :: {:ok, map()} | {:error, atom()}
-  def list_remote(url) do
+  def list_remote(opts) do
-    uri = url |> String.trim() |> URI.parse()
+    uri = opts[:url] |> String.trim() |> URI.parse()
 
     with :ok <- validate_shareable_packs_available(uri) do
       uri
-      |> URI.merge("/api/pleroma/emoji/packs")
+      |> URI.merge("/api/pleroma/emoji/packs?page=#{opts[:page]}&page_size=#{opts[:page_size]}")
       |> http_get()
     end
   end
@@ -244,7 +248,8 @@ defmodule Pleroma.Emoji.Pack do
     uri = url |> String.trim() |> URI.parse()
 
     with :ok <- validate_shareable_packs_available(uri),
-         {:ok, remote_pack} <- uri |> URI.merge("/api/pleroma/emoji/packs/#{name}") |> http_get(),
+         {:ok, remote_pack} <-
+           uri |> URI.merge("/api/pleroma/emoji/pack?name=#{name}") |> http_get(),
          {:ok, %{sha: sha, url: url} = pack_info} <- fetch_pack_info(remote_pack, uri, name),
          {:ok, archive} <- download_archive(url, sha),
          pack <- copy_as(remote_pack, as || name),
@@ -523,7 +528,7 @@ defmodule Pleroma.Emoji.Pack do
   defp http_get(%URI{} = url), do: url |> to_string() |> http_get()
 
   defp http_get(url) do
-    with {:ok, %{body: body}} <- url |> Pleroma.HTTP.get() do
+    with {:ok, %{body: body}} <- Pleroma.HTTP.get(url, [], pool: :default) do
       Jason.decode(body)
     end
   end
@@ -572,7 +577,7 @@ defmodule Pleroma.Emoji.Pack do
         {:ok,
          %{
            sha: sha,
-           url: URI.merge(uri, "/api/pleroma/emoji/packs/#{name}/archive") |> to_string()
+           url: URI.merge(uri, "/api/pleroma/emoji/packs/archive?name=#{name}") |> to_string()
          }}
 
       %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) ->
@@ -589,7 +594,7 @@ defmodule Pleroma.Emoji.Pack do
   end
 
   defp download_archive(url, sha) do
-    with {:ok, %{body: archive}} <- Tesla.get(url) do
+    with {:ok, %{body: archive}} <- Pleroma.HTTP.get(url) do
       if Base.decode16!(sha) == :crypto.hash(:sha256, archive) do
         {:ok, archive}
       else
@@ -612,7 +617,7 @@ defmodule Pleroma.Emoji.Pack do
   end
 
   defp update_sha_and_save_metadata(pack, data) do
-    with {:ok, %{body: zip}} <- Tesla.get(data[:"fallback-src"]),
+    with {:ok, %{body: zip}} <- Pleroma.HTTP.get(data[:"fallback-src"]),
          :ok <- validate_has_all_files(pack, zip) do
       fallback_sha = :sha256 |> :crypto.hash(zip) |> Base.encode16()
 

lib/pleroma/gun/connection_pool.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Gun.ConnectionPool do
   @registry __MODULE__
 

lib/pleroma/gun/connection_pool/reclaimer.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
   use GenServer, restart: :temporary
 

lib/pleroma/gun/connection_pool/worker.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Gun.ConnectionPool.Worker do
   alias Pleroma.Gun
   use GenServer, restart: :temporary

lib/pleroma/gun/connection_pool/worker_supervisor.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
   @moduledoc "Supervisor for pool workers. Does not do anything except enforce max connection limit"
 

lib/pleroma/helpers/inet_helper.ex

@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.InetHelper do
+  def parse_address(ip) when is_tuple(ip) do
+    {:ok, ip}
+  end
+
+  def parse_address(ip) when is_binary(ip) do
+    ip
+    |> String.to_charlist()
+    |> parse_address()
+  end
+
+  def parse_address(ip) do
+    :inet.parse_address(ip)
+  end
+end

lib/pleroma/helpers/media_helper.ex

@@ -9,6 +9,18 @@ defmodule Pleroma.Helpers.MediaHelper do
 
   alias Pleroma.HTTP
 
+  require Logger
+
+  def missing_dependencies do
+    Enum.reduce([imagemagick: "convert", ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc ->
+      if Pleroma.Utils.command_available?(executable) do
+        acc
+      else
+        [sym | acc]
+      end
+    end)
+  end
+
   def image_resize(url, options) do
     with executable when is_binary(executable) <- System.find_executable("convert"),
          {:ok, args} <- prepare_image_resize_args(options),

lib/pleroma/http/adapter_helper/default.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.HTTP.AdapterHelper.Default do
   alias Pleroma.HTTP.AdapterHelper
 

lib/pleroma/http/adapter_helper/hackney.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.HTTP.AdapterHelper.Hackney do
   @behaviour Pleroma.HTTP.AdapterHelper
 

lib/pleroma/http/web_push.ex

@@ -0,0 +1,12 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.WebPush do
+  @moduledoc false
+
+  def post(url, payload, headers) do
+    list_headers = Map.to_list(headers)
+    Pleroma.HTTP.post(url, payload, list_headers)
+  end
+end

lib/pleroma/instances.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Instances do
   defdelegate reachable?(url_or_host), to: @adapter
   defdelegate set_reachable(url_or_host), to: @adapter
   defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter
+  defdelegate get_consistently_unreachable(), to: @adapter
 
   def set_consistently_unreachable(url_or_host),
     do: set_unreachable(url_or_host, reachability_datetime_threshold())

lib/pleroma/instances/instance.ex

@@ -119,6 +119,17 @@ defmodule Pleroma.Instances.Instance do
 
   def set_unreachable(_, _), do: {:error, nil}
 
+  def get_consistently_unreachable do
+    reachability_datetime_threshold = Instances.reachability_datetime_threshold()
+
+    from(i in Instance,
+      where: ^reachability_datetime_threshold > i.unreachable_since,
+      order_by: i.unreachable_since,
+      select: {i.host, i.unreachable_since}
+    )
+    |> Repo.all()
+  end
+
   defp parse_datetime(datetime) when is_binary(datetime) do
     NaiveDateTime.from_iso8601(datetime)
   end

lib/pleroma/jwt.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.JWT do
   use Joken.Config
 

lib/pleroma/mime.ex

@@ -1,120 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.MIME do
-  @moduledoc """
-  Returns the mime-type of a binary and optionally a normalized file-name.
-  """
-  @default "application/octet-stream"
-  @read_bytes 35
-
-  @spec file_mime_type(String.t(), String.t()) ::
-          {:ok, content_type :: String.t(), filename :: String.t()} | {:error, any()} | :error
-  def file_mime_type(path, filename) do
-    with {:ok, content_type} <- file_mime_type(path),
-         filename <- fix_extension(filename, content_type) do
-      {:ok, content_type, filename}
-    end
-  end
-
-  @spec file_mime_type(String.t()) :: {:ok, String.t()} | {:error, any()} | :error
-  def file_mime_type(filename) do
-    File.open(filename, [:read], fn f ->
-      check_mime_type(IO.binread(f, @read_bytes))
-    end)
-  end
-
-  def bin_mime_type(binary, filename) do
-    with {:ok, content_type} <- bin_mime_type(binary),
-         filename <- fix_extension(filename, content_type) do
-      {:ok, content_type, filename}
-    end
-  end
-
-  @spec bin_mime_type(binary()) :: {:ok, String.t()} | :error
-  def bin_mime_type(<<head::binary-size(@read_bytes), _::binary>>) do
-    {:ok, check_mime_type(head)}
-  end
-
-  def bin_mime_type(_), do: :error
-
-  def mime_type(<<_::binary>>), do: {:ok, @default}
-
-  defp fix_extension(filename, content_type) do
-    parts = String.split(filename, ".")
-
-    new_filename =
-      if length(parts) > 1 do
-        Enum.drop(parts, -1) |> Enum.join(".")
-      else
-        Enum.join(parts)
-      end
-
-    cond do
-      content_type == "application/octet-stream" ->
-        filename
-
-      ext = List.first(MIME.extensions(content_type)) ->
-        new_filename <> "." <> ext
-
-      true ->
-        Enum.join([new_filename, String.split(content_type, "/") |> List.last()], ".")
-    end
-  end
-
-  defp check_mime_type(<<0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, _::binary>>) do
-    "image/png"
-  end
-
-  defp check_mime_type(<<0x47, 0x49, 0x46, 0x38, _, 0x61, _::binary>>) do
-    "image/gif"
-  end
-
-  defp check_mime_type(<<0xFF, 0xD8, 0xFF, _::binary>>) do
-    "image/jpeg"
-  end
-
-  defp check_mime_type(<<0x1A, 0x45, 0xDF, 0xA3, _::binary>>) do
-    "video/webm"
-  end
-
-  defp check_mime_type(<<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70, _::binary>>) do
-    "video/mp4"
-  end
-
-  defp check_mime_type(<<0x49, 0x44, 0x33, _::binary>>) do
-    "audio/mpeg"
-  end
-
-  defp check_mime_type(<<255, 251, _, 68, 0, 0, 0, 0, _::binary>>) do
-    "audio/mpeg"
-  end
-
-  defp check_mime_type(
-         <<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00, _::size(160), 0x80, 0x74, 0x68, 0x65,
-           0x6F, 0x72, 0x61, _::binary>>
-       ) do
-    "video/ogg"
-  end
-
-  defp check_mime_type(<<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00, _::binary>>) do
-    "audio/ogg"
-  end
-
-  defp check_mime_type(<<"RIFF", _::binary-size(4), "WAVE", _::binary>>) do
-    "audio/wav"
-  end
-
-  defp check_mime_type(<<"RIFF", _::binary-size(4), "WEBP", _::binary>>) do
-    "image/webp"
-  end
-
-  defp check_mime_type(<<"RIFF", _::binary-size(4), "AVI.", _::binary>>) do
-    "video/avi"
-  end
-
-  defp check_mime_type(_) do
-    @default
-  end
-end

lib/pleroma/moderation_log.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.ModerationLog do
   use Ecto.Schema
 
@@ -651,6 +655,16 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} deleted chat message ##{subject_id}"
   end
 
+  def get_log_entry_message(%ModerationLog{
+        data: %{
+          "actor" => %{"nickname" => actor_nickname},
+          "action" => "create_backup",
+          "subject" => %{"nickname" => user_nickname}
+        }
+      }) do
+    "@#{actor_nickname} requested account backup for @#{user_nickname}"
+  end
+
   defp nicknames_to_string(nicknames) do
     nicknames
     |> Enum.map(&"@#{&1}")

lib/pleroma/plugs/remote_ip.ex

@@ -1,54 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Plugs.RemoteIp do
-  @moduledoc """
-  This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
-  """
-
-  import Plug.Conn
-
-  @behaviour Plug
-
-  @headers ~w[
-    x-forwarded-for
-  ]
-
-  # https://en.wikipedia.org/wiki/Localhost
-  # https://en.wikipedia.org/wiki/Private_network
-  @reserved ~w[
-    127.0.0.0/8
-    ::1/128
-    fc00::/7
-    10.0.0.0/8
-    172.16.0.0/12
-    192.168.0.0/16
-  ]
-
-  def init(_), do: nil
-
-  def call(%{remote_ip: original_remote_ip} = conn, _) do
-    config = Pleroma.Config.get(__MODULE__, [])
-
-    if Keyword.get(config, :enabled, false) do
-      %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts(config))
-      assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip)
-    else
-      conn
-    end
-  end
-
-  defp remote_ip_opts(config) do
-    headers = config |> Keyword.get(:headers, @headers) |> MapSet.new()
-    reserved = Keyword.get(config, :reserved, @reserved)
-
-    proxies =
-      config
-      |> Keyword.get(:proxies, [])
-      |> Enum.concat(reserved)
-      |> Enum.map(&InetCidr.parse/1)
-
-    {headers, proxies}
-  end
-end

lib/pleroma/telemetry/logger.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Telemetry.Logger do
   @moduledoc "Transforms Pleroma telemetry events to logs"
 

lib/pleroma/tests/auth_test_controller.ex

@@ -8,9 +8,9 @@ defmodule Pleroma.Tests.AuthTestController do
 
   use Pleroma.Web, :controller
 
-  alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug
-  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.User
+  alias Pleroma.Web.Plugs.EnsurePublicOrAuthenticatedPlug
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   # Serves only with proper OAuth token (:api and :authenticated_api)
   # Skipping EnsurePublicOrAuthenticatedPlug has no effect in this case

lib/pleroma/upload.ex

@@ -66,6 +66,7 @@ defmodule Pleroma.Upload do
   end
 
   @spec store(source, options :: [option()]) :: {:ok, Map.t()} | {:error, any()}
+  @doc "Store a file. If using a `Plug.Upload{}` as the source, be sure to use `Majic.Plug` to ensure its content_type and filename is correct."
   def store(upload, opts \\ []) do
     opts = get_opts(opts)
 
@@ -139,14 +140,13 @@ defmodule Pleroma.Upload do
   end
 
   defp prepare_upload(%Plug.Upload{} = file, opts) do
-    with :ok <- check_file_size(file.path, opts.size_limit),
+    with :ok <- check_file_size(file.path, opts.size_limit) do
-         {:ok, content_type, name} <- Pleroma.MIME.file_mime_type(file.path, file.filename) do
       {:ok,
        %__MODULE__{
          id: UUID.generate(),
-         name: name,
+         name: file.filename,
          tempfile: file.path,
-         content_type: content_type
+         content_type: file.content_type
        }}
     end
   end
@@ -154,16 +154,17 @@ defmodule Pleroma.Upload do
   defp prepare_upload(%{img: "data:image/" <> image_data}, opts) do
     parsed = Regex.named_captures(~r/(?<filetype>jpeg|png|gif);base64,(?<data>.*)/, image_data)
     data = Base.decode64!(parsed["data"], ignore: :whitespace)
-    hash = String.downcase(Base.encode16(:crypto.hash(:sha256, data)))
+    hash = Base.encode16(:crypto.hash(:sha256, data), lower: true)
 
     with :ok <- check_binary_size(data, opts.size_limit),
          tmp_path <- tempfile_for_image(data),
-         {:ok, content_type, name} <-
+         {:ok, %{mime_type: content_type}} <-
-           Pleroma.MIME.bin_mime_type(data, hash <> "." <> parsed["filetype"]) do
+           Majic.perform({:bytes, data}, pool: Pleroma.MajicPool),
+         [ext | _] <- MIME.extensions(content_type) do
       {:ok,
        %__MODULE__{
          id: UUID.generate(),
-         name: name,
+         name: hash <> "." <> ext,
          tempfile: tmp_path,
          content_type: content_type
        }}
@@ -172,7 +173,7 @@ defmodule Pleroma.Upload do
 
   # For Mix.Tasks.MigrateLocalUploads
   defp prepare_upload(%__MODULE__{tempfile: path} = upload, _opts) do
-    with {:ok, content_type} <- Pleroma.MIME.file_mime_type(path) do
+    with {:ok, %{mime_type: content_type}} <- Majic.perform(path, pool: Pleroma.MajicPool) do
       {:ok, %__MODULE__{upload | content_type: content_type}}
     end
   end

lib/pleroma/uploaders/uploader.ex

@@ -12,7 +12,7 @@ defmodule Pleroma.Uploaders.Uploader do
   @doc """
   Instructs how to get the file from the backend.
 
-  Used by `Pleroma.Plugs.UploadedMedia`.
+  Used by `Pleroma.Web.Plugs.UploadedMedia`.
   """
   @type get_method :: {:static_dir, directory :: String.t()} | {:url, url :: String.t()}
   @callback get_file(file :: String.t()) :: {:ok, get_method()}

lib/pleroma/user.ex

@@ -107,7 +107,7 @@ defmodule Pleroma.User do
     field(:note_count, :integer, default: 0)
     field(:follower_count, :integer, default: 0)
     field(:following_count, :integer, default: 0)
-    field(:locked, :boolean, default: false)
+    field(:is_locked, :boolean, default: false)
     field(:confirmation_pending, :boolean, default: false)
     field(:password_reset_pending, :boolean, default: false)
     field(:approval_pending, :boolean, default: false)
@@ -128,7 +128,6 @@ defmodule Pleroma.User do
     field(:hide_followers, :boolean, default: false)
     field(:hide_follows, :boolean, default: false)
     field(:hide_favorites, :boolean, default: true)
-    field(:unread_conversation_count, :integer, default: 0)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:email_notifications, :map, default: %{"digest" => false})
     field(:mascot, :map, default: nil)
@@ -136,7 +135,7 @@ defmodule Pleroma.User do
     field(:pleroma_settings_store, :map, default: %{})
     field(:fields, {:array, :map}, default: [])
     field(:raw_fields, {:array, :map}, default: [])
-    field(:discoverable, :boolean, default: false)
+    field(:is_discoverable, :boolean, default: false)
     field(:invisible, :boolean, default: false)
     field(:allow_following_move, :boolean, default: true)
     field(:skip_thread_containment, :boolean, default: false)
@@ -426,7 +425,6 @@ defmodule Pleroma.User do
       params,
       [
         :bio,
-        :name,
         :emoji,
         :ap_id,
         :inbox,
@@ -436,7 +434,7 @@ defmodule Pleroma.User do
         :avatar,
         :ap_enabled,
         :banner,
-        :locked,
+        :is_locked,
         :last_refreshed_at,
         :uri,
         :follower_address,
@@ -448,14 +446,16 @@ defmodule Pleroma.User do
         :follower_count,
         :fields,
         :following_count,
-        :discoverable,
+        :is_discoverable,
         :invisible,
         :actor_type,
         :also_known_as,
         :accepts_chat_messages
       ]
     )
-    |> validate_required([:name, :ap_id])
+    |> cast(params, [:name], empty_values: [])
+    |> validate_required([:ap_id])
+    |> validate_required([:name], trim: false)
     |> unique_constraint(:nickname)
     |> validate_format(:nickname, @email_regex)
     |> validate_length(:bio, max: bio_limit)
@@ -479,7 +479,7 @@ defmodule Pleroma.User do
         :public_key,
         :inbox,
         :shared_inbox,
-        :locked,
+        :is_locked,
         :no_rich_text,
         :default_scope,
         :banner,
@@ -495,7 +495,7 @@ defmodule Pleroma.User do
         :fields,
         :raw_fields,
         :pleroma_settings_store,
-        :discoverable,
+        :is_discoverable,
         :actor_type,
         :also_known_as,
         :accepts_chat_messages
@@ -765,6 +765,16 @@ defmodule Pleroma.User do
     follow_all(user, autofollowed_users)
   end
 
+  defp autofollowing_users(user) do
+    candidates = Config.get([:instance, :autofollowing_nicknames])
+
+    User.Query.build(%{nickname: candidates, local: true, deactivated: false})
+    |> Repo.all()
+    |> Enum.each(&follow(&1, user, :follow_accept))
+
+    {:ok, :success}
+  end
+
   @doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)"
   def register(%Ecto.Changeset{} = changeset) do
     with {:ok, user} <- Repo.insert(changeset) do
@@ -774,6 +784,7 @@ defmodule Pleroma.User do
 
   def post_register_action(%User{} = user) do
     with {:ok, user} <- autofollow_users(user),
+         {:ok, _} <- autofollowing_users(user),
          {:ok, user} <- set_cache(user),
          {:ok, _} <- send_welcome_email(user),
          {:ok, _} <- send_welcome_message(user),
@@ -813,7 +824,8 @@ defmodule Pleroma.User do
   def send_welcome_email(_), do: {:ok, :noop}
 
   @spec try_send_confirmation_email(User.t()) :: {:ok, :enqueued | :noop}
-  def try_send_confirmation_email(%User{confirmation_pending: true} = user) do
+  def try_send_confirmation_email(%User{confirmation_pending: true, email: email} = user)
+      when is_binary(email) do
     if Config.get([:instance, :account_activation_required]) do
       send_confirmation_email(user)
       {:ok, :enqueued}
@@ -846,7 +858,7 @@ defmodule Pleroma.User do
   @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t()} | {:error, String.t()}
 
   # "Locked" (self-locked) users demand explicit authorization of follow requests
-  def maybe_direct_follow(%User{} = follower, %User{local: true, locked: true} = followed) do
+  def maybe_direct_follow(%User{} = follower, %User{local: true, is_locked: true} = followed) do
     follow(follower, followed, :follow_pending)
   end
 
@@ -914,9 +926,7 @@ defmodule Pleroma.User do
         FollowingRelationship.unfollow(follower, followed)
         {:ok, followed} = update_follower_count(followed)
 
-        {:ok, follower} =
+        {:ok, follower} = update_following_count(follower)
-          follower
-          |> update_following_count()
 
         {:ok, follower, followed}
 
@@ -955,7 +965,7 @@ defmodule Pleroma.User do
   end
 
   def locked?(%User{} = user) do
-    user.locked || false
+    user.is_locked || false
   end
 
   def get_by_id(id) do
@@ -1294,47 +1304,6 @@ defmodule Pleroma.User do
     |> update_and_set_cache()
   end
 
-  def set_unread_conversation_count(%User{local: true} = user) do
-    unread_query = Participation.unread_conversation_count_for_user(user)
-
-    User
-    |> join(:inner, [u], p in subquery(unread_query))
-    |> update([u, p],
-      set: [unread_conversation_count: p.count]
-    )
-    |> where([u], u.id == ^user.id)
-    |> select([u], u)
-    |> Repo.update_all([])
-    |> case do
-      {1, [user]} -> set_cache(user)
-      _ -> {:error, user}
-    end
-  end
-
-  def set_unread_conversation_count(user), do: {:ok, user}
-
-  def increment_unread_conversation_count(conversation, %User{local: true} = user) do
-    unread_query =
-      Participation.unread_conversation_count_for_user(user)
-      |> where([p], p.conversation_id == ^conversation.id)
-
-    User
-    |> join(:inner, [u], p in subquery(unread_query))
-    |> update([u, p],
-      inc: [unread_conversation_count: 1]
-    )
-    |> where([u], u.id == ^user.id)
-    |> where([u, p], p.count == 0)
-    |> select([u], u)
-    |> Repo.update_all([])
-    |> case do
-      {1, [user]} -> set_cache(user)
-      _ -> {:error, user}
-    end
-  end
-
-  def increment_unread_conversation_count(_, user), do: {:ok, user}
-
   @spec get_users_from_set([String.t()], keyword()) :: [User.t()]
   def get_users_from_set(ap_ids, opts \\ []) do
     local_only = Keyword.get(opts, :local_only, true)
@@ -1636,7 +1605,7 @@ defmodule Pleroma.User do
       note_count: 0,
       follower_count: 0,
       following_count: 0,
-      locked: false,
+      is_locked: false,
       confirmation_pending: false,
       password_reset_pending: false,
       approval_pending: false,
@@ -1653,7 +1622,7 @@ defmodule Pleroma.User do
       pleroma_settings_store: %{},
       fields: [],
       raw_fields: [],
-      discoverable: false,
+      is_discoverable: false,
       also_known_as: []
     })
   end
@@ -2105,6 +2074,13 @@ defmodule Pleroma.User do
     Enum.map(users, &toggle_confirmation/1)
   end
 
+  @spec need_confirmation(User.t(), boolean()) :: {:ok, User.t()} | {:error, Changeset.t()}
+  def need_confirmation(%User{} = user, bool) do
+    user
+    |> confirmation_changeset(need_confirmation: bool)
+    |> update_and_set_cache()
+  end
+
   def get_mascot(%{mascot: %{} = mascot}) when not is_nil(mascot) do
     mascot
   end
@@ -2319,7 +2295,9 @@ defmodule Pleroma.User do
 
     # if pinned activity was scheduled for deletion, we reschedule it for deletion
     if data["expires_at"] do
-      {:ok, expires_at, _} = DateTime.from_iso8601(data["expires_at"])
+      # MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation
+      {:ok, expires_at} =
+        data["expires_at"] |> Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast()
 
       Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
         activity_id: id,

lib/pleroma/user/backup.ex

@@ -0,0 +1,258 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.Backup do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+  import Ecto.Query
+  import Pleroma.Web.Gettext
+
+  require Pleroma.Constants
+
+  alias Pleroma.Activity
+  alias Pleroma.Bookmark
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.ActivityPub.UserView
+  alias Pleroma.Workers.BackupWorker
+
+  schema "backups" do
+    field(:content_type, :string)
+    field(:file_name, :string)
+    field(:file_size, :integer, default: 0)
+    field(:processed, :boolean, default: false)
+
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
+
+    timestamps()
+  end
+
+  def create(user, admin_id \\ nil) do
+    with :ok <- validate_email_enabled(),
+         :ok <- validate_user_email(user),
+         :ok <- validate_limit(user, admin_id),
+         {:ok, backup} <- user |> new() |> Repo.insert() do
+      BackupWorker.process(backup, admin_id)
+    end
+  end
+
+  def new(user) do
+    rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+    datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
+    name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
+
+    %__MODULE__{
+      user_id: user.id,
+      content_type: "application/zip",
+      file_name: name
+    }
+  end
+
+  def delete(backup) do
+    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+
+    with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do
+      Repo.delete(backup)
+    end
+  end
+
+  defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok
+
+  defp validate_limit(user, nil) do
+    case get_last(user.id) do
+      %__MODULE__{inserted_at: inserted_at} ->
+        days = Pleroma.Config.get([__MODULE__, :limit_days])
+        diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
+
+        if diff > days do
+          :ok
+        else
+          {:error,
+           dngettext(
+             "errors",
+             "Last export was less than a day ago",
+             "Last export was less than %{days} days ago",
+             days,
+             days: days
+           )}
+        end
+
+      nil ->
+        :ok
+    end
+  end
+
+  defp validate_email_enabled do
+    if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
+      :ok
+    else
+      {:error, dgettext("errors", "Backups require enabled email")}
+    end
+  end
+
+  defp validate_user_email(%User{email: nil}) do
+    {:error, dgettext("errors", "Email is required")}
+  end
+
+  defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok
+
+  def get_last(user_id) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> order_by(desc: :id)
+    |> limit(1)
+    |> Repo.one()
+  end
+
+  def list(%User{id: user_id}) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> order_by(desc: :id)
+    |> Repo.all()
+  end
+
+  def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> where([b], b.id != ^latest_id)
+    |> Repo.all()
+    |> Enum.each(&BackupWorker.delete/1)
+  end
+
+  def get(id), do: Repo.get(__MODULE__, id)
+
+  def process(%__MODULE__{} = backup) do
+    with {:ok, zip_file} <- export(backup),
+         {:ok, %{size: size}} <- File.stat(zip_file),
+         {:ok, _upload} <- upload(backup, zip_file) do
+      backup
+      |> cast(%{file_size: size, processed: true}, [:file_size, :processed])
+      |> Repo.update()
+    end
+  end
+
+  @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
+  def export(%__MODULE__{} = backup) do
+    backup = Repo.preload(backup, :user)
+    name = String.trim_trailing(backup.file_name, ".zip")
+    dir = dir(name)
+
+    with :ok <- File.mkdir(dir),
+         :ok <- actor(dir, backup.user),
+         :ok <- statuses(dir, backup.user),
+         :ok <- likes(dir, backup.user),
+         :ok <- bookmarks(dir, backup.user),
+         {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
+         {:ok, _} <- File.rm_rf(dir) do
+      {:ok, to_string(zip_path)}
+    end
+  end
+
+  def dir(name) do
+    dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!()
+    Path.join(dir, name)
+  end
+
+  def upload(%__MODULE__{} = backup, zip_path) do
+    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+
+    upload = %Pleroma.Upload{
+      name: backup.file_name,
+      tempfile: zip_path,
+      content_type: backup.content_type,
+      path: Path.join("backups", backup.file_name)
+    }
+
+    with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
+         :ok <- File.rm(zip_path) do
+      {:ok, upload}
+    end
+  end
+
+  defp actor(dir, user) do
+    with {:ok, json} <-
+           UserView.render("user.json", %{user: user})
+           |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
+           |> Jason.encode() do
+      File.write(Path.join(dir, "actor.json"), json)
+    end
+  end
+
+  defp write_header(file, name) do
+    IO.write(
+      file,
+      """
+      {
+        "@context": "https://www.w3.org/ns/activitystreams",
+        "id": "#{name}.json",
+        "type": "OrderedCollection",
+        "orderedItems": [
+
+      """
+    )
+  end
+
+  defp write(query, dir, name, fun) do
+    path = Path.join(dir, "#{name}.json")
+
+    with {:ok, file} <- File.open(path, [:write, :utf8]),
+         :ok <- write_header(file, name) do
+      total =
+        query
+        |> Pleroma.Repo.chunk_stream(100)
+        |> Enum.reduce(0, fn i, acc ->
+          with {:ok, data} <- fun.(i),
+               {:ok, str} <- Jason.encode(data),
+               :ok <- IO.write(file, str <> ",\n") do
+            acc + 1
+          else
+            _ -> acc
+          end
+        end)
+
+      with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n  \"totalItems\": #{total}}") do
+        File.close(file)
+      end
+    end
+  end
+
+  defp bookmarks(dir, %{id: user_id} = _user) do
+    Bookmark
+    |> where(user_id: ^user_id)
+    |> join(:inner, [b], activity in assoc(b, :activity))
+    |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
+    |> write(dir, "bookmarks", fn a -> {:ok, a.object} end)
+  end
+
+  defp likes(dir, user) do
+    user.ap_id
+    |> Activity.Queries.by_actor()
+    |> Activity.Queries.by_type("Like")
+    |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
+    |> write(dir, "likes", fn a -> {:ok, a.object} end)
+  end
+
+  defp statuses(dir, user) do
+    opts =
+      %{}
+      |> Map.put(:type, ["Create", "Announce"])
+      |> Map.put(:actor_id, user.ap_id)
+
+    [
+      [Pleroma.Constants.as_public(), user.ap_id],
+      User.following(user),
+      Pleroma.List.memberships(user)
+    ]
+    |> Enum.concat()
+    |> ActivityPub.fetch_activities_query(opts)
+    |> write(dir, "outbox", fn a ->
+      with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
+        {:ok, Map.delete(activity, "@context")}
+      end
+    end)
+  end
+end

lib/pleroma/user/query.ex

@@ -43,10 +43,12 @@ defmodule Pleroma.User.Query do
             active: boolean(),
             deactivated: boolean(),
             need_approval: boolean(),
+            unconfirmed: boolean(),
             is_admin: boolean(),
             is_moderator: boolean(),
             super_users: boolean(),
             invisible: boolean(),
+            internal: boolean(),
             followers: User.t(),
             friends: User.t(),
             recipients_from_activity: [String.t()],
@@ -54,7 +56,8 @@ defmodule Pleroma.User.Query do
             ap_id: [String.t()],
             order_by: term(),
             select: term(),
-            limit: pos_integer()
+            limit: pos_integer(),
+            actor_types: [String.t()]
           }
           | map()
 
@@ -80,7 +83,9 @@ defmodule Pleroma.User.Query do
   end
 
   defp prepare_query(query, criteria) do
-    Enum.reduce(criteria, query, &compose_query/2)
+    criteria
+    |> Map.put_new(:internal, false)
+    |> Enum.reduce(query, &compose_query/2)
   end
 
   defp compose_query({key, value}, query)
@@ -107,12 +112,16 @@ defmodule Pleroma.User.Query do
     where(query, [u], fragment("? && ?", u.tags, ^tags))
   end
 
-  defp compose_query({:is_admin, _}, query) do
+  defp compose_query({:is_admin, bool}, query) do
-    where(query, [u], u.is_admin)
+    where(query, [u], u.is_admin == ^bool)
   end
 
-  defp compose_query({:is_moderator, _}, query) do
+  defp compose_query({:actor_types, actor_types}, query) when is_list(actor_types) do
-    where(query, [u], u.is_moderator)
+    where(query, [u], u.actor_type in ^actor_types)
+  end
+
+  defp compose_query({:is_moderator, bool}, query) do
+    where(query, [u], u.is_moderator == ^bool)
   end
 
   defp compose_query({:super_users, _}, query) do
@@ -129,14 +138,12 @@ defmodule Pleroma.User.Query do
 
   defp compose_query({:active, _}, query) do
     User.restrict_deactivated(query)
-    |> where([u], not is_nil(u.nickname))
     |> where([u], u.approval_pending == false)
   end
 
   defp compose_query({:legacy_active, _}, query) do
     query
     |> where([u], fragment("not (?->'deactivated' @> 'true')", u.info))
-    |> where([u], not is_nil(u.nickname))
   end
 
   defp compose_query({:deactivated, false}, query) do
@@ -145,13 +152,20 @@ defmodule Pleroma.User.Query do
 
   defp compose_query({:deactivated, true}, query) do
     where(query, [u], u.deactivated == ^true)
-    |> where([u], not is_nil(u.nickname))
+  end
+
+  defp compose_query({:confirmation_pending, bool}, query) do
+    where(query, [u], u.confirmation_pending == ^bool)
   end
 
   defp compose_query({:need_approval, _}, query) do
     where(query, [u], u.approval_pending)
   end
 
+  defp compose_query({:unconfirmed, _}, query) do
+    where(query, [u], u.confirmation_pending)
+  end
+
   defp compose_query({:followers, %User{id: id}}, query) do
     query
     |> where([u], u.id != ^id)
@@ -199,10 +213,15 @@ defmodule Pleroma.User.Query do
     limit(query, ^limit)
   end
 
+  defp compose_query({:internal, false}, query) do
+    query
+    |> where([u], not is_nil(u.nickname))
+    |> where([u], not like(u.nickname, "internal.%"))
+  end
+
   defp compose_query(_unsupported_param, query), do: query
 
   defp location_query(query, local) do
     where(query, [u], u.local == ^local)
-    |> where([u], not is_nil(u.nickname))
   end
 end

lib/pleroma/user/search.ex

@@ -3,8 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.User.Search do
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators.Uri, as: UriType
   alias Pleroma.Pagination
   alias Pleroma.User
+
   import Ecto.Query
 
   @limit 20
@@ -19,16 +21,47 @@ defmodule Pleroma.User.Search do
 
     query_string = format_query(query_string)
 
-    maybe_resolve(resolve, for_user, query_string)
+    # If this returns anything, it should bounce to the top
+    maybe_resolved = maybe_resolve(resolve, for_user, query_string)
+
+    top_user_ids =
+      []
+      |> maybe_add_resolved(maybe_resolved)
+      |> maybe_add_ap_id_match(query_string)
+      |> maybe_add_uri_match(query_string)
 
     results =
       query_string
-      |> search_query(for_user, following)
+      |> search_query(for_user, following, top_user_ids)
       |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset)
 
     results
   end
 
+  defp maybe_add_resolved(list, {:ok, %User{} = user}) do
+    [user.id | list]
+  end
+
+  defp maybe_add_resolved(list, _), do: list
+
+  defp maybe_add_ap_id_match(list, query) do
+    if user = User.get_cached_by_ap_id(query) do
+      [user.id | list]
+    else
+      list
+    end
+  end
+
+  defp maybe_add_uri_match(list, query) do
+    with {:ok, query} <- UriType.cast(query),
+         q = from(u in User, where: u.uri == ^query, select: u.id),
+         users = Pleroma.Repo.all(q) do
+      users ++ list
+    else
+      _ -> list
+    end
+  end
+
   defp format_query(query_string) do
     # Strip the beginning @ off if there is a query
     query_string = String.trim_leading(query_string, "@")
@@ -47,7 +80,7 @@ defmodule Pleroma.User.Search do
     end
   end
 
-  defp search_query(query_string, for_user, following) do
+  defp search_query(query_string, for_user, following, top_user_ids) do
     for_user
     |> base_query(following)
     |> filter_blocked_user(for_user)
@@ -56,13 +89,20 @@ defmodule Pleroma.User.Search do
     |> filter_internal_users()
     |> filter_blocked_domains(for_user)
     |> fts_search(query_string)
+    |> select_top_users(top_user_ids)
     |> trigram_rank(query_string)
-    |> boost_search_rank(for_user)
+    |> boost_search_rank(for_user, top_user_ids)
     |> subquery()
     |> order_by(desc: :search_rank)
     |> maybe_restrict_local(for_user)
   end
 
+  defp select_top_users(query, top_user_ids) do
+    from(u in query,
+      or_where: u.id in ^top_user_ids
+    )
+  end
+
   defp fts_search(query, query_string) do
     query_string = to_tsquery(query_string)
 
@@ -124,7 +164,7 @@ defmodule Pleroma.User.Search do
   end
 
   defp filter_discoverable_users(query) do
-    from(q in query, where: q.discoverable == true)
+    from(q in query, where: q.is_discoverable == true)
   end
 
   defp filter_internal_users(query) do
@@ -180,7 +220,7 @@ defmodule Pleroma.User.Search do
 
   defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
 
-  defp boost_search_rank(query, %User{} = for_user) do
+  defp boost_search_rank(query, %User{} = for_user, top_user_ids) do
     friends_ids = User.get_friends_ids(for_user)
     followers_ids = User.get_followers_ids(for_user)
 
@@ -192,6 +232,7 @@ defmodule Pleroma.User.Search do
              CASE WHEN (?) THEN (?) * 1.5
              WHEN (?) THEN (?) * 1.3
              WHEN (?) THEN (?) * 1.1
+             WHEN (?) THEN 9001
              ELSE (?) END
             """,
             u.id in ^friends_ids and u.id in ^followers_ids,
@@ -200,11 +241,26 @@ defmodule Pleroma.User.Search do
             u.search_rank,
             u.id in ^followers_ids,
             u.search_rank,
+            u.id in ^top_user_ids,
             u.search_rank
           )
       }
     )
   end
 
-  defp boost_search_rank(query, _for_user), do: query
+  defp boost_search_rank(query, _for_user, top_user_ids) do
+    from(u in subquery(query),
+      select_merge: %{
+        search_rank:
+          fragment(
+            """
+             CASE WHEN (?) THEN 9001
+             ELSE (?) END
+            """,
+            u.id in ^top_user_ids,
+            u.search_rank
+          )
+      }
+    )
+  end
 end

lib/pleroma/web.ex

@@ -2,11 +2,6 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.Plug do
-  # Substitute for `call/2` which is defined with `use Pleroma.Web, :plug`
-  @callback perform(Plug.Conn.t(), Plug.opts()) :: Plug.Conn.t()
-end
-
 defmodule Pleroma.Web do
   @moduledoc """
   A module that keeps using definitions for controllers,
@@ -25,12 +20,12 @@ defmodule Pleroma.Web do
   below.
   """
 
-  alias Pleroma.Plugs.EnsureAuthenticatedPlug
+  alias Pleroma.Web.Plugs.EnsureAuthenticatedPlug
-  alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug
+  alias Pleroma.Web.Plugs.EnsurePublicOrAuthenticatedPlug
-  alias Pleroma.Plugs.ExpectAuthenticatedCheckPlug
+  alias Pleroma.Web.Plugs.ExpectAuthenticatedCheckPlug
-  alias Pleroma.Plugs.ExpectPublicOrAuthenticatedCheckPlug
+  alias Pleroma.Web.Plugs.ExpectPublicOrAuthenticatedCheckPlug
-  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
-  alias Pleroma.Plugs.PlugHelper
+  alias Pleroma.Web.Plugs.PlugHelper
 
   def controller do
     quote do
@@ -177,7 +172,7 @@ defmodule Pleroma.Web do
   def channel do
     quote do
       # credo:disable-for-next-line Credo.Check.Consistency.MultiAliasImportRequireUse
-      use Phoenix.Channel
+      import Phoenix.Channel
       import Pleroma.Web.Gettext
     end
   end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -790,7 +790,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       [activity, object] in query,
       where:
         fragment(
-          "?->>'inReplyTo' is null OR ? && array_remove(?, ?) OR ? = ?",
+          """
+          ?->>'type' != 'Create'     -- This isn't a Create      
+          OR ?->>'inReplyTo' is null -- this isn't a reply
+          OR ? && array_remove(?, ?) -- The recipient is us or one of our friends, 
+                                     -- unless they are the author (because authors 
+                                     -- are also part of the recipients). This leads
+                                     -- to a bug that self-replies by friends won't
+                                     -- show up.
+          OR ? = ?                   -- The actor is us
+          """,
+          activity.data,
           object.data,
           ^[user.ap_id | User.get_cached_user_friends_ap_ids(user)],
           activity.recipients,
@@ -817,7 +827,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     query =
       from([activity] in query,
         where: fragment("not (? = ANY(?))", activity.actor, ^mutes),
-        where: fragment("not (?->'to' \\?| ?)", activity.data, ^mutes)
+        where:
+          fragment(
+            "not (?->'to' \\?| ?) or ? = ?",
+            activity.data,
+            ^mutes,
+            activity.actor,
+            ^user.ap_id
+          )
       )
 
     unless opts[:skip_preload] do
@@ -920,16 +937,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_muted_reblogs(query, _), do: query
 
-  defp restrict_instance(query, %{instance: instance}) do
+  defp restrict_instance(query, %{instance: instance}) when is_binary(instance) do
-    users =
     from(
-        u in User,
+      activity in query,
-        select: u.ap_id,
+      where: fragment("split_part(actor::text, '/'::text, 3) = ?", ^instance)
-        where: fragment("? LIKE ?", u.nickname, ^"%@#{instance}")
     )
-      |> Repo.all()
-
-    from(activity in query, where: activity.actor in ^users)
   end
 
   defp restrict_instance(query, _), do: query
@@ -1218,11 +1230,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         {String.trim(name, ":"), url}
       end)
 
-    locked = data["manuallyApprovesFollowers"] || false
+    is_locked = data["manuallyApprovesFollowers"] || false
     capabilities = data["capabilities"] || %{}
     accepts_chat_messages = capabilities["acceptsChatMessages"]
     data = Transmogrifier.maybe_fix_user_object(data)
-    discoverable = data["discoverable"] || false
+    is_discoverable = data["discoverable"] || false
     invisible = data["invisible"] || false
     actor_type = data["type"] || "Person"
 
@@ -1247,8 +1259,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       banner: banner,
       fields: fields,
       emoji: emojis,
-      locked: locked,
+      is_locked: is_locked,
-      discoverable: discoverable,
+      is_discoverable: is_discoverable,
       invisible: invisible,
       avatar: avatar,
       name: data["name"],
@@ -1361,6 +1373,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          {:ok, data} <- user_data_from_user_object(data) do
       {:ok, maybe_update_follow_information(data)}
     else
+      # If this has been deleted, only log a debug and not an error
       {:error, "Object has been deleted" = e} ->
         Logger.debug("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
         {:error, e}