Merge remote-tracking branch 'upstream/develop' into earmark

.formatter.exs

@@ -1,3 +1,3 @@
 [
-  inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}", "priv/repo/migrations/*.exs", "priv/scrubbers/*.ex"]
+  inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}", "priv/repo/migrations/*.exs", "priv/repo/optional_migrations/**/*.exs", "priv/scrubbers/*.ex"]
 ]

.gitignore

@@ -3,6 +3,7 @@
 /db
 /deps
 /*.ez
+/test/instance
 /test/uploads
 /.elixir_ls
 /test/fixtures/DSCN0010_tmp.jpg
@@ -31,7 +32,7 @@ erl_crash.dump
 
 
 # Database setup file, some may forget to delete it
-/config/setup_db.psql
+/config/setup_db*.psql
 
 .DS_Store
 .env
@@ -52,3 +53,7 @@ pleroma.iml
 
 # asdf
 .tool-versions
+
+# Editor temp files
+/*~
+/*#

.gitlab-ci.yml

@@ -8,7 +8,9 @@ variables: &global_variables
   MIX_ENV: test
 
 cache: &global_cache_policy
-  key: ${CI_COMMIT_REF_SLUG}
+  key:
+    files:
+      - mix.lock
   paths:
     - deps
     - _build
@@ -22,18 +24,30 @@ stages:
   - docker
 
 before_script:
+  - rm -rf _build/*/lib/pleroma
   - apt-get update && apt-get install -y cmake
   - mix local.hex --force
   - mix local.rebar --force
+  - mix deps.get
   - apt-get -qq update
   - apt-get install -y libmagic-dev
 
+after_script:
+  - rm -rf _build/*/lib/pleroma
+
 build:
   stage: build
   script:
-  - mix deps.get
   - mix compile --force
 
+spec-build:
+  stage: test
+  artifacts:
+    paths:
+    - spec.json
+  script:
+  - mix pleroma.openapi_spec spec.json
+
 benchmark:
   stage: benchmark
   when: manual
@@ -44,7 +58,6 @@ benchmark:
     alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
-    - mix deps.get
     - mix ecto.create
     - mix ecto.migrate
     - mix pleroma.load_testing
@@ -62,7 +75,6 @@ unit-testing:
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
     - apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg
-    - mix deps.get
     - mix ecto.create
     - mix ecto.migrate
     - mix coveralls --preload-modules
@@ -96,7 +108,6 @@ unit-testing-rum:
     RUM_ENABLED: "true"
   script:
     - apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg
-    - mix deps.get
     - mix ecto.create
     - mix ecto.migrate
     - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
@@ -112,7 +123,6 @@ analysis:
   stage: test
   cache: *testing_cache_policy
   script:
-    - mix deps.get
     - mix credo --strict --only=warnings,todo,fixme,consistency,readability
 
 docs-deploy:
@@ -155,6 +165,20 @@ review_app:
     - (ssh -t dokku@pleroma.online -- certs:add "$CI_ENVIRONMENT_SLUG" /home/dokku/server.crt /home/dokku/server.key) || true
     - git push -f dokku@pleroma.online:$CI_ENVIRONMENT_SLUG $CI_COMMIT_SHA:refs/heads/master
 
+spec-deploy:
+  stage: deploy
+  artifacts:
+    paths:
+    - spec.json
+  only:
+    - develop@pleroma/pleroma
+  image: alpine:latest
+  before_script:
+    - apk add curl
+  script:
+    - curl -X POST -F"token=$API_DOCS_PIPELINE_TRIGGER" -F'ref=master' -F"variables[BRANCH]=$CI_COMMIT_REF_NAME" -F"variables[JOB_REF]=$CI_JOB_ID" https://git.pleroma.social/api/v4/projects/1130/trigger/pipeline
+
+
 stop_review_app:
   image: alpine:3.9
   stage: deploy
@@ -349,3 +373,26 @@ docker-release:
     - dind
   only:
     - /^release/.*$/@pleroma/pleroma
+
+docker-adhoc:
+  stage: docker
+  image: docker:latest
+  cache: {}
+  dependencies: []
+  variables: *docker-variables
+  before_script: *before-docker
+  allow_failure: true
+  script:
+  script:
+    - mkdir -p /root/.docker/cli-plugins
+    - wget "${DOCKER_BUILDX_URL}" -O ~/.docker/cli-plugins/docker-buildx
+    - echo "${DOCKER_BUILDX_HASH}  /root/.docker/cli-plugins/docker-buildx" | sha1sum -c
+    - chmod +x ~/.docker/cli-plugins/docker-buildx
+    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+    - docker buildx create --name mbuilder --driver docker-container --use
+    - docker buildx inspect --bootstrap
+    - docker buildx build --platform linux/amd64,linux/arm/v7,linux/arm64/v8 --push --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG .
+  tags:
+    - dind
+  only:
+    - /^build-docker/.*$/@pleroma/pleroma

.mailmap

@@ -1,2 +1,3 @@
 Ariadne Conill <ariadne@dereferenced.org> <nenolod@dereferenced.org>
 Ariadne Conill <ariadne@dereferenced.org> <nenolod@gmail.com>
+rinpatch <rin@patch.cx> <rinpatch@sdf.org>

CHANGELOG.md

@@ -8,66 +8,161 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 
+- The `application` metadata returned with statuses is no longer hardcoded. Apps that want to display these details will now have valid data for new posts after this change.
+- HTTPSecurityPlug now sends a response header to opt out of Google's FLoC (Federated Learning of Cohorts) targeted advertising.
+
+### Added
+
+- MRF (`FollowBotPolicy`): New MRF Policy which makes a designated local Bot account attempt to follow all users in public Notes received by your instance. Users who require approving follower requests or have #nobot in their profile are excluded.
+- Return OAuth token `id` (primary key) in POST `/oauth/token`.
+
+## Unreleased (Patch)
+
+### Fixed
+
+- Try to save exported ConfigDB settings (migrate_from_db) in the system temp directory if default location is not writable.
+- Uploading custom instance thumbnail via AdminAPI/AdminFE generated invalid URL to the image
+- Applying ConcurrentLimiter settings via AdminAPI
+- User login failures if their `notification_settings` were in a NULL state.
+- Mix task `pleroma.user delete_activities` query transaction timeout is now :infinity
+
+## [2.3.0] - 2020-03-01
+
+### Security
+
+- Fixed client user agent leaking through MediaProxy
+
+### Removed
+
+- `:auth, :enforce_oauth_admin_scope_usage` configuration option.
+
+### Changed
+
+- **Breaking**: Changed `mix pleroma.user toggle_confirmed` to `mix pleroma.user confirm`
+- **Breaking**: Changed `mix pleroma.user toggle_activated` to `mix pleroma.user activate/deactivate`
+- **Breaking:** NSFW hashtag is no longer added on sensitive posts
 - Polls now always return a `voters_count`, even if they are single-choice.
 - Admin Emails: The ap id is used as the user link in emails now.
+- Improved registration workflow for email confirmation and account approval modes.
 - Search: When using Postgres 11+, Pleroma will use the `websearch_to_tsvector` function to parse search queries.
 - Emoji: Support the full Unicode 13.1 set of Emoji for reactions, plus regional indicators.
+- Deprecated `Pleroma.Uploaders.S3, :public_endpoint`. Now `Pleroma.Upload, :base_url` is the standard configuration key for all uploaders.
+- Improved Apache webserver support: updated sample configuration, MediaProxy cache invalidation verified with the included sample script
+- Improve OAuth 2.0 provider support. A missing `fqn` field was added to the response, but does not expose the user's email address.
+- Provide redirect of external posts from `/notice/:id` to their original URL
+- Admins no longer receive notifications for reports if they are the actor making the report.
+- Improved Mailer configuration setting descriptions for AdminFE.
+- Updated default avatar to look nicer.
+
+<details>
+  <summary>API Changes</summary>
+
+- **Breaking:** AdminAPI changed User field `confirmation_pending` to `is_confirmed`
+- **Breaking:** AdminAPI changed User field `approval_pending` to `is_approved`
+- **Breaking**: AdminAPI changed User field `deactivated` to `is_active`
+- **Breaking:** AdminAPI `GET /api/pleroma/admin/users/:nickname_or_id/statuses` changed response format and added the number of total users posts.
+- **Breaking:** AdminAPI `GET /api/pleroma/admin/instances/:instance/statuses` changed response format and added the number of total users posts.
+- Admin API: Reports now ordered by newest
+- Pleroma API: `GET /api/v1/pleroma/chats` is deprecated in favor of `GET /api/v2/pleroma/chats`.
+- Pleroma API: Reroute `/api/pleroma/*` to `/api/v1/pleroma/*`
+
+</details>
+- Improved hashtag timeline performance (requires a background migration).
 
 ### Added
 
 - Reports now generate notifications for admins and mods.
-- Experimental websocket-based federation between Pleroma instances.
+- Support for local-only statuses.
-- Support for local-only statuses
 - Support pagination of blocks and mutes.
 - Account backup.
 - Configuration: Add `:instance, autofollowing_nicknames` setting to provide a way to make accounts automatically follow new users that register on the local Pleroma instance.
 - Ability to view remote timelines, with ex. `/api/v1/timelines/public?instance=lain.com` and streams `public:remote` and `public:remote:media`.
 - The site title is now injected as a `title` tag like preloads or metadata.
 - Password reset tokens now are not accepted after a certain age.
-- Mix tasks to help with displaying and removing ConfigDB entries. See `mix pleroma.config`
+- Mix tasks to help with displaying and removing ConfigDB entries. See `mix pleroma.config`.
 - OAuth form improvements: users are remembered by their cookie, the CSS is overridable by the admin, and the style has been improved.
 - OAuth improvements and fixes: more secure session-based authentication (by token that could be revoked anytime), ability to revoke belonging OAuth token from any client etc.
+- Ability to set ActivityPub aliases for follower migration.
+- Configurable background job limits for RichMedia (link previews) and MediaProxyWarmingPolicy
+- Ability to define custom HTTP headers per each frontend
+- MRF (`NoEmptyPolicy`): New MRF Policy which will deny empty statuses or statuses of only mentions from being created by local users
+- New users will receive a simple email confirming their registration if no other emails will be dispatched. (e.g., Welcome, Confirmation, or Approval Required)
 
 <details>
   <summary>API Changes</summary>
 - Admin API: (`GET /api/pleroma/admin/users`) filter users by `unconfirmed` status and `actor_type`.
+- Admin API: OpenAPI spec for the user-related operations
+- Pleroma API: `GET /api/v2/pleroma/chats` added. It is exactly like `GET /api/v1/pleroma/chats` except supports pagination.
 - Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
 - Pleroma API: (`GET /api/v1/pleroma/federation_status`) Add a way to get a list of unreachable instances.
 - Mastodon API: User and conversation mutes can now auto-expire if `expires_in` parameter was given while adding the mute.
-- Admin API: An endpoint to manage frontends
+- Admin API: An endpoint to manage frontends.
-- Streaming API: Add follow relationships updates  
+- Streaming API: Add follow relationships updates.
+- WebPush: Introduce `pleroma:chat_mention` and `pleroma:emoji_reaction` notification types.
+- Mastodon API: Add monthly active users to `/api/v1/instance` (`pleroma.stats.mau`).
+- Mastodon API: Home, public, hashtag & list timelines accept `only_media`, `remote` & `local` parameters for filtration.
+- Mastodon API: `/api/v1/accounts/:id` & `/api/v1/mutes` endpoints accept `with_relationships` parameter and return filled `pleroma.relationship` field.
+- Mastodon API: Endpoint to remove a conversation (`DELETE /api/v1/conversations/:id`).
+- Mastodon API: `expires_in` in the scheduled post `params` field on `/api/v1/statuses` and `/api/v1/scheduled_statuses/:id` endpoints.
 </details>
 
 ### Fixed
 
 - Users with `is_discoverable` field set to false (default value) will appear in in-service search results but be hidden from external services (search bots etc.).
+- Streaming API: Posts and notifications are not dropped, when CLI task is executing.
+- Creating incorrect IPv4 address-style HTTP links when encountering certain numbers.
+- Reblog API Endpoint: Do not set visibility parameter to public by default and let CommonAPI to infer it from status, so a user can reblog their private status without explicitly setting reblog visibility to private.
+- Tag URLs in statuses are now absolute
+- Removed duplicate jobs to purge expired activities
+- File extensions of some attachments were incorrectly changed. This feature has been disabled for now.
+- Mix task pleroma.instance creates missing parent directories if the configuration or SQL output paths are changed.
 
 <details>
   <summary>API Changes</summary>
   - Mastodon API: Current user is now included in conversation if it's the only participant.
   - Mastodon API: Fixed last_status.account being not filled with account data.
+  - Mastodon API: Fix not being able to add or remove multiple users at once in lists.
+  - Mastodon API: Fixed own_votes being not returned with poll data.
+  - Mastodon API: Fixed creation of scheduled posts with polls.
+  - Mastodon API: Support for expires_in/expires_at in the Filters.
 </details>
 
-## Unreleased (Patch)
+## [2.2.2] - 2020-01-18
+
+### Fixed
+
+- StealEmojiPolicy creates dir for emojis, if it doesn't exist.
+- Updated `elixir_make` to a non-retired version
+
+### Upgrade notes
+
+1. Restart Pleroma
+
+## [2.2.1] - 2020-12-22
 
 ### Changed
+- Updated Pleroma FE
 
 ### Fixed
 
 - Config generation: rename `Pleroma.Upload.Filter.ExifTool` to `Pleroma.Upload.Filter.Exiftool`.
-- Search: RUM index search speed has been fixed.
 - S3 Uploads with Elixir 1.11.
-- Emoji Reaction activity filtering from blocked and muted accounts.
 - Mix task pleroma.user delete_activities for source installations.
-- Fix ability to update Pleroma Chat push notifications with PUT /api/v1/push/subscription and alert type pleroma:chat_mention
+- Search: RUM index search speed has been fixed.
+- Rich Media Previews sometimes showed the wrong preview due to a bug following redirects.
+- Fixes for the autolinker.
 - Forwarded reports duplication from Pleroma instances.
+- Emoji Reaction activity filtering from blocked and muted accounts.
 
-<details>
+- <details>
     <summary>API</summary>
   - Statuses were not displayed for Mastodon forwarded reports.
-
   </details>
 
+### Upgrade notes
+
+1. Restart Pleroma
+
 ## [2.2.0] - 2020-11-12
 
 ### Security
@@ -104,7 +199,7 @@ switched to a new configuration mechanism, however it was not officially removed
 
 - Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
 - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
-- Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
+- Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email resend_confirmation_emails`)
 - Mix task option for force-unfollowing relays
 - App metrics: ability to restrict access to specified IP whitelist.
 
@@ -428,7 +523,6 @@ switched to a new configuration mechanism, however it was not officially removed
 - Static-FE: Fix remote posts not being sanitized
 
 ### Fixed
-=======
 - Rate limiter crashes when there is no explicitly specified ip in the config
 - 500 errors when no `Accept` header is present if Static-FE is enabled
 - Instance panel not being updated immediately due to wrong `Cache-Control` headers

COPYING

@@ -1,10 +1,17 @@
-Unless otherwise stated this repository is copyright © 2017-2020
+Unless otherwise stated this repository is copyright © 2017-2021
 Pleroma Authors <https://pleroma.social/>, and is distributed under
 The GNU Affero General Public License Version 3, you should have received a
 copy of the license file as AGPL-3.
 
 ---
 
+Files inside docs directory are copyright © 2021 Pleroma Authors
+<https://pleroma.social/>, and are distributed under the Creative Commons
+Attribution 4.0 International license, you should have received
+a copy of the license file as CC-BY-4.0.
+
+---
+
 The following files are copyright © 2019 shitposter.club, and are distributed
 under the Creative Commons Attribution-ShareAlike 4.0 International license,
 you should have received a copy of the license file as CC-BY-SA-4.0.

benchmarks/load_testing/fetcher.ex

@@ -33,10 +33,11 @@ defmodule Pleroma.LoadTesting.Fetcher do
   end
 
   defp create_filter(user) do
-    Pleroma.Filter.create(%Pleroma.Filter{
+    Pleroma.Filter.create(%{
       user_id: user.id,
       phrase: "must be filtered",
-      hide: true
+      hide: true,
+      context: ["home"]
     })
   end
 

benchmarks/load_testing/users.ex

@@ -55,7 +55,7 @@ defmodule Pleroma.LoadTesting.Users do
       name: "Test テスト User #{i}",
       email: "user#{i}@example.com",
       nickname: "nick#{i}",
-      password_hash: Pbkdf2.hash_pwd_salt("test"),
+      password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt("test"),
       bio: "Tester Number #{i}",
       local: !remote
     }

config/config.exs

@@ -47,7 +47,6 @@ use Mix.Config
 config :pleroma, ecto_repos: [Pleroma.Repo]
 
 config :pleroma, Pleroma.Repo,
-  types: Pleroma.PostgresTypes,
   telemetry_event: [Pleroma.Repo.Instrumenter],
   migration_lock: nil
 
@@ -64,23 +63,24 @@ config :pleroma, Pleroma.Upload,
   filters: [Pleroma.Upload.Filter.Dedupe],
   link_name: false,
   proxy_remote: false,
-  proxy_opts: [
-    redirect_on_failure: false,
-    max_body_length: 25 * 1_048_576,
-    http: [
-      follow_redirect: true,
-      pool: :upload
-    ]
-  ],
   filename_display_max_length: 30,
-  default_description: nil
+  default_description: nil,
+  base_url: nil
 
 config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
 
 config :pleroma, Pleroma.Uploaders.S3,
   bucket: nil,
-  streaming_enabled: true,
+  bucket_namespace: nil,
-  public_endpoint: "https://s3.amazonaws.com"
+  truncated_namespace: nil,
+  streaming_enabled: true
+
+config :ex_aws, :s3,
+  # host: "s3.wasabisys.com", # required if not Amazon AWS
+  access_key_id: nil,
+  secret_access_key: nil,
+  # region: "us-east-1", # may be required for Amazon AWS
+  scheme: "https://"
 
 config :pleroma, :emoji,
   shortcode_globs: ["/emoji/custom/**/*.png"],
@@ -306,7 +306,7 @@ config :pleroma, :frontend_configurations,
     hideSitename: false,
     hideUserStats: false,
     loginMethod: "password",
-    logo: "/static/logo.png",
+    logo: "/static/logo.svg",
     logoMargin: ".1em",
     logoMask: true,
     minimalScopesMode: false,
@@ -343,8 +343,8 @@ config :pleroma, :assets,
 config :pleroma, :manifest,
   icons: [
     %{
-      src: "/static/logo.png",
+      src: "/static/logo.svg",
-      type: "image/png"
+      type: "image/svg+xml"
     }
   ],
   theme_color: "#282c37",
@@ -391,6 +391,11 @@ config :pleroma, :mrf_keyword,
   federated_timeline_removal: [],
   replace: []
 
+config :pleroma, :mrf_hashtag,
+  sensitive: ["nsfw"],
+  reject: [],
+  federated_timeline_removal: []
+
 config :pleroma, :mrf_subchain, match_actor: %{}
 
 config :pleroma, :mrf_activity_expiration, days: 365
@@ -404,6 +409,8 @@ config :pleroma, :mrf_object_age,
   threshold: 604_800,
   actions: [:delist, :strip_followers]
 
+config :pleroma, :mrf_follow_bot, follower_nickname: nil
+
 config :pleroma, :rich_media,
   enabled: true,
   ignore_hosts: [],
@@ -438,7 +445,9 @@ config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http,
   headers: [],
   options: []
 
-config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil
+config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script,
+  script_path: nil,
+  url_format: nil
 
 # Note: media preview proxy depends on media proxy to be enabled
 config :pleroma, :media_preview_proxy,
@@ -541,6 +550,7 @@ config :pleroma, Oban,
   queues: [
     activity_expiration: 10,
     token_expiration: 5,
+    filter_expiration: 1,
     backup: 1,
     federator_incoming: 50,
     federator_outgoing: 50,
@@ -551,7 +561,7 @@ config :pleroma, Oban,
     scheduled_activities: 10,
     background: 5,
     remote_fetcher: 2,
-    attachments_cleanup: 5,
+    attachments_cleanup: 1,
     new_users_digest: 1,
     mute_expire: 5
   ],
@@ -608,10 +618,7 @@ config :ueberauth,
        base_path: "/oauth",
        providers: ueberauth_providers
 
-config :pleroma,
+config :pleroma, :auth, oauth_consumer_strategies: oauth_consumer_strategies
-       :auth,
-       enforce_oauth_admin_scope_usage: true,
-       oauth_consumer_strategies: oauth_consumer_strategies
 
 config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Sendmail, enabled: false
 
@@ -648,12 +655,16 @@ config :pleroma, :email_notifications,
   }
 
 config :pleroma, :oauth2,
-  token_expires_in: 3600 * 24 * 30,
+  token_expires_in: 3600 * 24 * 365 * 100,
   issue_new_refresh_token: true,
   clean_expired_tokens: false
 
 config :pleroma, :database, rum_enabled: false
 
+config :pleroma, :features, improved_hashtag_timeline: :auto
+
+config :pleroma, :populate_hashtags_table, fault_rate_allowance: 0.01
+
 config :pleroma, :env, Mix.env()
 
 config :http_signatures,
@@ -723,7 +734,10 @@ config :pleroma, :frontends,
       "git" => "https://git.pleroma.social/pleroma/fedi-fe",
       "build_url" =>
         "https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build",
-      "ref" => "master"
+      "ref" => "master",
+      "custom-http-headers" => [
+        {"service-worker-allowed", "/"}
+      ]
     },
     "admin-fe" => %{
       "name" => "admin-fe",
@@ -832,6 +846,11 @@ config :pleroma, Pleroma.User.Backup,
   limit_days: 7,
   dir: nil
 
+config :pleroma, ConcurrentLimiter, [
+  {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},
+  {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}
+]
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/description.exs

@@ -60,6 +60,12 @@ frontend_options = [
     label: "Build directory",
     type: :string,
     description: "The directory inside the zip file "
+  },
+  %{
+    key: "custom-http-headers",
+    label: "Custom HTTP headers",
+    type: {:list, :string},
+    description: "The custom HTTP headers for the frontend"
   }
 ]
 
@@ -93,7 +99,8 @@ config :pleroma, :config_description, [
         key: :base_url,
         label: "Base URL",
         type: :string,
-        description: "Base URL for the uploads, needed if you use CDN",
+        description:
+          "Base URL for the uploads. Required if you use a CDN or host attachments under a different domain.",
         suggestions: [
           "https://cdn-host.com"
         ]
@@ -101,74 +108,10 @@ config :pleroma, :config_description, [
       %{
         key: :proxy_remote,
         type: :boolean,
-        description:
+        description: """
-          "If enabled, requests to media stored using a remote uploader will be proxied instead of being redirected"
+        Proxy requests to the remote uploader.\n
-      },
+        Useful if media upload endpoint is not internet accessible.
-      %{
+        """
-        key: :proxy_opts,
-        label: "Proxy Options",
-        type: :keyword,
-        description: "Options for Pleroma.ReverseProxy",
-        suggestions: [
-          redirect_on_failure: false,
-          max_body_length: 25 * 1_048_576,
-          http: [
-            follow_redirect: true,
-            pool: :media
-          ]
-        ],
-        children: [
-          %{
-            key: :redirect_on_failure,
-            type: :boolean,
-            description:
-              "Redirects the client to the real remote URL if there's any HTTP errors. " <>
-                "Any error during body processing will not be redirected as the response is chunked."
-          },
-          %{
-            key: :max_body_length,
-            type: :integer,
-            description:
-              "Limits the content length to be approximately the " <>
-                "specified length. It is validated with the `content-length` header and also verified when proxying."
-          },
-          %{
-            key: :http,
-            label: "HTTP",
-            type: :keyword,
-            description: "HTTP options",
-            children: [
-              %{
-                key: :adapter,
-                type: :keyword,
-                description: "Adapter specific options",
-                children: [
-                  %{
-                    key: :ssl_options,
-                    type: :keyword,
-                    label: "SSL Options",
-                    description: "SSL options for HTTP adapter",
-                    children: [
-                      %{
-                        key: :versions,
-                        type: {:list, :atom},
-                        description: "List of TLS versions to use",
-                        suggestions: [:tlsv1, ":tlsv1.1", ":tlsv1.2"]
-                      }
-                    ]
-                  }
-                ]
-              },
-              %{
-                key: :proxy_url,
-                label: "Proxy URL",
-                type: [:string, :tuple],
-                description: "Proxy URL",
-                suggestions: ["127.0.0.1:8123", {:socks5, :localhost, 9050}]
-              }
-            ]
-          }
-        ]
       },
       %{
         key: :filename_display_max_length,
@@ -213,18 +156,12 @@ config :pleroma, :config_description, [
         description: "S3 bucket namespace",
         suggestions: ["pleroma"]
       },
-      %{
-        key: :public_endpoint,
-        type: :string,
-        description: "S3 endpoint",
-        suggestions: ["https://s3.amazonaws.com"]
-      },
       %{
         key: :truncated_namespace,
         type: :string,
         description:
           "If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or \"\" etc." <>
-            " For example, when using CDN to S3 virtual host format, set \"\". At this time, write CNAME to CDN in public_endpoint."
+            " For example, when using CDN to S3 virtual host format, set \"\". At this time, write CNAME to CDN in Upload base_url."
       },
       %{
         key: :streaming_enabled,
@@ -278,253 +215,216 @@ config :pleroma, :config_description, [
     type: :group,
     description: "Mailer-related settings",
     children: [
+      %{
+        key: :enabled,
+        label: "Mailer Enabled",
+        type: :boolean
+      },
       %{
         key: :adapter,
         type: :module,
         description:
-          "One of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters)," <>
+          "One of the mail adapters listed in [Swoosh documentation](https://hexdocs.pm/swoosh/Swoosh.html#module-adapters)",
-            " or Swoosh.Adapters.Local for in-memory mailbox",
         suggestions: [
+          Swoosh.Adapters.AmazonSES,
+          Swoosh.Adapters.Dyn,
+          Swoosh.Adapters.Gmail,
+          Swoosh.Adapters.Mailgun,
+          Swoosh.Adapters.Mailjet,
+          Swoosh.Adapters.Mandrill,
+          Swoosh.Adapters.Postmark,
           Swoosh.Adapters.SMTP,
           Swoosh.Adapters.Sendgrid,
           Swoosh.Adapters.Sendmail,
-          Swoosh.Adapters.Mandrill,
-          Swoosh.Adapters.Mailgun,
-          Swoosh.Adapters.Mailjet,
-          Swoosh.Adapters.Postmark,
-          Swoosh.Adapters.SparkPost,
-          Swoosh.Adapters.AmazonSES,
-          Swoosh.Adapters.Dyn,
           Swoosh.Adapters.SocketLabs,
-          Swoosh.Adapters.Gmail,
+          Swoosh.Adapters.SparkPost
-          Swoosh.Adapters.Local
         ]
       },
-      %{
-        key: :enabled,
-        type: :boolean,
-        description: "Allow/disallow send emails"
-      },
       %{
         group: {:subgroup, Swoosh.Adapters.SMTP},
         key: :relay,
         type: :string,
-        description: "`Swoosh.Adapters.SMTP` adapter specific setting",
+        description: "Hostname or IP address",
-        suggestions: ["smtp.gmail.com"]
+        suggestions: ["smtp.example.com"]
-      },
-      %{
-        group: {:subgroup, Swoosh.Adapters.SMTP},
-        key: :username,
-        type: :string,
-        description: "`Swoosh.Adapters.SMTP` adapter specific setting",
-        suggestions: ["pleroma"]
-      },
-      %{
-        group: {:subgroup, Swoosh.Adapters.SMTP},
-        key: :password,
-        type: :string,
-        description: "`Swoosh.Adapters.SMTP` adapter specific setting",
-        suggestions: ["password"]
-      },
-      %{
-        group: {:subgroup, Swoosh.Adapters.SMTP},
-        key: :ssl,
-        label: "SSL",
-        type: :boolean,
-        description: "`Swoosh.Adapters.SMTP` adapter specific setting"
-      },
-      %{
-        group: {:subgroup, Swoosh.Adapters.SMTP},
-        key: :tls,
-        label: "TLS",
-        type: :atom,
-        description: "`Swoosh.Adapters.SMTP` adapter specific setting",
-        suggestions: [:always, :never, :if_available]
-      },
-      %{
-        group: {:subgroup, Swoosh.Adapters.SMTP},
-        key: :auth,
-        type: :atom,
-        description: "`Swoosh.Adapters.SMTP` adapter specific setting",
-        suggestions: [:always, :never, :if_available]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.SMTP},
         key: :port,
         type: :integer,
-        description: "`Swoosh.Adapters.SMTP` adapter specific setting",
+        description: "SMTP port",
-        suggestions: [1025]
+        suggestions: ["1025"]
+      },
+      %{
+        group: {:subgroup, Swoosh.Adapters.SMTP},
+        key: :username,
+        type: :string,
+        description: "SMTP AUTH username",
+        suggestions: ["user@example.com"]
+      },
+      %{
+        group: {:subgroup, Swoosh.Adapters.SMTP},
+        key: :password,
+        type: :string,
+        description: "SMTP AUTH password",
+        suggestions: ["password"]
+      },
+      %{
+        group: {:subgroup, Swoosh.Adapters.SMTP},
+        key: :ssl,
+        label: "Use SSL",
+        type: :boolean,
+        description: "Use Implicit SSL/TLS. e.g. port 465"
+      },
+      %{
+        group: {:subgroup, Swoosh.Adapters.SMTP},
+        key: :tls,
+        label: "STARTTLS Mode",
+        type: {:dropdown, :atom},
+        description: "Explicit TLS (STARTTLS) enforcement mode",
+        suggestions: [:if_available, :always, :never]
+      },
+      %{
+        group: {:subgroup, Swoosh.Adapters.SMTP},
+        key: :auth,
+        label: "AUTH Mode",
+        type: {:dropdown, :atom},
+        description: "SMTP AUTH enforcement mode",
+        suggestions: [:if_available, :always, :never]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.SMTP},
         key: :retries,
         type: :integer,
-        description: "`Swoosh.Adapters.SMTP` adapter specific setting",
+        description: "SMTP temporary (4xx) error retries",
-        suggestions: [5]
+        suggestions: [1]
-      },
-      %{
-        group: {:subgroup, Swoosh.Adapters.SMTP},
-        key: :no_mx_lookups,
-        label: "No MX lookups",
-        type: :boolean,
-        description: "`Swoosh.Adapters.SMTP` adapter specific setting"
       },
       %{
         group: {:subgroup, Swoosh.Adapters.Sendgrid},
         key: :api_key,
-        label: "API key",
+        label: "SendGrid API Key",
         type: :string,
-        description: "`Swoosh.Adapters.Sendgrid` adapter specific setting",
+        suggestions: ["YOUR_API_KEY"]
-        suggestions: ["my-api-key"]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.Sendmail},
         key: :cmd_path,
         type: :string,
-        description: "`Swoosh.Adapters.Sendmail` adapter specific setting",
         suggestions: ["/usr/bin/sendmail"]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.Sendmail},
         key: :cmd_args,
         type: :string,
-        description: "`Swoosh.Adapters.Sendmail` adapter specific setting",
         suggestions: ["-N delay,failure,success"]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.Sendmail},
         key: :qmail,
-        type: :boolean,
+        label: "Qmail compat mode",
-        description: "`Swoosh.Adapters.Sendmail` adapter specific setting"
+        type: :boolean
       },
       %{
         group: {:subgroup, Swoosh.Adapters.Mandrill},
         key: :api_key,
-        label: "API key",
+        label: "Mandrill API Key",
         type: :string,
-        description: "`Swoosh.Adapters.Mandrill` adapter specific setting",
+        suggestions: ["YOUR_API_KEY"]
-        suggestions: ["my-api-key"]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.Mailgun},
         key: :api_key,
-        label: "API key",
+        label: "Mailgun API Key",
         type: :string,
-        description: "`Swoosh.Adapters.Mailgun` adapter specific setting",
+        suggestions: ["YOUR_API_KEY"]
-        suggestions: ["my-api-key"]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.Mailgun},
         key: :domain,
         type: :string,
-        description: "`Swoosh.Adapters.Mailgun` adapter specific setting",
+        suggestions: ["YOUR_DOMAIN_NAME"]
-        suggestions: ["pleroma.com"]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.Mailjet},
         key: :api_key,
-        label: "API key",
+        label: "MailJet Public API Key",
         type: :string,
-        description: "`Swoosh.Adapters.Mailjet` adapter specific setting",
+        suggestions: ["MJ_APIKEY_PUBLIC"]
-        suggestions: ["my-api-key"]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.Mailjet},
         key: :secret,
+        label: "MailJet Private API Key",
         type: :string,
-        description: "`Swoosh.Adapters.Mailjet` adapter specific setting",
+        suggestions: ["MJ_APIKEY_PRIVATE"]
-        suggestions: ["my-secret-key"]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.Postmark},
         key: :api_key,
-        label: "API key",
+        label: "Postmark API Key",
         type: :string,
-        description: "`Swoosh.Adapters.Postmark` adapter specific setting",
+        suggestions: ["X-Postmark-Server-Token"]
-        suggestions: ["my-api-key"]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.SparkPost},
         key: :api_key,
-        label: "API key",
+        label: "SparkPost API key",
         type: :string,
-        description: "`Swoosh.Adapters.SparkPost` adapter specific setting",
+        suggestions: ["YOUR_API_KEY"]
-        suggestions: ["my-api-key"]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.SparkPost},
         key: :endpoint,
         type: :string,
-        description: "`Swoosh.Adapters.SparkPost` adapter specific setting",
         suggestions: ["https://api.sparkpost.com/api/v1"]
       },
-      %{
-        group: {:subgroup, Swoosh.Adapters.AmazonSES},
-        key: :region,
-        type: :string,
-        description: "`Swoosh.Adapters.AmazonSES` adapter specific setting",
-        suggestions: ["us-east-1", "us-east-2"]
-      },
       %{
         group: {:subgroup, Swoosh.Adapters.AmazonSES},
         key: :access_key,
+        label: "AWS Access Key",
         type: :string,
-        description: "`Swoosh.Adapters.AmazonSES` adapter specific setting",
+        suggestions: ["AWS_ACCESS_KEY"]
-        suggestions: ["aws-access-key"]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.AmazonSES},
         key: :secret,
+        label: "AWS Secret Key",
         type: :string,
-        description: "`Swoosh.Adapters.AmazonSES` adapter specific setting",
+        suggestions: ["AWS_SECRET_KEY"]
-        suggestions: ["aws-secret-key"]
+      },
+      %{
+        group: {:subgroup, Swoosh.Adapters.AmazonSES},
+        key: :region,
+        label: "AWS Region",
+        type: :string,
+        suggestions: ["us-east-1", "us-east-2"]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.Dyn},
         key: :api_key,
-        label: "API key",
+        label: "Dyn API Key",
         type: :string,
-        description: "`Swoosh.Adapters.Dyn` adapter specific setting",
+        suggestions: ["apikey"]
-        suggestions: ["my-api-key"]
-      },
-      %{
-        group: {:subgroup, Swoosh.Adapters.SocketLabs},
-        key: :server_id,
-        type: :string,
-        description: "`Swoosh.Adapters.SocketLabs` adapter specific setting"
       },
       %{
         group: {:subgroup, Swoosh.Adapters.SocketLabs},
         key: :api_key,
-        label: "API key",
+        label: "SocketLabs API Key",
         type: :string,
-        description: "`Swoosh.Adapters.SocketLabs` adapter specific setting"
+        suggestions: ["INJECTION_API_KEY"]
+      },
+      %{
+        group: {:subgroup, Swoosh.Adapters.SocketLabs},
+        key: :server_id,
+        label: "Server ID",
+        type: :string,
+        suggestions: ["SERVER_ID"]
       },
       %{
         group: {:subgroup, Swoosh.Adapters.Gmail},
         key: :access_token,
+        label: "GMail API Access Token",
         type: :string,
-        description: "`Swoosh.Adapters.Gmail` adapter specific setting"
+        suggestions: ["GMAIL_API_ACCESS_TOKEN"]
-      }
-    ]
-  },
-  %{
-    group: :swoosh,
-    type: :group,
-    description: "`Swoosh.Adapters.Local` adapter specific settings",
-    children: [
-      %{
-        group: {:subgroup, Swoosh.Adapters.Local},
-        key: :serve_mailbox,
-        type: :boolean,
-        description: "Run the preview server together as part of your app"
-      },
-      %{
-        group: {:subgroup, Swoosh.Adapters.Local},
-        key: :preview_port,
-        type: :integer,
-        description: "The preview server port",
-        suggestions: [4001]
       }
     ]
   },
@@ -559,6 +459,42 @@ config :pleroma, :config_description, [
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: :features,
+    type: :group,
+    description: "Customizable features",
+    children: [
+      %{
+        key: :improved_hashtag_timeline,
+        type: {:dropdown, :atom},
+        description:
+          "Setting to force toggle / force disable improved hashtags timeline. `:enabled` forces hashtags to be fetched from `hashtags` table for hashtags timeline. `:disabled` forces object-embedded hashtags to be used (slower). Keep it `:auto` for automatic behaviour (it is auto-set to `:enabled` [unless overridden] when HashtagsTableMigrator completes).",
+        suggestions: [:auto, :enabled, :disabled]
+      }
+    ]
+  },
+  %{
+    group: :pleroma,
+    key: :populate_hashtags_table,
+    type: :group,
+    description: "`populate_hashtags_table` background migration settings",
+    children: [
+      %{
+        key: :fault_rate_allowance,
+        type: :float,
+        description:
+          "Max accepted rate of objects that failed in the migration. Any value from 0.0 which tolerates no errors to 1.0 which will enable the feature even if hashtags transfer failed for all records.",
+        suggestions: [0.01]
+      },
+      %{
+        key: :sleep_interval_ms,
+        type: :integer,
+        description:
+          "Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances)."
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: :instance,
@@ -1254,7 +1190,7 @@ config :pleroma, :config_description, [
             hideSitename: false,
             hideUserStats: false,
             loginMethod: "password",
-            logo: "/static/logo.png",
+            logo: "/static/logo.svg",
             logoMargin: ".1em",
             logoMask: true,
             minimalScopesMode: false,
@@ -1340,7 +1276,7 @@ config :pleroma, :config_description, [
             key: :logo,
             type: {:string, :image},
             description: "URL of the logo, defaults to Pleroma's logo",
-            suggestions: ["/static/logo.png"]
+            suggestions: ["/static/logo.svg"]
           },
           %{
             key: :logoMargin,
@@ -1550,7 +1486,7 @@ config :pleroma, :config_description, [
       %{
         key: :enabled,
         type: :boolean,
-        description: "Enables proxying of remote media to the instance's proxy"
+        description: "Enables proxying of remote media via the instance's proxy"
       },
       %{
         key: :base_url,
@@ -1587,80 +1523,42 @@ config :pleroma, :config_description, [
       },
       %{
         key: :proxy_opts,
-        label: "Proxy Options",
+        label: "Advanced MediaProxy Options",
         type: :keyword,
-        description: "Options for Pleroma.ReverseProxy",
+        description: "Internal Pleroma.ReverseProxy settings",
         suggestions: [
           redirect_on_failure: false,
           max_body_length: 25 * 1_048_576,
-          max_read_duration: 30_000,
+          max_read_duration: 30_000
-          http: [
-            follow_redirect: true,
-            pool: :media
-          ]
         ],
         children: [
           %{
             key: :redirect_on_failure,
             type: :boolean,
-            description:
+            description: """
-              "Redirects the client to the real remote URL if there's any HTTP errors. " <>
+            Redirects the client to the origin server upon encountering HTTP errors.\n
-                "Any error during body processing will not be redirected as the response is chunked."
+            Note that files larger than Max Body Length will trigger an error. (e.g., Peertube videos)\n\n
+            **WARNING:** This setting will allow larger files to be accessed, but exposes the\n
+            IP addresses of your users to the other servers, bypassing the MediaProxy.
+            """
           },
           %{
             key: :max_body_length,
             type: :integer,
             description:
-              "Limits the content length to be approximately the " <>
+              "Maximum file size (in bytes) allowed through the Pleroma MediaProxy cache."
-                "specified length. It is validated with the `content-length` header and also verified when proxying."
           },
           %{
             key: :max_read_duration,
             type: :integer,
-            description: "Timeout (in milliseconds) of GET request to remote URI."
+            description: "Timeout (in milliseconds) of GET request to the remote URI."
-          },
-          %{
-            key: :http,
-            label: "HTTP",
-            type: :keyword,
-            description: "HTTP options",
-            children: [
-              %{
-                key: :adapter,
-                type: :keyword,
-                description: "Adapter specific options",
-                children: [
-                  %{
-                    key: :ssl_options,
-                    type: :keyword,
-                    label: "SSL Options",
-                    description: "SSL options for HTTP adapter",
-                    children: [
-                      %{
-                        key: :versions,
-                        type: {:list, :atom},
-                        description: "List of TLS version to use",
-                        suggestions: [:tlsv1, ":tlsv1.1", ":tlsv1.2"]
-                      }
-                    ]
-                  }
-                ]
-              },
-              %{
-                key: :proxy_url,
-                label: "Proxy URL",
-                type: [:string, :tuple],
-                description: "Proxy URL",
-                suggestions: ["127.0.0.1:8123", {:socks5, :localhost, 9050}]
-              }
-            ]
           }
         ]
       },
       %{
         key: :whitelist,
         type: {:list, :string},
-        description: "List of hosts with scheme to bypass the mediaproxy",
+        description: "List of hosts with scheme to bypass the MediaProxy",
         suggestions: ["http://example.com"]
       }
     ]
@@ -1698,7 +1596,7 @@ config :pleroma, :config_description, [
         key: :min_content_length,
         type: :integer,
         description:
-          "Min content length to perform preview, in bytes. If greater than 0, media smaller in size will be served as is, without thumbnailing."
+          "Min content length (in bytes) to perform preview. Media smaller in size will be served without thumbnailing."
       }
     ]
   },
@@ -1736,13 +1634,21 @@ config :pleroma, :config_description, [
     group: :pleroma,
     key: Pleroma.Web.MediaProxy.Invalidation.Script,
     type: :group,
-    description: "Script invalidate settings",
+    description: "Invalidation script settings",
     children: [
       %{
         key: :script_path,
         type: :string,
-        description: "Path to shell script. Which will run purge cache.",
+        description: "Path to executable script which will purge cached items.",
         suggestions: ["./installation/nginx-cache-purge.sh.example"]
+      },
+      %{
+        key: :url_format,
+        label: "URL Format",
+        type: :string,
+        description:
+          "Optional URL format preprocessing. Only required for Apache's htcacheclean.",
+        suggestions: [":htcacheclean"]
       }
     ]
   },
@@ -1953,14 +1859,8 @@ config :pleroma, :config_description, [
     group: :pleroma,
     key: Oban,
     type: :group,
-    description: """
+    description:
-    [Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration.
+      "[Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration.",
-
-    Note: if you are running PostgreSQL in [`silent_mode`](https://postgresqlco.nf/en/doc/param/silent_mode?version=9.1),
-      it's advised to set [`log_destination`](https://postgresqlco.nf/en/doc/param/log_destination?version=9.1) to `syslog`,
-      otherwise `postmaster.log` file may grow because of "you don't own a lock of type ShareLock" warnings
-      (see https://github.com/sorentwo/oban/issues/52).
-    """,
     children: [
       %{
         key: :log,
@@ -2990,7 +2890,7 @@ config :pleroma, :config_description, [
         type: :integer,
         description:
           "Activity pub routes (except question activities). Default: `nil` (no expiration).",
-        suggestions: [30_000, nil]
+        suggestions: [nil]
       },
       %{
         key: :activity_pub_question,
@@ -3042,6 +2942,23 @@ config :pleroma, :config_description, [
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: :mrf_follow_bot,
+    tab: :mrf,
+    related_policy: "Pleroma.Web.ActivityPub.MRF.FollowBotPolicy",
+    label: "MRF FollowBot Policy",
+    type: :group,
+    description: "Automatically follows newly discovered accounts.",
+    children: [
+      %{
+        key: :follower_nickname,
+        type: :string,
+        description: "The name of the bot account to use for following newly discovered users.",
+        suggestions: ["followbot"]
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: :modules,
@@ -3326,6 +3243,12 @@ config :pleroma, :config_description, [
         type: :string,
         description: "S3 host",
         suggestions: ["s3.eu-central-1.amazonaws.com"]
+      },
+      %{
+        key: :region,
+        type: :string,
+        description: "S3 region (for AWS)",
+        suggestions: ["us-east-1"]
       }
     ]
   },
@@ -3422,9 +3345,9 @@ config :pleroma, :config_description, [
       },
       %{
         key: :ip_whitelist,
+        label: "IP Whitelist",
         type: [{:list, :string}, {:list, :charlist}, {:list, :tuple}],
-        description:
+        description: "Restrict access of app metrics endpoint to the specified IP addresses."
-          "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses."
       },
       %{
         key: :auth,
@@ -3445,5 +3368,53 @@ config :pleroma, :config_description, [
         suggestions: [:text, :protobuf]
       }
     ]
+  },
+  %{
+    group: :pleroma,
+    key: ConcurrentLimiter,
+    type: :group,
+    description: "Limits configuration for background tasks.",
+    children: [
+      %{
+        key: Pleroma.Web.RichMedia.Helpers,
+        type: :keyword,
+        description: "Concurrent limits configuration for getting RichMedia for activities.",
+        suggestions: [max_running: 5, max_waiting: 5],
+        children: [
+          %{
+            key: :max_running,
+            type: :integer,
+            description: "Max running concurrently jobs.",
+            suggestion: [5]
+          },
+          %{
+            key: :max_waiting,
+            type: :integer,
+            description: "Max waiting jobs.",
+            suggestion: [5]
+          }
+        ]
+      },
+      %{
+        key: Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy,
+        type: :keyword,
+        description: "Concurrent limits configuration for MediaProxyWarmingPolicy.",
+        suggestions: [max_running: 5, max_waiting: 5],
+        children: [
+          %{
+            key: :max_running,
+            type: :integer,
+            description: "Max running concurrently jobs.",
+            suggestion: [5]
+          },
+          %{
+            key: :max_waiting,
+            type: :integer,
+            description: "Max waiting jobs.",
+            suggestion: [5]
+          }
+        ]
+      }
+    ]
   }
 ]

config/emoji.txt

@@ -1,2 +1,3 @@
 firefox, /emoji/Firefox.gif, Gif,Fun
 blank, /emoji/blank.png, Fun
+dinosaur, /emoji/dino walking.gif, Gif

config/test.exs

@@ -38,7 +38,7 @@ config :pleroma, :instance,
   external_user_synchronization: false,
   static_dir: "test/instance_static/"
 
-config :pleroma, :activitypub, sign_object_fetches: false
+config :pleroma, :activitypub, sign_object_fetches: false, follow_handshake_timeout: 0
 
 # Configure your database
 config :pleroma, Pleroma.Repo,
@@ -47,10 +47,13 @@ config :pleroma, Pleroma.Repo,
   password: "postgres",
   database: "pleroma_test",
   hostname: System.get_env("DB_HOST") || "localhost",
-  pool: Ecto.Adapters.SQL.Sandbox
+  pool: Ecto.Adapters.SQL.Sandbox,
+  pool_size: 50
+
+config :pleroma, :dangerzone, override_repo_pool_size: true
 
 # Reduce hash rounds for testing
-config :pbkdf2_elixir, rounds: 1
+config :pleroma, :password, iterations: 1
 
 config :tesla, adapter: Tesla.Mock
 
@@ -112,15 +115,24 @@ config :pleroma, Pleroma.Web.Plugs.RemoteIp, enabled: false
 
 config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true
 
-config :pleroma, Pleroma.Uploaders.S3,
-  bucket: nil,
-  streaming_enabled: true,
-  public_endpoint: nil
-
 config :tzdata, :autoupdate, :disabled
 
 config :pleroma, :mrf, policies: []
 
+config :pleroma, :pipeline,
+  object_validator: Pleroma.Web.ActivityPub.ObjectValidatorMock,
+  mrf: Pleroma.Web.ActivityPub.MRFMock,
+  activity_pub: Pleroma.Web.ActivityPub.ActivityPubMock,
+  side_effects: Pleroma.Web.ActivityPub.SideEffectsMock,
+  federator: Pleroma.Web.FederatorMock,
+  config: Pleroma.ConfigMock
+
+config :pleroma, :cachex, provider: Pleroma.CachexMock
+
+config :pleroma, :side_effects,
+  ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock,
+  logger: Pleroma.LoggerMock
+
 if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"
 else

docs/administration/CLI_tasks/config.md

@@ -32,16 +32,20 @@
     config :pleroma, configurable_from_database: false
     ```
 
-To delete transferred settings from database optional flag `-d` can be used. `<env>` is `prod` by default.
+Options:
+
+- `<path>` - where to save migrated config. E.g. `--path=/tmp`. If file saved into non standart folder, you must manually copy file into directory where Pleroma can read it. For OTP install path will be `PLEROMA_CONFIG_PATH` or `/etc/pleroma`. For installation from source - `config` directory in the pleroma folder.
+- `<env>` - environment, for which is migrated config. By default is `prod`.
+- To delete transferred settings from database optional flag `-d` can be used
 
 === "OTP"
     ```sh
-     ./bin/pleroma_ctl config migrate_from_db [--env=<env>] [-d]
+     ./bin/pleroma_ctl config migrate_from_db [--env=<env>] [-d] [--path=<path>]
     ```
 
 === "From Source"
     ```sh
-    mix pleroma.config migrate_from_db [--env=<env>] [-d]
+    mix pleroma.config migrate_from_db [--env=<env>] [-d] [--path=<path>]
     ```
 
 ## Dump all of the config settings defined in the database

docs/administration/CLI_tasks/database.md

@@ -141,3 +141,21 @@ but should only be run if necessary. **It is safe to cancel this.**
     ```sh
     mix pleroma.database ensure_expiration
     ```
+
+## Change Text Search Configuration
+
+Change `default_text_search_config` for database and (if necessary) text_search_config used in index, then rebuild index (it may take time). 
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl database set_text_search_config english
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.database set_text_search_config english
+    ```
+
+See [PostgreSQL documentation](https://www.postgresql.org/docs/current/textsearch-configuration.html) and `docs/configuration/howto_search_cjk.md` for more detail.

docs/administration/CLI_tasks/email.md

@@ -16,7 +16,6 @@
     mix pleroma.email test [--to <destination email address>]
     ```
 
-
 Example:
 
 === "OTP"
@@ -36,11 +35,11 @@ Example:
 === "OTP"
 
     ```sh
-     ./bin/pleroma_ctl email send_confirmation_mails
+     ./bin/pleroma_ctl email resend_confirmation_emails
     ```
 
 === "From Source"
 
     ```sh
-    mix pleroma.email send_confirmation_mails
+    mix pleroma.email resend_confirmation_emails
     ```

docs/administration/CLI_tasks/user.md

@@ -133,22 +133,20 @@
     mix pleroma.user sign_out <nickname>
     ```
 
-
+## Activate a user
-## Deactivate or activate a user
 
 === "OTP"
 
     ```sh
-     ./bin/pleroma_ctl user toggle_activated <nickname>
+     ./bin/pleroma_ctl user activate NICKNAME
     ```
 
 === "From Source"
 
     ```sh
-    mix pleroma.user toggle_activated <nickname>
+    mix pleroma.user activate NICKNAME
     ```
 
-
 ## Deactivate a user and unsubscribes local users from the user
 
 === "OTP"
@@ -264,13 +262,13 @@
 === "OTP"
 
     ```sh
-     ./bin/pleroma_ctl user toggle_confirmed <nickname>
+     ./bin/pleroma_ctl user confirm <nickname>
     ```
 
 === "From Source"
 
     ```sh
-    mix pleroma.user toggle_confirmed <nickname>
+    mix pleroma.user confirm <nickname>
     ```
 
 ## Set confirmation status for all regular active users

docs/clients.md

@@ -17,7 +17,7 @@ Feel free to contact us to be added to this list!
 - Features: MastoAPI
 
 ### Whalebird
-- Homepage: <https://whalebird.org/>
+- Homepage: <https://whalebird.social/>
 - Source Code: <https://github.com/h3poteto/whalebird-desktop>
 - Contact: [@h3poteto@pleroma.io](https://pleroma.io/users/h3poteto)
 - Platforms: Windows, Mac, Linux

docs/configuration/auth.md

@@ -0,0 +1 @@
+See `Authentication` section of [the configuration cheatsheet](../configuration/cheatsheet.md#authentication).

docs/configuration/cheatsheet.md

@@ -49,7 +49,7 @@ To add configuration to your config file, you can copy it from the base config.
 * `attachment_links`: Set to true to enable automatically adding attachment link text to statuses.
 * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`).
 * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). Default: `false`.
-* `healthcheck`: If set to true, system data will be shown on ``/api/pleroma/healthcheck``.
+* `healthcheck`: If set to true, system data will be shown on ``/api/v1/pleroma/healthcheck``.
 * `remote_post_retention_days`: The default amount of days to retain remote posts when pruning the database.
 * `user_bio_length`: A user bio maximum length (default: `5000`).
 * `user_name_length`: A user name maximum length (default: `100`).
@@ -65,6 +65,13 @@ To add configuration to your config file, you can copy it from the base config.
 * `show_reactions`: Let favourites and emoji reactions be viewed through the API (default: `true`).
 * `password_reset_token_validity`: The time after which reset tokens aren't accepted anymore, in seconds (default: one day).
 
+## :database
+* `improved_hashtag_timeline`: Setting to force toggle / force disable improved hashtags timeline. `:enabled` forces hashtags to be fetched from `hashtags` table for hashtags timeline. `:disabled` forces object-embedded hashtags to be used (slower). Keep it `:auto` for automatic behaviour (it is auto-set to `:enabled` [unless overridden] when HashtagsTableMigrator completes).
+
+## Background migrations
+* `populate_hashtags_table/sleep_interval_ms`: Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances).
+* `populate_hashtags_table/fault_rate_allowance`: Max rate of failed objects to actually processed objects in order to enable the feature (any value from 0.0 which tolerates no errors to 1.0 which will enable the feature even if hashtags transfer failed for all records).
+
 ## Welcome
 * `direct_message`: - welcome message sent as a direct message.
   * `enabled`: Enables the send a direct message to a newly registered user. Defaults to `false`.
@@ -117,6 +124,7 @@ To add configuration to your config file, you can copy it from the base config.
     * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)).
     * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.Workers.PurgeExpiredActivity` to be enabled for processing the scheduled delections.
     * `Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy`: Makes all bot posts to disappear from public timelines.
+    * `Pleroma.Web.ActivityPub.MRF.FollowBotPolicy`: Automatically follows newly discovered users from the specified bot account. Local accounts, locked accounts, and users with "#nobot" in their bio are respected and excluded from being followed.
 * `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
 * `transparency_exclusions`: Exclude specific instance names from MRF transparency.  The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.
 
@@ -203,6 +211,21 @@ config :pleroma, :mrf_user_allowlist, %{
 
 * `days`: Default global expiration time for all local Create activities (in days)
 
+#### :mrf_hashtag
+
+* `sensitive`: List of hashtags to mark activities as sensitive (default: `nsfw`)
+* `federated_timeline_removal`: List of hashtags to remove activities from the federated timeline (aka TWNK)
+* `reject`: List of hashtags to reject activities from
+
+Notes:
+- The hashtags in the configuration do not have a leading `#`.
+- This MRF Policy is always enabled, if you want to disable it you have to set empty lists
+
+#### :mrf_follow_bot
+
+* `follower_nickname`: The name of the bot account to use for following newly discovered users. Using `followbot` or similar is strongly suggested.
+
+
 ### :activitypub
 * `unfollow_blocked`: Whether blocks result in people getting unfollowed
 * `outgoing_blocks`: Whether to federate blocks to other instances
@@ -225,7 +248,7 @@ config :pleroma, :mrf_user_allowlist, %{
 
 This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` and `masto_fe` are configured. You can find the documentation for `pleroma_fe` configuration into [Pleroma-FE configuration and customization for instance administrators](/frontend/CONFIGURATION/#options).
 
-Frontends can access these settings at `/api/pleroma/frontend_configurations`
+Frontends can access these settings at `/api/v1/pleroma/frontend_configurations`
 
 To add your own configuration for PleromaFE, use it like this:
 
@@ -321,9 +344,10 @@ This section describe PWA manifest instance-specific values. Currently this opti
 #### Pleroma.Web.MediaProxy.Invalidation.Script
 
 This strategy allow perform external shell script to purge cache.
-Urls of attachments pass to script as arguments.
+Urls of attachments are passed to the script as arguments.
 
-* `script_path`: path to external script.
+* `script_path`: Path to the external script.
+* `url_format`: Set to `:htcacheclean` if using Apache's htcacheclean utility.
 
 Example:
 
@@ -549,7 +573,7 @@ the source code is here: [kocaptcha](https://github.com/koto-bank/kocaptcha). Th
 * `uploader`: Which one of the [uploaders](#uploaders) to use.
 * `filters`: List of [upload filters](#upload-filters) to use.
 * `link_name`: When enabled Pleroma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers when using filters like `Pleroma.Upload.Filter.Dedupe`
-* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host.
+* `base_url`: The base URL to access a user-uploaded file. Useful when you want to host the media files via another domain or are using a 3rd party S3 provider.
 * `proxy_remote`: If you're using a remote uploader, Pleroma will proxy media requests instead of redirecting to it.
 * `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation.
 * `filename_display_max_length`: Set max length of a filename to display. 0 = no limit. Default: 30.
@@ -570,10 +594,7 @@ Don't forget to configure [Ex AWS S3](#ex-aws-s3-settings)
 
 * `bucket`: S3 bucket name.
 * `bucket_namespace`: S3 bucket namespace.
-* `public_endpoint`: S3 endpoint that the user finally accesses(ex. "https://s3.dualstack.ap-northeast-1.amazonaws.com")
 * `truncated_namespace`: If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or "" etc.
-For example, when using CDN to S3 virtual host format, set "".
-At this time, write CNAME to CDN in public_endpoint.
 * `streaming_enabled`: Enable streaming uploads, when enabled the file will be sent to the server in chunks as it's being read. This may be unsupported by some providers, try disabling this if you have upload problems.
 
 #### Ex AWS S3 settings
@@ -850,13 +871,13 @@ config :pleroma, :admin_token, "somerandomtoken"
 You can then do
 
 ```shell
-curl "http://localhost:4000/api/pleroma/admin/users/invites?admin_token=somerandomtoken"
+curl "http://localhost:4000/api/v1/pleroma/admin/users/invites?admin_token=somerandomtoken"
 ```
 
 or
 
 ```shell
-curl -H "X-Admin-Token: somerandomtoken" "http://localhost:4000/api/pleroma/admin/users/invites"
+curl -H "X-Admin-Token: somerandomtoken" "http://localhost:4000/api/v1/pleroma/admin/users/invites"
 ```
 
 Warning: it's discouraged to use this feature because of the associated security risk: static / rarely changed instance-wide token is much weaker compared to email-password pair of a real admin user; consider using HTTP Basic Auth or OAuth-based authentication instead.
@@ -895,6 +916,22 @@ Pleroma account will be created with the same name as the LDAP user name.
 Note, if your LDAP server is an Active Directory server the correct value is commonly `uid: "cn"`, but if you use an
 OpenLDAP server the value may be `uid: "uid"`.
 
+### :oauth2 (Pleroma as OAuth 2.0 provider settings)
+
+OAuth 2.0 provider settings:
+
+* `token_expires_in` - The lifetime in seconds of the access token.
+* `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token.
+* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`.
+
+OAuth 2.0 provider and related endpoints:
+
+* `POST /api/v1/apps` creates client app basing on provided params.
+* `GET/POST /oauth/authorize` renders/submits authorization form.
+* `POST /oauth/token` creates/renews OAuth token.
+* `POST /oauth/revoke` revokes provided OAuth token.
+* `GET /api/v1/accounts/verify_credentials` (with proper `Authorization` header or `access_token` URI param) returns user info on requester (with `acct` field containing local nickname and `fqn` field containing fully-qualified nickname which could generally be used as email stub for OAuth software that demands email field in identity endpoint response, like Peertube).
+
 ### OAuth consumer mode
 
 OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.).
@@ -967,14 +1004,6 @@ config :ueberauth, Ueberauth,
   ]
 ```
 
-### OAuth 2.0 provider - :oauth2
-
-Configure OAuth 2 provider capabilities:
-
-* `token_expires_in` - The lifetime in seconds of the access token.
-* `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token.
-* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`.
-
 ## Link parsing
 
 ### :uri_schemes
@@ -1113,3 +1142,15 @@ Settings to enable and configure expiration for ephemeral activities
 
 * `:enabled` - enables ephemeral activities creation
 * `:min_lifetime` - minimum lifetime for ephemeral activities (in seconds). Default: 10 minutes.
+
+## ConcurrentLimiter
+
+Settings to restrict concurrently running jobs. Jobs which can be configured:
+
+* `Pleroma.Web.RichMedia.Helpers` - generating link previews of URLs in activities
+* `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy` - warming remote media cache via MediaProxyWarmingPolicy
+
+Each job has these settings:
+
+* `:max_running` - max concurrently runnings jobs
+* `:max_waiting` - max waiting jobs

docs/configuration/howto_search_cjk.md

@@ -0,0 +1,42 @@
+# How to enable text search for Chinese, Japanese and Korean
+
+Pleroma's full text search feature is powered by PostgreSQL's native [text search](https://www.postgresql.org/docs/current/textsearch.html), it works well out of box for most of languages, but needs extra configurations for some asian languages like Chinese, Japanese and Korean (CJK).
+
+
+## Setup and test the new search config
+
+In most cases, you would need an extension installed to support parsing CJK text. Here are a few extension you may choose from, or you are more than welcome to share additional ones you found working for you with the rest of Pleroma community.
+
+ * [a generic n-gram parser](https://github.com/huangjimmy/pg_cjk_parser) supports Simplifed/Traditional Chinese, Japanese, and Korean
+ * [a Korean parser](https://github.com/i0seph/textsearch_ko) based on mecab
+ * [a Japanese parser](https://www.amris.co.jp/tsja/index.html) based on mecab
+ * [zhparser](https://github.com/amutu/zhparser/) is a PostgreSQL extension base on the Simple Chinese Word Segmentation(SCWS)
+ * [another Chinese parser](https://github.com/jaiminpan/pg_jieba) based on Jieba Chinese Word Segmentation
+ 
+Once you have the new search config , make sure you test it with the `pleroma` user in PostgreSQL (change `YOUR.CONFIG` to your real configuration name)
+```
+SELECT ts_debug('YOUR.CONFIG', '安装和配置Nginx, ElixirとErlangをインストールします');
+```
+Check output of the query, and see if it matches your expectation.
+
+
+## Update text search config and index in database
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl database set_text_search_config YOUR.CONFIG
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.database set_text_search_config YOUR.CONFIG
+    ```
+
+Note: index update may take a while.
+
+## Restart database connection
+Since some changes above will only apply with a new database connection, you will have to restart either Pleroma or PostgreSQL process, or use `pg_terminate_backend` SQL command without restarting either. 
+
+Now the search results of statuses should be much more friendly for your language of choice, the results for searching users and tags were not changed, as the default parsing/matching should work for most cases. 

docs/configuration/mrf.md

@@ -133,3 +133,26 @@ config :pleroma, :mrf,
 ```
 
 Please note that the Pleroma developers consider custom MRF policy modules to fall under the purview of the AGPL. As such, you are obligated to release the sources to your custom MRF policy modules upon request.
+
+### MRF policies descriptions
+
+If MRF policy depends on config, it can be added into MRF tab to adminFE by adding `config_description/0` method, which returns a map with a specific structure. See existing MRF's like `lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex` for examples. Note that more complex inputs, like tuples or maps, may need extra changes in the adminFE and just adding it to `config_description/0` may not be enough to get these inputs working from the adminFE.
+
+Example:
+
+```elixir
+%{
+      key: :mrf_activity_expiration,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy",
+      label: "MRF Activity Expiration Policy",
+      description: "Adds automatic expiration to all local activities",
+      children: [
+        %{
+          key: :days,
+          type: :integer,
+          description: "Default global expiration time for all local activities (in days)",
+          suggestions: [90, 365]
+        }
+      ]
+    }
+```

docs/configuration/postgresql.md

@@ -1,10 +1,28 @@
-# Optimizing your PostgreSQL performance
+# Optimizing PostgreSQL performance
 
-Pleroma performance depends to a large extent on good database performance. The default PostgreSQL settings are mostly fine, but often you can get better performance by changing a few settings.
+Pleroma performance is largely dependent on performance of the underlying database. Better performance can be achieved by adjusting a few settings.
 
-You can use [PGTune](https://pgtune.leopard.in.ua) to get recommendations for your setup. If you do, set the "Number of Connections" field to 20, as Pleroma will only use 10 concurrent connections anyway. If you don't, it will give you advice that might even hurt your performance.
+## PGTune
 
-We also recommend not using the "Network Storage" option.
+[PgTune](https://pgtune.leopard.in.ua) can be used to get recommended settings. Be sure to set "Number of Connections" to 20, otherwise it might produce settings hurtful to database performance. It is also recommended to not use "Network Storage" option.
+
+## Disable generic query plans
+
+When PostgreSQL receives a query, it decides on a strategy for searching the requested data, this is called a query plan. The query planner has two modes: generic and custom. Generic makes a plan for all queries of the same shape, ignoring the parameters, which is then cached and reused. Custom, on the contrary, generates a unique query plan based on query parameters.
+
+By default PostgreSQL has an algorithm to decide which mode is more efficient for particular query, however this algorithm has been observed to be wrong on some of the queries Pleroma sends, leading to serious performance loss. Therefore, it is recommended to disable generic mode.
+
+
+Pleroma already avoids generic query plans by default, however the method it uses is not the most efficient because it needs to be compatible with all supported PostgreSQL versions. For PostgreSQL 12 and higher additional performance can be gained by adding the following to Pleroma configuration:
+```elixir
+config :pleroma, Pleroma.Repo,
+  prepare: :named,
+  parameters: [
+    plan_cache_mode: "force_custom_plan"
+  ]
+```
+
+A more detailed explaination of the issue can be found at <https://blog.soykaf.com/post/postgresql-elixir-troubles/>.
 
 ## Example configurations
 
@@ -28,4 +46,3 @@ max_worker_processes = 2
 max_parallel_workers_per_gather = 1
 max_parallel_workers = 2
 ```
-

docs/development/API/admin_api.md

@@ -2,14 +2,9 @@
 
 Authentication is required and the user must be an admin.
 
-Configuration options:
+The `/api/v1/pleroma/admin/*` path is backwards compatible with `/api/pleroma/admin/*` (`/api/pleroma/admin/*` will be deprecated in the future).
 
-* `[:auth, :enforce_oauth_admin_scope_usage]` — OAuth admin scope requirement toggle.
+## `GET /api/v1/pleroma/admin/users`
-    If `true`, admin actions explicitly demand admin OAuth scope(s) presence in OAuth token (client app must support admin scopes).
-    If `false` and token doesn't have admin scope(s), `is_admin` user flag grants access to admin-specific actions.
-    Note that client app needs to explicitly support admin scopes and request them when obtaining auth token.
-
-## `GET /api/pleroma/admin/users`
 
 ### List users
 
@@ -30,7 +25,7 @@ Configuration options:
   - *optional* `actor_types`: **[string]** actor type list (`Person`, `Service`, `Application`)
   - *optional* `name`: **string** user display name
   - *optional* `email`: **string** user email
-- Example: `https://mypleroma.org/api/pleroma/admin/users?query=john&filters=local,active&page=1&page_size=10&tags[]=some_tag&tags[]=another_tag&name=display_name&email=email@example.com`
+- Example: `https://mypleroma.org/api/v1/pleroma/admin/users?query=john&filters=local,active&page=1&page_size=10&tags[]=some_tag&tags[]=another_tag&name=display_name&email=email@example.com`
 - Response:
 
 ```json
@@ -59,7 +54,7 @@ Configuration options:
 }
 ```
 
-## DEPRECATED `DELETE /api/pleroma/admin/users`
+## DEPRECATED `DELETE /api/v1/pleroma/admin/users`
 
 ### Remove a user
 
@@ -67,7 +62,7 @@ Configuration options:
   - `nickname`
 - Response: User’s nickname
 
-## `DELETE /api/pleroma/admin/users`
+## `DELETE /api/v1/pleroma/admin/users`
 
 ### Remove a user
 
@@ -88,7 +83,7 @@ Configuration options:
   ]
 - Response: User’s nickname
 
-## `POST /api/pleroma/admin/users/follow`
+## `POST /api/v1/pleroma/admin/users/follow`
 
 ### Make a user follow another user
 
@@ -98,7 +93,7 @@ Configuration options:
 - Response:
   - "ok"
 
-## `POST /api/pleroma/admin/users/unfollow`
+## `POST /api/v1/pleroma/admin/users/unfollow`
 
 ### Make a user unfollow another user
 
@@ -108,7 +103,7 @@ Configuration options:
 - Response:
   - "ok"
 
-## `PATCH /api/pleroma/admin/users/:nickname/toggle_activation`
+## `PATCH /api/v1/pleroma/admin/users/:nickname/toggle_activation`
 
 ### Toggle user activation
 
@@ -124,7 +119,7 @@ Configuration options:
 }
 ```
 
-## `PUT /api/pleroma/admin/users/tag`
+## `PUT /api/v1/pleroma/admin/users/tag`
 
 ### Tag a list of users
 
@@ -132,7 +127,7 @@ Configuration options:
   - `nicknames` (array)
   - `tags` (array)
 
-## `DELETE /api/pleroma/admin/users/tag`
+## `DELETE /api/v1/pleroma/admin/users/tag`
 
 ### Untag a list of users
 
@@ -140,7 +135,7 @@ Configuration options:
   - `nicknames` (array)
   - `tags` (array)
 
-## `GET /api/pleroma/admin/users/:nickname/permission_group`
+## `GET /api/v1/pleroma/admin/users/:nickname/permission_group`
 
 ### Get user user permission groups membership
 
@@ -154,7 +149,7 @@ Configuration options:
 }
 ```
 
-## `GET /api/pleroma/admin/users/:nickname/permission_group/:permission_group`
+## `GET /api/v1/pleroma/admin/users/:nickname/permission_group/:permission_group`
 
 Note: Available `:permission_group` is currently moderator and admin. 404 is returned when the permission group doesn’t exist.
 
@@ -170,7 +165,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 }
 ```
 
-## DEPRECATED `POST /api/pleroma/admin/users/:nickname/permission_group/:permission_group`
+## DEPRECATED `POST /api/v1/pleroma/admin/users/:nickname/permission_group/:permission_group`
 
 ### Add user to permission group
 
@@ -179,7 +174,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - On failure: `{"error": "…"}`
   - On success: JSON of the user
 
-## `POST /api/pleroma/admin/users/permission_group/:permission_group`
+## `POST /api/v1/pleroma/admin/users/permission_group/:permission_group`
 
 ### Add users to permission group
 
@@ -189,9 +184,9 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - On failure: `{"error": "…"}`
   - On success: JSON of the user
 
-## DEPRECATED `DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group`
+## DEPRECATED `DELETE /api/v1/pleroma/admin/users/:nickname/permission_group/:permission_group`
 
-## `DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group`
+## `DELETE /api/v1/pleroma/admin/users/:nickname/permission_group/:permission_group`
 
 ### Remove user from permission group
 
@@ -201,7 +196,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - On success: JSON of the user
 - Note: An admin cannot revoke their own admin status.
 
-## `DELETE /api/pleroma/admin/users/permission_group/:permission_group`
+## `DELETE /api/v1/pleroma/admin/users/permission_group/:permission_group`
 
 ### Remove users from permission group
 
@@ -212,7 +207,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - On success: JSON of the user
 - Note: An admin cannot revoke their own admin status.
 
-## `PATCH /api/pleroma/admin/users/activate`
+## `PATCH /api/v1/pleroma/admin/users/activate`
 
 ### Activate user
 
@@ -230,7 +225,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 }
 ```
 
-## `PATCH /api/pleroma/admin/users/deactivate`
+## `PATCH /api/v1/pleroma/admin/users/deactivate`
 
 ### Deactivate user
 
@@ -248,7 +243,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 }
 ```
 
-## `PATCH /api/pleroma/admin/users/approve`
+## `PATCH /api/v1/pleroma/admin/users/approve`
 
 ### Approve user
 
@@ -266,7 +261,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 }
 ```
 
-## `GET /api/pleroma/admin/users/:nickname_or_id`
+## `GET /api/v1/pleroma/admin/users/:nickname_or_id`
 
 ### Retrive the details of a user
 
@@ -276,7 +271,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - On failure: `Not found`
   - On success: JSON of the user
 
-## `GET /api/pleroma/admin/users/:nickname_or_id/statuses`
+## `GET /api/v1/pleroma/admin/users/:nickname_or_id/statuses`
 
 ### Retrive user's latest statuses
 
@@ -287,9 +282,20 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - *optional* `with_reblogs`: `true`/`false` – allows to see reblogs (default is false)
 - Response:
   - On failure: `Not found`
-  - On success: JSON array of user's latest statuses
+ - On success: JSON, where:
+    - `total`: total count of the statuses for the user
+    - `activities`: list of the statuses for the user
 
-## `GET /api/pleroma/admin/instances/:instance/statuses`
+```json
+{
+  "total" : 1,
+  "activities": [
+    // activities list
+  ]
+}
+```
+
+## `GET /api/v1/pleroma/admin/instances/:instance/statuses`
 
 ### Retrive instance's latest statuses
 
@@ -300,9 +306,20 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - *optional* `with_reblogs`: `true`/`false` – allows to see reblogs (default is false)
 - Response:
   - On failure: `Not found`
-  - On success: JSON array of instance's latest statuses
+  - On success: JSON, where:
+    - `total`: total count of the statuses for the instance
+    - `activities`: list of the statuses for the instance
 
-## `GET /api/pleroma/admin/statuses`
+```json
+{
+  "total" : 1,
+  "activities": [
+    // activities list
+  ]
+}
+```
+
+## `GET /api/v1/pleroma/admin/statuses`
 
 ### Retrives all latest statuses
 
@@ -315,7 +332,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - On failure: `Not found`
   - On success: JSON array of user's latest statuses
 
-## `GET /api/pleroma/admin/relay`
+## `GET /api/v1/pleroma/admin/relay`
 
 ### List Relays
 
@@ -331,7 +348,7 @@ Response:
 ]
 ```
 
-## `POST /api/pleroma/admin/relay`
+## `POST /api/v1/pleroma/admin/relay`
 
 ### Follow a Relay
 
@@ -347,7 +364,7 @@ Response:
 {"actor": "https://example.com/relay", "followed_back": true}
 ```
 
-## `DELETE /api/pleroma/admin/relay`
+## `DELETE /api/v1/pleroma/admin/relay`
 
 ### Unfollow a Relay
 
@@ -363,7 +380,7 @@ Response:
 {"https://example.com/relay"}
 ```
 
-## `POST /api/pleroma/admin/users/invite_token`
+## `POST /api/v1/pleroma/admin/users/invite_token`
 
 ### Create an account registration invite token
 
@@ -384,7 +401,7 @@ Response:
 }
 ```
 
-## `GET /api/pleroma/admin/users/invites`
+## `GET /api/v1/pleroma/admin/users/invites`
 
 ### Get a list of generated invites
 
@@ -409,7 +426,7 @@ Response:
 }
 ```
 
-## `POST /api/pleroma/admin/users/revoke_invite`
+## `POST /api/v1/pleroma/admin/users/revoke_invite`
 
 ### Revoke invite by token
 
@@ -430,7 +447,7 @@ Response:
 }
 ```
 
-## `POST /api/pleroma/admin/users/email_invite`
+## `POST /api/v1/pleroma/admin/users/email_invite`
 
 ### Sends registration invite via email
 
@@ -451,7 +468,7 @@ Response:
       ]
     ```
 
-## `GET /api/pleroma/admin/users/:nickname/password_reset`
+## `GET /api/v1/pleroma/admin/users/:nickname/password_reset`
 
 ### Get a password reset token for a given nickname
 
@@ -462,11 +479,11 @@ Response:
 ```json
 {
   "token": "base64 reset token",
-  "link": "https://pleroma.social/api/pleroma/password_reset/url-encoded-base64-token"
+  "link": "https://pleroma.social/api/v1/pleroma/password_reset/url-encoded-base64-token"
 }
 ```
 
-## `PATCH /api/pleroma/admin/users/force_password_reset`
+## `PATCH /api/v1/pleroma/admin/users/force_password_reset`
 
 ### Force passord reset for a user with a given nickname
 
@@ -474,7 +491,7 @@ Response:
   - `nicknames`
 - Response: none (code `204`)
 
-## PUT `/api/pleroma/admin/users/disable_mfa`
+## PUT `/api/v1/pleroma/admin/users/disable_mfa`
 
 ### Disable mfa for user's account.
 
@@ -482,7 +499,7 @@ Response:
   - `nickname`
 - Response: User’s nickname
 
-## `GET /api/pleroma/admin/users/:nickname/credentials`
+## `GET /api/v1/pleroma/admin/users/:nickname/credentials`
 
 ### Get the user's email, password, display and settings-related fields
 
@@ -530,7 +547,7 @@ Response:
 }
 ```
 
-## `PATCH /api/pleroma/admin/users/:nickname/credentials`
+## `PATCH /api/v1/pleroma/admin/users/:nickname/credentials`
 
 ### Change the user's email, password, display and settings-related fields
 
@@ -581,7 +598,7 @@ Status: 404
 {"error": "Not found"}
 ```
 
-## `GET /api/pleroma/admin/reports`
+## `GET /api/v1/pleroma/admin/reports`
 
 ### Get a list of reports
 
@@ -741,17 +758,17 @@ Status: 404
 }
 ```
 
-## `GET /api/pleroma/admin/grouped_reports`
+## `GET /api/v1/pleroma/admin/grouped_reports`
 
 ### Get a list of reports, grouped by status
 
 - Params: none
 - On success: JSON, returns a list of reports, where:
   - `date`: date of the latest report
-  - `account`: the user who has been reported (see `/api/pleroma/admin/reports` for reference)
+  - `account`: the user who has been reported (see `/api/v1/pleroma/admin/reports` for reference)
-  - `status`: reported status (see `/api/pleroma/admin/reports` for reference)
+  - `status`: reported status (see `/api/v1/pleroma/admin/reports` for reference)
-  - `actors`: users who had reported this status (see `/api/pleroma/admin/reports` for reference)
+  - `actors`: users who had reported this status (see `/api/v1/pleroma/admin/reports` for reference)
-  - `reports`: reports (see `/api/pleroma/admin/reports` for reference)
+  - `reports`: reports (see `/api/v1/pleroma/admin/reports` for reference)
 
 ```json
   "reports": [
@@ -765,7 +782,7 @@ Status: 404
   ]
 ```
 
-## `GET /api/pleroma/admin/reports/:id`
+## `GET /api/v1/pleroma/admin/reports/:id`
 
 ### Get an individual report
 
@@ -777,7 +794,7 @@ Status: 404
     - 404 Not Found `"Not found"`
   - On success: JSON, Report object (see above)
 
-## `PATCH /api/pleroma/admin/reports`
+## `PATCH /api/v1/pleroma/admin/reports`
 
 ### Change the state of one or multiple reports
 
@@ -808,7 +825,7 @@ Status: 404
 
   - On success: `204`, empty response
 
-## `POST /api/pleroma/admin/reports/:id/notes`
+## `POST /api/v1/pleroma/admin/reports/:id/notes`
 
 ### Create report note
 
@@ -820,7 +837,7 @@ Status: 404
     - 400 Bad Request `"Invalid parameters"` when `status` is missing
   - On success: `204`, empty response
 
-## `DELETE /api/pleroma/admin/reports/:report_id/notes/:id`
+## `DELETE /api/v1/pleroma/admin/reports/:report_id/notes/:id`
 
 ### Delete report note
 
@@ -832,7 +849,7 @@ Status: 404
     - 400 Bad Request `"Invalid parameters"` when `status` is missing
   - On success: `204`, empty response
 
-## `GET /api/pleroma/admin/statuses/:id`
+## `GET /api/v1/pleroma/admin/statuses/:id`
 
 ### Show status by id
 
@@ -843,7 +860,7 @@ Status: 404
     - 404 Not Found `"Not Found"`
   - On success: JSON, Mastodon Status entity
 
-## `PUT /api/pleroma/admin/statuses/:id`
+## `PUT /api/v1/pleroma/admin/statuses/:id`
 
 ### Change the scope of an individual reported status
 
@@ -858,7 +875,7 @@ Status: 404
     - 404 Not Found `"Not found"`
   - On success: JSON, Mastodon Status entity
 
-## `DELETE /api/pleroma/admin/statuses/:id`
+## `DELETE /api/v1/pleroma/admin/statuses/:id`
 
 ### Delete an individual reported status
 
@@ -870,7 +887,7 @@ Status: 404
     - 404 Not Found `"Not found"`
   - On success: 200 OK `{}`
 
-## `GET /api/pleroma/admin/restart`
+## `GET /api/v1/pleroma/admin/restart`
 
 ### Restarts pleroma application
 
@@ -885,7 +902,7 @@ Status: 404
 {}
 ```
 
-## `GET /api/pleroma/admin/need_reboot`
+## `GET /api/v1/pleroma/admin/need_reboot`
 
 ### Returns the flag whether the pleroma should be restarted
 
@@ -898,7 +915,7 @@ Status: 404
 }
 ```
 
-## `GET /api/pleroma/admin/config`
+## `GET /api/v1/pleroma/admin/config`
 
 ### Get list of merged default settings with saved in database.
 
@@ -925,7 +942,7 @@ Status: 404
 }
 ```
 
-## `POST /api/pleroma/admin/config`
+## `POST /api/v1/pleroma/admin/config`
 
 ### Update config settings
 
@@ -1074,7 +1091,7 @@ config :quack,
 }
 ```
 
-## ` GET /api/pleroma/admin/config/descriptions`
+## ` GET /api/v1/pleroma/admin/config/descriptions`
 
 ### Get JSON with config descriptions.
 Loads json generated from `config/descriptions.exs`.
@@ -1107,7 +1124,7 @@ Loads json generated from `config/descriptions.exs`.
 }]
 ```
 
-## `GET /api/pleroma/admin/moderation_log`
+## `GET /api/v1/pleroma/admin/moderation_log`
 
 ### Get moderation log
 
@@ -1123,6 +1140,7 @@ Loads json generated from `config/descriptions.exs`.
 ```json
 [
   {
+    "id": 1234,
     "data": {
       "actor": {
         "id": 1,
@@ -1136,7 +1154,7 @@ Loads json generated from `config/descriptions.exs`.
 ]
 ```
 
-## `POST /api/pleroma/admin/reload_emoji`
+## `POST /api/v1/pleroma/admin/reload_emoji`
 
 ### Reload the instance's custom emoji
 
@@ -1144,7 +1162,7 @@ Loads json generated from `config/descriptions.exs`.
 - Params: None
 - Response: JSON, "ok" and 200 status
 
-## `PATCH /api/pleroma/admin/users/confirm_email`
+## `PATCH /api/v1/pleroma/admin/users/confirm_email`
 
 ### Confirm users' emails
 
@@ -1152,7 +1170,7 @@ Loads json generated from `config/descriptions.exs`.
   - `nicknames`
 - Response: Array of user nicknames
 
-## `PATCH /api/pleroma/admin/users/resend_confirmation_email`
+## `PATCH /api/v1/pleroma/admin/users/resend_confirmation_email`
 
 ### Resend confirmation email
 
@@ -1160,13 +1178,13 @@ Loads json generated from `config/descriptions.exs`.
   - `nicknames`
 - Response: Array of user nicknames
 
-## `GET /api/pleroma/admin/stats`
+## `GET /api/v1/pleroma/admin/stats`
 
 ### Stats
 
 - Query Params:
   - *optional* `instance`: **string** instance hostname (without protocol) to get stats for
-- Example: `https://mypleroma.org/api/pleroma/admin/stats?instance=lain.com`
+- Example: `https://mypleroma.org/api/v1/pleroma/admin/stats?instance=lain.com`
 
 - Response:
 
@@ -1181,7 +1199,7 @@ Loads json generated from `config/descriptions.exs`.
 }
 ```
 
-## `GET /api/pleroma/admin/oauth_app`
+## `GET /api/v1/pleroma/admin/oauth_app`
 
 ### List OAuth app
 
@@ -1213,7 +1231,7 @@ Loads json generated from `config/descriptions.exs`.
 ```
 
 
-## `POST /api/pleroma/admin/oauth_app`
+## `POST /api/v1/pleroma/admin/oauth_app`
 
 ### Create OAuth App
 
@@ -1246,7 +1264,7 @@ Loads json generated from `config/descriptions.exs`.
 }
 ```
 
-## `PATCH /api/pleroma/admin/oauth_app/:id`
+## `PATCH /api/v1/pleroma/admin/oauth_app/:id`
 
 ### Update OAuth App
 
@@ -1271,7 +1289,7 @@ Loads json generated from `config/descriptions.exs`.
 }
 ```
 
-## `DELETE /api/pleroma/admin/oauth_app/:id`
+## `DELETE /api/v1/pleroma/admin/oauth_app/:id`
 
 ### Delete OAuth App
 
@@ -1282,7 +1300,7 @@ Loads json generated from `config/descriptions.exs`.
   - On failure:
     - 400 Bad Request `"Invalid parameters"` when `status` is missing
 
-## `GET /api/pleroma/admin/media_proxy_caches`
+## `GET /api/v1/pleroma/admin/media_proxy_caches`
 
 ### Get a list of all banned MediaProxy URLs in Cachex
 
@@ -1306,7 +1324,7 @@ Loads json generated from `config/descriptions.exs`.
 
 ```
 
-## `POST /api/pleroma/admin/media_proxy_caches/delete`
+## `POST /api/v1/pleroma/admin/media_proxy_caches/delete`
 
 ### Remove a banned MediaProxy URL from Cachex
 
@@ -1321,7 +1339,7 @@ Loads json generated from `config/descriptions.exs`.
 
 ```
 
-## `POST /api/pleroma/admin/media_proxy_caches/purge`
+## `POST /api/v1/pleroma/admin/media_proxy_caches/purge`
 
 ### Purge a MediaProxy URL
 
@@ -1337,7 +1355,7 @@ Loads json generated from `config/descriptions.exs`.
 
 ```
 
-## GET /api/pleroma/admin/users/:nickname/chats
+## GET /api/v1/pleroma/admin/users/:nickname/chats
 
 ### List a user's chats
 
@@ -1366,7 +1384,7 @@ Loads json generated from `config/descriptions.exs`.
 ]
 ```
 
-## GET /api/pleroma/admin/chats/:chat_id
+## GET /api/v1/pleroma/admin/chats/:chat_id
 
 ### View a single chat
 
@@ -1393,7 +1411,7 @@ Loads json generated from `config/descriptions.exs`.
 }
 ```
 
-## GET /api/pleroma/admin/chats/:chat_id/messages
+## GET /api/v1/pleroma/admin/chats/:chat_id/messages
 
 ### List the messages in a chat
 
@@ -1431,7 +1449,7 @@ Loads json generated from `config/descriptions.exs`.
 ]
 ```
 
-## DELETE /api/pleroma/admin/chats/:chat_id/messages/:message_id
+## DELETE /api/v1/pleroma/admin/chats/:chat_id/messages/:message_id
 
 ### Delete a single message
 
@@ -1458,7 +1476,7 @@ Loads json generated from `config/descriptions.exs`.
 }
 ```
 
-## `GET /api/pleroma/admin/instance_document/:document_name`
+## `GET /api/v1/pleroma/admin/instance_document/:document_name`
 
 ### Get an instance document
 
@@ -1472,7 +1490,7 @@ Returns the content of the document
 <h1>Instance panel</h1>
 ```
 
-## `PATCH /api/pleroma/admin/instance_document/:document_name`
+## `PATCH /api/v1/pleroma/admin/instance_document/:document_name`
 - Params:
   - `file` (the file to be uploaded, using multipart form data.)
 
@@ -1488,7 +1506,7 @@ Returns the content of the document
 }
 ```
 
-## `DELETE /api/pleroma/admin/instance_document/:document_name`
+## `DELETE /api/v1/pleroma/admin/instance_document/:document_name`
 
 ### Delete an instance document
 
@@ -1500,7 +1518,7 @@ Returns the content of the document
 }
 ```
 
-## `GET /api/pleroma/admin/frontends
+## `GET /api/v1/pleroma/admin/frontends
 
 ### List available frontends
 
@@ -1525,7 +1543,7 @@ Returns the content of the document
 ]
 ```
 
-## `POST /api/pleroma/admin/frontends/install`
+## `POST /api/v1/pleroma/admin/frontends/install`
 
 ### Install a frontend
 

docs/development/API/differences_in_mastoapi_responses.md

@@ -16,6 +16,12 @@ Adding the parameter `reply_visibility` to the public and home timelines queries
 
 Adding the parameter `instance=lain.com` to the public timeline will show only statuses originating from `lain.com` (or any remote instance).
 
+Home, public, hashtag & list timelines accept these parameters:
+
+- `only_media`: show only statuses with media attached
+- `local`: show only local statuses
+- `remote`: show only remote statuses
+
 ## Statuses
 
 - `visibility`: has additional possible values `list` and `local` (for local-only statuses)
@@ -32,6 +38,13 @@ Has these additional fields under the `pleroma` object:
 - `thread_muted`: true if the thread the post belongs to is muted
 - `emoji_reactions`: A list with emoji / reaction maps. The format is `{name: "☕", count: 1, me: true}`. Contains no information about the reacting users, for that use the `/statuses/:id/reactions` endpoint.
 - `parent_visible`: If the parent of this post is visible to the user or not.
+- `pinned_at`: a datetime (iso8601) when status was pinned, `null` otherwise.
+
+## Scheduled statuses
+
+Has these additional fields in `params`:
+
+- `expires_in`: the number of seconds the posted activity should expire in.
 
 ## Media Attachments
 
@@ -54,6 +67,23 @@ The `id` parameter can also be the `nickname` of the user. This only works in th
 - `/api/v1/accounts/:id`
 - `/api/v1/accounts/:id/statuses`
 
+`/api/v1/accounts/:id/statuses` endpoint accepts these parameters:
+
+- `pinned`: include only pinned statuses
+- `tagged`: with tag
+- `only_media`: include only statuses with media attached
+- `with_muted`: include statuses/reactions from muted accounts
+- `exclude_reblogs`: exclude reblogs
+- `exclude_replies`: exclude replies
+- `exclude_visibilities`: exclude visibilities
+
+Endpoints which accept `with_relationships` parameter:
+
+- `/api/v1/accounts/:id`
+- `/api/v1/accounts/:id/followers`
+- `/api/v1/accounts/:id/following`
+- `/api/v1/mutes`
+
 Has these additional fields under the `pleroma` object:
 
 - `ap_id`: nullable URL string, ActivityPub id of the user
@@ -69,12 +99,12 @@ Has these additional fields under the `pleroma` object:
 - `hide_followers_count`: boolean, true when the user has follower stat hiding enabled
 - `hide_follows_count`: boolean, true when the user has follow stat hiding enabled
 - `settings_store`: A generic map of settings for frontends. Opaque to the backend. Only returned in `/api/v1/accounts/verify_credentials` and `/api/v1/accounts/update_credentials`
-- `chat_token`: The token needed for Pleroma chat. Only returned in `/api/v1/accounts/verify_credentials`
+- `chat_token`: The token needed for Pleroma shoutbox. Only returned in `/api/v1/accounts/verify_credentials`
 - `deactivated`: boolean, true when the user is deactivated
 - `allow_following_move`: boolean, true when the user allows automatically follow moved following accounts
 - `unread_conversation_count`: The count of unread conversations. Only returned to the account owner.
 - `unread_notifications_count`: The count of unread notifications. Only returned to the account owner.
-- `notification_settings`: object, can be absent. See `/api/pleroma/notification_settings` for the parameters/keys returned.
+- `notification_settings`: object, can be absent. See `/api/v1/pleroma/notification_settings` for the parameters/keys returned.
 - `accepts_chat_messages`: boolean, but can be null if we don't have that information about a user
 - `favicon`: nullable URL string, Favicon image of the user's instance
 
@@ -206,6 +236,7 @@ Additional parameters can be added to the JSON body/Form data:
 - `pleroma_settings_store` - Opaque user settings to be saved on the backend.
 - `skip_thread_containment` - if true, skip filtering out broken threads
 - `allow_following_move` - if true, allows automatically follow moved following accounts
+- `also_known_as` - array of ActivityPub IDs, needed for following move
 - `pleroma_background_image` - sets the background image of the user. Can be set to "" (an empty string) to reset.
 - `discoverable` - if true, external services (search bots) etc. are allowed to index / list the account (regardless of this setting, user will still appear in regular search results).
 - `actor_type` - the type of this account.
@@ -225,9 +256,29 @@ This information is returned in the `/api/v1/accounts/verify_credentials` endpoi
 
 *Pleroma supports refreshing tokens.*
 
-`POST /oauth/token`
+### POST `/oauth/token`
 
-Post here request with `grant_type=refresh_token` to obtain new access token. Returns an access token.
+You can obtain access tokens for a user in a few additional ways.
+
+#### Refreshing a token
+
+To obtain a new access token from a refresh token, pass `grant_type=refresh_token` with the following extra parameters:
+
+- `refresh_token`: The refresh token.
+
+#### Getting a token with a password
+
+To obtain a token from a user's password, pass `grant_type=password` with the following extra parameters:
+
+- `username`: Username to authenticate.
+- `password`: The user's password.
+
+#### Response body
+
+Additional fields are returned in the response:
+
+- `id`: The primary key of this token in Pleroma's database.
+- `me` (user tokens only): The ActivityPub ID of the user who owns the token.
 
 ## Account Registration
 

docs/development/API/pleroma_api.md

@@ -4,7 +4,9 @@ Requests that require it can be authenticated with [an OAuth token](https://tool
 
 Request parameters can be passed via [query strings](https://en.wikipedia.org/wiki/Query_string) or as [form data](https://www.w3.org/TR/html401/interact/forms.html). Files must be uploaded as `multipart/form-data`.
 
-## `/api/pleroma/emoji`
+The `/api/v1/pleroma/*` path is backwards compatible with `/api/pleroma/*` (`/api/pleroma/*` will be deprecated in the future).
+
+## `/api/v1/pleroma/emoji`
 ### Lists the custom emoji on that server.
 * Method: `GET`
 * Authentication: not required
@@ -35,7 +37,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 ```
 * Note: Same data as Mastodon API’s `/api/v1/custom_emojis` but in a different format
 
-## `/api/pleroma/follow_import`
+## `/api/v1/pleroma/follow_import`
 ### Imports your follows, for example from a Mastodon CSV file.
 * Method: `POST`
 * Authentication: required
@@ -44,7 +46,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: HTTP 200 on success, 500 on error
 * Note: Users that can't be followed are silently skipped.
 
-## `/api/pleroma/blocks_import`
+## `/api/v1/pleroma/blocks_import`
 ### Imports your blocks.
 * Method: `POST`
 * Authentication: required
@@ -52,7 +54,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
     * `list`: STRING or FILE containing a whitespace-separated list of accounts to block
 * Response: HTTP 200 on success, 500 on error
 
-## `/api/pleroma/mutes_import`
+## `/api/v1/pleroma/mutes_import`
 ### Imports your mutes.
 * Method: `POST`
 * Authentication: required
@@ -60,7 +62,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
     * `list`: STRING or FILE containing a whitespace-separated list of accounts to mute
 * Response: HTTP 200 on success, 500 on error
 
-## `/api/pleroma/captcha`
+## `/api/v1/pleroma/captcha`
 ### Get a new captcha
 * Method: `GET`
 * Authentication: not required
@@ -68,7 +70,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: Provider specific JSON, the only guaranteed parameter is `type`
 * Example response: `{"type": "kocaptcha", "token": "whatever", "url": "https://captcha.kotobank.ch/endpoint", "seconds_valid": 300}`
 
-## `/api/pleroma/delete_account`
+## `/api/v1/pleroma/delete_account`
 ### Delete an account
 * Method `POST`
 * Authentication: required
@@ -77,7 +79,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: JSON. Returns `{"status": "success"}` if the deletion was successful, `{"error": "[error message]"}` otherwise
 * Example response: `{"error": "Invalid password."}`
 
-## `/api/pleroma/disable_account`
+## `/api/v1/pleroma/disable_account`
 ### Disable an account
 * Method `POST`
 * Authentication: required
@@ -86,21 +88,21 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: JSON. Returns `{"status": "success"}` if the account was successfully disabled, `{"error": "[error message]"}` otherwise
 * Example response: `{"error": "Invalid password."}`
 
-## `/api/pleroma/accounts/mfa`
+## `/api/v1/pleroma/accounts/mfa`
 #### Gets current MFA settings
 * method: `GET`
 * Authentication: required
 * OAuth scope: `read:security`
 * Response: JSON. Returns `{"enabled": "false", "totp": false }`
 
-## `/api/pleroma/accounts/mfa/setup/totp`
+## `/api/v1/pleroma/accounts/mfa/setup/totp`
 #### Pre-setup the MFA/TOTP method
 * method: `GET`
 * Authentication: required
 * OAuth scope: `write:security`
 * Response: JSON. Returns `{"key": [secret_key], "provisioning_uri": "[qr code uri]"  }` when successful, otherwise returns HTTP 422 `{"error": "error_msg"}`
 
-## `/api/pleroma/accounts/mfa/confirm/totp`
+## `/api/v1/pleroma/accounts/mfa/confirm/totp`
 #### Confirms & enables MFA/TOTP support for user account.
 * method: `POST`
 * Authentication: required
@@ -111,7 +113,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: JSON. Returns `{}` if the enable was successful, HTTP 422 `{"error": "[error message]"}` otherwise
 
 
-## `/api/pleroma/accounts/mfa/totp`
+## `/api/v1/pleroma/accounts/mfa/totp`
 ####  Disables MFA/TOTP method for user account.
 * method: `DELETE`
 * Authentication: required
@@ -121,14 +123,14 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: JSON. Returns `{}` if the disable was successful, HTTP 422 `{"error": "[error message]"}` otherwise
 * Example response: `{"error": "Invalid password."}`
 
-## `/api/pleroma/accounts/mfa/backup_codes`
+## `/api/v1/pleroma/accounts/mfa/backup_codes`
 ####  Generstes backup codes MFA for user account.
 * method: `GET`
 * Authentication: required
 * OAuth scope: `write:security`
 * Response: JSON. Returns `{"codes": codes}`when successful, otherwise HTTP 422 `{"error": "[error message]"}`
 
-## `/api/pleroma/admin/`
+## `/api/v1/pleroma/admin/`
 See [Admin-API](admin_api.md)
 
 ## `/api/v1/pleroma/notifications/read`
@@ -298,7 +300,7 @@ See [Admin-API](admin_api.md)
 * Note: Behaves exactly the same as `POST /api/v1/upload`.
   Can only accept images - any attempt to upload non-image files will be met with `HTTP 415 Unsupported Media Type`.
 
-## `/api/pleroma/notification_settings`
+## `/api/v1/pleroma/notification_settings`
 ### Updates user notification settings
 * Method `PUT`
 * Authentication: required
@@ -307,7 +309,7 @@ See [Admin-API](admin_api.md)
     * `hide_notification_contents`: BOOLEAN field. When set to true, it removes the contents of a message from the push notification.
 * Response: JSON. Returns `{"status": "success"}` if the update was successful, otherwise returns `{"error": "error_msg"}`
 
-## `/api/pleroma/healthcheck`
+## `/api/v1/pleroma/healthcheck`
 ### Healthcheck endpoint with additional system data.
 * Method `GET`
 * Authentication: not required
@@ -325,7 +327,7 @@ See [Admin-API](admin_api.md)
 }
 ```
 
-## `/api/pleroma/change_email`
+## `/api/v1/pleroma/change_email`
 ### Change account email
 * Method `POST`
 * Authentication: required
@@ -378,7 +380,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Params: None
 * Response: JSON, returns a list of Mastodon Conversation entities that were marked as read (200 - healthy, 503 unhealthy).
 
-## `GET /api/pleroma/emoji/pack?name=:name`
+## `GET /api/v1/pleroma/emoji/pack?name=:name`
 
 ### Get pack.json for the pack
 
@@ -397,7 +399,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 }
 ```
 
-## `POST /api/pleroma/emoji/pack?name=:name`
+## `POST /api/v1/pleroma/emoji/pack?name=:name`
 
 ### Creates an empty pack
 
@@ -407,7 +409,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
   * `name`: pack name
 * Response: JSON, "ok" and 200 status or 409 if the pack with that name already exists
 
-## `PATCH /api/pleroma/emoji/pack?name=:name`
+## `PATCH /api/v1/pleroma/emoji/pack?name=:name`
 
 ### Updates (replaces) pack metadata
 
@@ -425,7 +427,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Response: JSON, updated "metadata" section of the pack and 200 status or 400 if there was a
   problem with the new metadata (the error is specified in the "error" part of the response JSON)
 
-## `DELETE /api/pleroma/emoji/pack?name=:name`
+## `DELETE /api/v1/pleroma/emoji/pack?name=:name`
 
 ### Delete a custom emoji pack
 
@@ -435,7 +437,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
   * `name`: pack name
 * Response: JSON, "ok" and 200 status or 500 if there was an error deleting the pack
 
-## `GET /api/pleroma/emoji/packs/import`
+## `GET /api/v1/pleroma/emoji/packs/import`
 
 ### Imports packs from filesystem
 
@@ -444,7 +446,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Params: None
 * Response: JSON, returns a list of imported packs.
 
-## `GET /api/pleroma/emoji/packs/remote`
+## `GET /api/v1/pleroma/emoji/packs/remote`
 
 ### Make request to another instance for packs list
 
@@ -456,7 +458,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
   * `page_size`: page size for packs (default 50)
 * Response: JSON with the pack list, hashmap with pack name and pack contents
 
-## `POST /api/pleroma/emoji/packs/download`
+## `POST /api/v1/pleroma/emoji/packs/download`
 
 ### Download pack from another instance
 
@@ -469,7 +471,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Response: JSON, "ok" with 200 status if the pack was downloaded, or 500 if there were
   errors downloading the pack
 
-## `POST /api/pleroma/emoji/packs/files?name=:name`
+## `POST /api/v1/pleroma/emoji/packs/files?name=:name`
 
 ### Add new file to the pack
 
@@ -482,7 +484,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
   * `filename`: (*optional*) new emoji file name. If not specified will be taken from original filename.
 * Response: JSON, list of files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
-## `PATCH /api/pleroma/emoji/packs/files?name=:name`
+## `PATCH /api/v1/pleroma/emoji/packs/files?name=:name`
 
 ### Update emoji file from pack
 
@@ -496,7 +498,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
   * `force`: (*optional*) with true value to overwrite existing emoji with new shortcode
 * Response: JSON, list with updated files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
-## `DELETE /api/pleroma/emoji/packs/files?name=:name`
+## `DELETE /api/v1/pleroma/emoji/packs/files?name=:name`
 
 ### Delete emoji file from pack
 
@@ -507,7 +509,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
   * `shortcode`: emoji file shortcode
 * Response: JSON, list with updated files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
-## `GET /api/pleroma/emoji/packs`
+## `GET /api/v1/pleroma/emoji/packs`
 
 ### Lists local custom emoji packs
 
@@ -528,7 +530,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 }
 ```
 
-## `GET /api/pleroma/emoji/packs/archive?name=:name`
+## `GET /api/v1/pleroma/emoji/packs/archive?name=:name`
 
 ### Requests a local pack archive from the instance
 

docs/development/authentication_authorization.md

@@ -1,5 +1,3 @@
-This document contains notes and guidelines for Pleroma developers.
-
 # Authentication & Authorization
 
 ## OAuth token-based authentication & authorization
@@ -20,27 +18,4 @@ This document contains notes and guidelines for Pleroma developers.
 
 ## Auth-related configuration, OAuth consumer mode etc.
 
-See `Authentication` section of [the configuration cheatsheet](configuration/cheatsheet.md#authentication).
+See `Authentication` section of [the configuration cheatsheet](../configuration/cheatsheet.md#authentication).
-
-## MRF policies descriptions
-
-If MRF policy depends on config, it can be added into MRF tab to adminFE by adding `config_description/0` method, which returns map with special structure.
-
-Example:
-
-```elixir
-%{
-      key: :mrf_activity_expiration,
-      related_policy: "Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy",
-      label: "MRF Activity Expiration Policy",
-      description: "Adds automatic expiration to all local activities",
-      children: [
-        %{
-          key: :days,
-          type: :integer,
-          description: "Default global expiration time for all local activities (in days)",
-          suggestions: [90, 365]
-        }
-      ]
-    }
-```

docs/development/index.md

@@ -0,0 +1 @@
+This section contains notes and guidelines for developers.

docs/development/setting_up_pleroma_dev.md

@@ -0,0 +1,70 @@
+# Setting up a Pleroma development environment
+
+Pleroma requires some adjustments from the defaults for running the instance locally. The following should help you to get started.
+
+## Installing
+
+1. Install Pleroma as explained in [the docs](../installation/debian_based_en.md), with some exceptions:
+    * You can use your own fork of the repository and add pleroma as a remote `git remote add pleroma 'https://git.pleroma.social/pleroma/pleroma'`
+    * You can skip systemd and nginx and all that stuff
+    * No need to create a dedicated pleroma user, it's easier to just use your own user
+    * For the DB you can still choose a dedicated user, the mix tasks set it up for you so it's no extra work for you
+    * For domain you can use `localhost`
+    * instead of creating a `prod.secret.exs`, create `dev.secret.exs`
+    * No need to prefix with `MIX_ENV=prod`. We're using dev and that's the default MIX_ENV
+2. Change the dev.secret.exs
+    * Change the scheme in `config :pleroma, Pleroma.Web.Endpoint` to http (see examples below)
+    * If you want to change other settings, you can do that too
+3. You can now start the server `mix phx.server`. Once it's build and started, you can access the instance on `http://<host>:<port>` (e.g.http://localhost:4000 ) and should be able to do everything locally you normaly can.
+
+Example config to change the scheme to http. Change the port if you want to run on another port.
+```elixir
+  config :pleroma, Pleroma.Web.Endpoint,
+   url: [host: "localhost", scheme: "http", port: 4000],
+```
+
+Example config to disable captcha. This makes it a bit easier to create test-users.
+```elixir
+config :pleroma, Pleroma.Captcha,
+  enabled: false
+```
+
+Example config to change the log level to info
+```elixir
+config :logger, :console,
+  # :debug :info :warning :error
+  level: :info
+```
+
+## Testing
+
+1. Create a `test.secret.exs` file with the content as shown below
+2. Create the database user and test database.
+    1. You can use the `config/setup_db.psql` as a template. Copy the file if you want and change the database name, user and password to the values for the test-database (e.g. 'pleroma_local_test' for database and user). Then run this file like you did during installation.
+    2. The tests will try to create the Database, so we'll have to allow our test-database user to create databases, `sudo -Hu postgres psql -c "ALTER USER pleroma_local_test WITH CREATEDB;"`
+3. Run the tests with `mix test`. The tests should succeed.
+
+Example content for the `test.secret.exs` file. Feel free to use another user, database name or password, just make sure the database is dedicated for the testing environment.
+```elixir
+# Pleroma test configuration
+
+# NOTE: This file should not be committed to a repo or otherwise made public
+# without removing sensitive information.
+
+import Config
+
+config :pleroma, Pleroma.Repo,
+  username: "pleroma_local_test",
+  password: "mysuperduperpassword",
+  database: "pleroma_local_test",
+  hostname: "localhost"
+
+```
+
+## Updating
+
+Update Pleroma as explained in [the docs](../administration/updating.md). Just make sure you pull from upstream and not from your own fork.
+
+## Working on multiple branches
+
+If you develop on a separate branch, it's possible you did migrations that aren't merged into another branch you're working on. If you have multiple things you're working on, it's probably best to set up multiple pleroma's each with their own database. If you finished with a branch and want to switch back to develop to start a new branch from there, you can drop the database and recreate the database (e.g. by using `config/setup_db.psql`). The commands to drop and recreate the database can be found in [the docs](../administration/backup.md).

docs/index.md

@@ -20,7 +20,7 @@ The default front-end used by Pleroma is Pleroma-FE. You can find more informati
 
 ### Mastodon interface
 If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too!
-Just add a "/web" after your instance url (e.g. <https://pleroma.soycaf.com/web>) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC!
+Just add a "/web" after your instance url (e.g. <https://pleroma.soykaf.com/web>) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC!
 The Mastodon interface is from the Glitch-soc fork. For more information on the Mastodon interface you can check the [Mastodon](https://docs.joinmastodon.org/) and [Glitch-soc](https://glitch-soc.github.io/docs/) documentation.
 
 Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma.

docs/installation/alpine_linux_en.md

@@ -80,7 +80,7 @@ sudo /etc/init.d/postgresql start
 sudo rc-update add postgresql
 ```
 
-### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md))
 
 ```shell
 sudo apk add ffmpeg imagemagick exiftool
@@ -125,7 +125,7 @@ sudo -Hu pleroma mix deps.get
 * Check the configuration and if all looks right, rename it, so Pleroma will load it (`prod.secret.exs` for productive instance, `dev.secret.exs` for development instances):
 
 ```shell
-mv config/{generated_config.exs,prod.secret.exs}
+sudo -Hu pleroma mv config/{generated_config.exs,prod.secret.exs}
 ```
 
 * The previous command creates also the file `config/setup_db.psql`, with which you can create the database:

docs/installation/arch_linux_en.md

@@ -56,7 +56,7 @@ sudo -iu postgres initdb -D /var/lib/postgres/data
 sudo systemctl enable --now postgresql.service
 ```
 
-### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md))
 
 ```shell
 sudo pacman -S ffmpeg imagemagick perl-image-exiftool
@@ -100,7 +100,7 @@ sudo -Hu pleroma mix deps.get
 * Check the configuration and if all looks right, rename it, so Pleroma will load it (`prod.secret.exs` for productive instance, `dev.secret.exs` for development instances):
 
 ```shell
-mv config/{generated_config.exs,prod.secret.exs}
+sudo -Hu pleroma mv config/{generated_config.exs,prod.secret.exs}
 ```
 
 * The previous command creates also the file `config/setup_db.psql`, with which you can create the database:

docs/installation/debian_based_en.md

@@ -54,7 +54,7 @@ sudo apt update
 sudo apt install elixir erlang-dev erlang-nox
 ```
 
-### Optional packages: [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)
+### Optional packages: [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md)
 
 ```shell
 sudo apt install imagemagick ffmpeg libimage-exiftool-perl
@@ -98,7 +98,7 @@ sudo -Hu pleroma mix deps.get
 * Check the configuration and if all looks right, rename it, so Pleroma will load it (`prod.secret.exs` for productive instance, `dev.secret.exs` for development instances):
 
 ```shell
-mv config/{generated_config.exs,prod.secret.exs}
+sudo -Hu pleroma mv config/{generated_config.exs,prod.secret.exs}
 ```
 
 

docs/installation/debian_based_jp.md

@@ -54,7 +54,7 @@ sudo apt update
 sudo apt install elixir erlang-dev erlang-nox
 ```
 
-### オプションパッケージ: [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)
+### オプションパッケージ: [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md)
 
 ```shell
 sudo apt install imagemagick ffmpeg libimage-exiftool-perl
@@ -98,7 +98,7 @@ sudo -Hu pleroma mix pleroma.instance gen
 
 * コンフィギュレーションを確認して、もし問題なければ、ファイル名を変更してください。
 ```
-mv config/{generated_config.exs,prod.secret.exs}
+sudo -Hu pleroma mv config/{generated_config.exs,prod.secret.exs}
 ```
 
 * 先程のコマンドで、すでに `config/setup_db.psql` というファイルが作られています。このファイルをもとに、データベースを作成します。

docs/installation/freebsd_en.md

@@ -26,7 +26,7 @@ Setup the required services to automatically start at boot, using `sysrc(8)`.
 # service postgresql start
 ```
 
-### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md))
 
 ```shell
 # pkg install imagemagick ffmpeg p5-Image-ExifTool

docs/installation/netbsd_en.md

@@ -44,7 +44,7 @@ pgsql=YES
 
 First, run `# /etc/rc.d/pgsql start`. Then, `$ sudo -Hu pgsql -g pgsql createdb`.
 
-### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md))
 
 `# pkgin install ImageMagick ffmpeg4 p5-Image-ExifTool`
 

docs/installation/openbsd_en.md

@@ -27,7 +27,7 @@ Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in thi
 
 #### Optional software
 
-Per [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md):
+Per [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md):
   * ImageMagick
   * ffmpeg
   * exiftool

docs/installation/openbsd_fi.md

@@ -20,7 +20,7 @@ Asenna tarvittava ohjelmisto:
 
 #### Optional software
 
-[`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md):
+[`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md):
   * ImageMagick
   * ffmpeg
   * exiftool

docs/installation/otp_en.md

@@ -89,6 +89,8 @@ RUM indexes are an alternative indexing scheme that is not included in PostgreSQ
 #### (Optional) Performance configuration
 It is encouraged to check [Optimizing your PostgreSQL performance](../configuration/postgresql.md) document, for tips on PostgreSQL tuning.
 
+Restart PostgreSQL to apply configuration changes:
+
 === "Alpine"
     ```
     rc-service postgresql restart
@@ -99,17 +101,6 @@ It is encouraged to check [Optimizing your PostgreSQL performance](../configurat
     systemctl restart postgresql
     ```
 
-If you are using PostgreSQL 12 or higher, add this to your Ecto database configuration
-
-```elixir
-#
-config :pleroma, Pleroma.Repo,
-prepare: :named,
-parameters: [
-  plan_cache_mode: "force_custom_plan"
-]
-```
-
 ### Installing Pleroma
 ```sh
 # Create a Pleroma user
@@ -159,7 +150,7 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
 # su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
 
 # Start the instance to verify that everything is working as expected
-su pleroma -s $SHELL -lc "export $(cat /opt/pleroma/config/pleroma.env); ./bin/pleroma daemon"
+su pleroma -s $SHELL -lc "./bin/pleroma daemon"
 
 # Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly
 sleep 20 && curl http://localhost:4000/api/v1/instance
@@ -299,7 +290,7 @@ nginx -t
 
 ## Create your first user and set as admin
 ```sh
-cd /opt/pleroma/bin
+cd /opt/pleroma
 su pleroma -s $SHELL -lc "./bin/pleroma_ctl user new joeuser joeuser@sld.tld --admin"
 ```
 This will create an account withe the username of 'joeuser' with the email address of joeuser@sld.tld, and set that user's account as an admin. This will result in a link that you can paste into the browser, which logs you in and enables you to set the password.

installation/apache-cache-purge.sh.example

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+# A simple shell script to delete a media from Apache's mod_disk_cache.
+# You will likely need to setup a sudo rule like the following:
+#
+# Cmnd_Alias HTCACHECLEAN = /usr/local/sbin/htcacheclean
+# pleroma ALL=HTCACHECLEAN, NOPASSWD: HTCACHECLEAN
+#
+# Please also ensure you have enabled:
+#
+# config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, url_format: :htcacheclean
+#
+# which will correctly format the URLs passed to this script for the htcacheclean utility.
+#
+
+SCRIPTNAME=${0##*/}
+
+# mod_disk_cache directory
+CACHE_DIRECTORY="/tmp/pleroma-media-cache"
+
+## Removes an item via the htcacheclean utility
+## $1 - the filename, can be a pattern .
+## $2 - the cache directory.
+purge_item() {
+    sudo htcacheclean -v -p "${2}" "${1}"
+} # purge_item
+
+purge() {
+  for url in $@
+  do
+    echo "$SCRIPTNAME delete \`$url\` from cache ($CACHE_DIRECTORY)"
+    purge_item "$url" $CACHE_DIRECTORY
+  done
+}
+
+purge $@

installation/download-mastofe-build.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 project_id="74"
 project_branch="rebase/glitch-soc"
@@ -9,29 +9,32 @@ static_dir="instance/static"
 # project_branch="pleroma"
 # static_dir="priv/static"
 
-if [[ ! -d "${static_dir}" ]]
+if [ ! -d "${static_dir}" ]
 then
 	echo "Error: ${static_dir} directory is missing, are you sure you are running this script at the root of pleroma’s repository?"
 	exit 1
 fi
 
-last_modified="$(curl -s -I 'https://git.pleroma.social/api/v4/projects/'${project_id}'/jobs/artifacts/'${project_branch}'/download?job=build' | grep '^Last-Modified:' | cut -d: -f2-)"
+last_modified="$(curl --fail -s -I 'https://git.pleroma.social/api/v4/projects/'${project_id}'/jobs/artifacts/'${project_branch}'/download?job=build' | grep '^Last-Modified:' | cut -d: -f2-)"
 
 echo "branch:${project_branch}"
 echo "Last-Modified:${last_modified}"
 
 artifact="mastofe.zip"
 
-if [[ -e mastofe.timestamp ]] && [[ "${last_modified}" != "" ]]
+if [ "${last_modified}x" = "x" ]
 then
-	if [[ "$(cat mastofe.timestamp)" == "${last_modified}" ]]
+	echo "ERROR: Couldn't get the modification date of the latest build archive, maybe it expired, exiting..."
-	then
+	exit 1
-		echo "MastoFE is up-to-date, exiting…"
-		exit 0
-	fi
 fi
 
-curl -c - "https://git.pleroma.social/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=build" -o "${artifact}" || exit
+if [ -e mastofe.timestamp ] && [ "$(cat mastofe.timestamp)" = "${last_modified}" ]
+then
+	echo "MastoFE is up-to-date, exiting..."
+	exit 0
+fi
+
+curl --fail -c - "https://git.pleroma.social/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=build" -o "${artifact}" || exit
 
 # TODO: Update the emoji as well
 rm -fr "${static_dir}/sw.js" "${static_dir}/packs" || exit

installation/pleroma-apache.conf

@@ -1,73 +1,84 @@
-# default Apache site config for Pleroma
+# Sample Apache config for Pleroma
-#
-# needed modules: define headers proxy proxy_http proxy_wstunnel rewrite ssl
-# optional modules: cache cache_disk
 #
 # Simple installation instructions:
-# 1. Install your TLS certificate, possibly using Let's Encrypt.
+# 1. Install your TLS certificate. We recommend using Let's Encrypt via Certbot
-# 2. Replace 'example.tld' with your instance's domain wherever it appears.
+# 2. Replace 'example.tld' with your instance's domain.
-# 3. This assumes a Debian style Apache config. Copy this file to
+# 3. This assumes a Debian-style Apache config. Copy this file to
-#    /etc/apache2/sites-available/ and then add a symlink to it in
+#    /etc/apache2/sites-available/ and then activate the site by running
-#    /etc/apache2/sites-enabled/ by running 'a2ensite pleroma-apache.conf', then restart Apache.
+#    'a2ensite pleroma-apache.conf', then restart Apache.
 #
 # Optional: enable disk-based caching for the media proxy
 # For details, see https://git.pleroma.social/pleroma/pleroma/wikis/How%20to%20activate%20mediaproxy
 #
-# 1. Create the directory listed below as the CacheRoot, and make sure
+# 1. Create a directory as shown below for the CacheRoot and make sure
 #    the Apache user can write to it.
 # 2. Configure Apache's htcacheclean to clean the directory periodically.
-# 3. Run 'a2enmod cache cache_disk' and restart Apache.
+#    Your OS may provide a service you can enable to do this automatically.
 
 Define servername example.tld
 
+<IfModule !proxy_module>
+    LoadModule proxy_module libexec/apache24/mod_proxy.so
+</IfModule>
+<IfModule !proxy_http_module>
+    LoadModule proxy_http_module libexec/apache24/mod_proxy_http.so
+</IfModule>
+<IfModule !proxy_wstunnel_module>
+    LoadModule proxy_wstunnel_module libexec/apache24/mod_proxy_wstunnel.so
+</IfModule>
+<IfModule !rewrite_module>
+    LoadModule rewrite_module libexec/apache24/mod_rewrite.so
+</IfModule>
+<IfModule !ssl_module>
+    LoadModule ssl_module libexec/apache24/mod_ssl.so
+</IfModule>
+<IfModule !cache_module>
+    LoadModule cache_module libexec/apache24/mod_cache.so
+</IfModule>
+<IfModule !cache_disk_module>
+    LoadModule cache_disk_module libexec/apache24/mod_cache_disk.so
+</IfModule>
+
 ServerName ${servername}
 ServerTokens Prod
 
-ErrorLog ${APACHE_LOG_DIR}/error.log
+# If you want Pleroma-specific logs
-CustomLog ${APACHE_LOG_DIR}/access.log combined
+#ErrorLog /var/log/httpd-pleroma-error.log
+#CustomLog /var/log/httpd-pleroma-access.log combined
 
 <VirtualHost *:80>
-    Redirect permanent / https://${servername}
+    RewriteEngine on
+    RewriteCond %{SERVER_NAME} =${servername}
+    RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [END,NE,R=permanent]
 </VirtualHost>
 
 <VirtualHost *:443>
     SSLEngine on
     SSLCertificateFile      /etc/letsencrypt/live/${servername}/fullchain.pem
     SSLCertificateKeyFile   /etc/letsencrypt/live/${servername}/privkey.pem
+    # Make sure you have the certbot-apache module installed
+    Include /etc/letsencrypt/options-ssl-apache.conf
 
-    # Mozilla modern configuration, tweak to your needs
+    # Uncomment the following to enable MediaProxy caching on disk
-    SSLProtocol             all -SSLv3 -TLSv1 -TLSv1.1
+    #CacheRoot /tmp/pleroma-media-cache/
-    SSLCipherSuite          ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
-    SSLHonorCipherOrder     on
-    SSLCompression          off
-    SSLSessionTickets       off
-
-    # uncomment the following to enable mediaproxy caching on disk
-    # <IfModule mod_cache_disk.c>
-    #     CacheRoot /var/cache/apache2/mod_cache_disk
     #CacheDirLevels 1
     #CacheDirLength 2
     #CacheEnable disk /proxy
     #CacheLock on
-    # </IfModule>
+    #CacheHeader on
+    #CacheDetailHeader on
+    ## 16MB max filesize for caching, configure as desired
+    #CacheMaxFileSize 16000000
+    #CacheDefaultExpire 86400
 
     RewriteEngine On
     RewriteCond %{HTTP:Connection} Upgrade [NC]
     RewriteCond %{HTTP:Upgrade} websocket [NC]
-    RewriteRule /(.*) ws://localhost:4000/$1 [P,L]
+    RewriteRule /(.*) ws://127.0.0.1:4000/$1 [P,L]
 
+    #ProxyRequests must be off or you open your server to abuse as an open proxy
     ProxyRequests off
-    # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
-    # and `localhost.` resolves to [::0] on some systems: see issue #930
     ProxyPass / http://127.0.0.1:4000/
     ProxyPassReverse / http://127.0.0.1:4000/
-
-    RequestHeader set Host ${servername}
     ProxyPreserveHost On
 </VirtualHost>
-
-# OCSP Stapling, only in httpd 2.3.3 and later
-SSLUseStapling          on
-SSLStaplingResponderTimeout 5
-SSLStaplingReturnResponderErrors off
-SSLStaplingCache        shmcb:/var/run/ocsp(128000)

installation/pleroma.vcl

@@ -59,6 +59,13 @@ sub vcl_backend_response {
       set beresp.http.CR = beresp.http.content-range;
     }
 
+    # Bypass cache for large files
+    # 50000000 ~ 50MB
+    if (std.integer(beresp.http.content-length, 0) > 50000000) {
+       set beresp.uncacheable = true;
+       return(deliver);
+    }
+
     # Don't cache objects that require authentication
     if (beresp.http.Authorization && !beresp.http.Cache-Control ~ "public") {
       set beresp.uncacheable = true;

lib/jason_types.ex

@@ -1,9 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-Postgrex.Types.define(
-  Pleroma.PostgresTypes,
-  [] ++ Ecto.Adapters.Postgres.extensions(),
-  json: Jason
-)

lib/mix/pleroma.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Pleroma do
@@ -12,7 +12,9 @@ defmodule Mix.Pleroma do
     :cachex,
     :flake_id,
     :swoosh,
-    :timex
+    :timex,
+    :fast_html,
+    :oban
   ]
   @cachex_children ["object", "user", "scrubber", "web_resp"]
   @doc "Common functions to be reused in mix tasks"
@@ -37,12 +39,23 @@ defmodule Mix.Pleroma do
 
     Enum.each(apps, &Application.ensure_all_started/1)
 
+    oban_config = [
+      crontab: [],
+      repo: Pleroma.Repo,
+      log: false,
+      queues: [],
+      plugins: []
+    ]
+
     children =
       [
         Pleroma.Repo,
+        Pleroma.Emoji,
         {Pleroma.Config.TransferTask, false},
         Pleroma.Web.Endpoint,
-        {Oban, Pleroma.Config.get(Oban)}
+        {Oban, oban_config},
+        {Majic.Pool,
+         [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]}
       ] ++
         http_children(adapter)
 

lib/mix/tasks/pleroma/app.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.App do

lib/mix/tasks/pleroma/benchmark.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Benchmark do

lib/mix/tasks/pleroma/config.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Config do
@@ -27,7 +27,7 @@ defmodule Mix.Tasks.Pleroma.Config do
 
       {opts, _} =
         OptionParser.parse!(options,
-          strict: [env: :string, delete: :boolean],
+          strict: [env: :string, delete: :boolean, path: :string],
           aliases: [d: :delete]
         )
 
@@ -259,18 +259,43 @@ defmodule Mix.Tasks.Pleroma.Config do
   defp migrate_from_db(opts) do
     env = opts[:env] || Pleroma.Config.get(:env)
 
+    filename = "#{env}.exported_from_db.secret.exs"
+
     config_path =
-      if Pleroma.Config.get(:release) do
+      cond do
+        opts[:path] ->
+          opts[:path]
+
+        Pleroma.Config.get(:release) ->
           :config_path
           |> Pleroma.Config.get()
           |> Path.dirname()
-      else
+
+        true ->
           "config"
       end
-      |> Path.join("#{env}.exported_from_db.secret.exs")
+      |> Path.join(filename)
 
-    file = File.open!(config_path, [:write, :utf8])
+    with {:ok, file} <- File.open(config_path, [:write, :utf8]) do
+      write_config(file, config_path, opts)
+      shell_info("Database configuration settings have been exported to #{config_path}")
+    else
+      _ ->
+        shell_error("Impossible to save settings to this directory #{Path.dirname(config_path)}")
+        tmp_config_path = Path.join(System.tmp_dir!(), filename)
+        file = File.open!(tmp_config_path)
 
+        shell_info(
+          "Saving database configuration settings to #{tmp_config_path}. Copy it to the #{
+            Path.dirname(config_path)
+          } manually."
+        )
+
+        write_config(file, tmp_config_path, opts)
+    end
+  end
+
+  defp write_config(file, path, opts) do
     IO.write(file, config_header())
 
     ConfigDB
@@ -278,11 +303,7 @@ defmodule Mix.Tasks.Pleroma.Config do
     |> Enum.each(&write_and_delete(&1, file, opts[:delete]))
 
     :ok = File.close(file)
-    System.cmd("mix", ["format", config_path])
+    System.cmd("mix", ["format", path])
-
-    shell_info(
-      "Database configuration settings have been exported to config/#{env}.exported_from_db.secret.exs"
-    )
   end
 
   if Code.ensure_loaded?(Config.Reader) do

lib/mix/tasks/pleroma/count_statuses.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.CountStatuses do

lib/mix/tasks/pleroma/database.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Database do
@@ -8,10 +8,13 @@ defmodule Mix.Tasks.Pleroma.Database do
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
+
   require Logger
   require Pleroma.Constants
+
   import Ecto.Query
   import Mix.Pleroma
+
   use Mix.Task
 
   @shortdoc "A collection of database related tasks"
@@ -48,9 +51,15 @@ defmodule Mix.Tasks.Pleroma.Database do
   def run(["update_users_following_followers_counts"]) do
     start_pleroma()
 
-    User
+    Repo.transaction(
-    |> Repo.all()
+      fn ->
-    |> Enum.each(&User.update_follower_count/1)
+        from(u in User, select: u)
+        |> Repo.stream()
+        |> Stream.each(&User.update_follower_count/1)
+        |> Stream.run()
+      end,
+      timeout: :infinity
+    )
   end
 
   def run(["prune_objects" | args]) do
@@ -161,4 +170,79 @@ defmodule Mix.Tasks.Pleroma.Database do
     end)
     |> Stream.run()
   end
+
+  def run(["set_text_search_config", tsconfig]) do
+    start_pleroma()
+    %{rows: [[tsc]]} = Ecto.Adapters.SQL.query!(Pleroma.Repo, "SHOW default_text_search_config;")
+    shell_info("Current default_text_search_config: #{tsc}")
+
+    %{rows: [[db]]} = Ecto.Adapters.SQL.query!(Pleroma.Repo, "SELECT current_database();")
+    shell_info("Update default_text_search_config: #{tsconfig}")
+
+    %{messages: msg} =
+      Ecto.Adapters.SQL.query!(
+        Pleroma.Repo,
+        "ALTER DATABASE #{db} SET default_text_search_config = '#{tsconfig}';"
+      )
+
+    # non-exist config will not raise excpetion but only give >0 messages
+    if length(msg) > 0 do
+      shell_info("Error: #{inspect(msg, pretty: true)}")
+    else
+      rum_enabled = Pleroma.Config.get([:database, :rum_enabled])
+      shell_info("Recreate index, RUM: #{rum_enabled}")
+
+      # Note SQL below needs to be kept up-to-date with latest GIN or RUM index definition in future
+      if rum_enabled do
+        Ecto.Adapters.SQL.query!(
+          Pleroma.Repo,
+          "CREATE OR REPLACE FUNCTION objects_fts_update() RETURNS trigger AS $$ BEGIN
+          new.fts_content := to_tsvector(new.data->>'content');
+          RETURN new;
+          END
+          $$ LANGUAGE plpgsql"
+        )
+
+        shell_info("Refresh RUM index")
+        Ecto.Adapters.SQL.query!(Pleroma.Repo, "UPDATE objects SET updated_at = NOW();")
+      else
+        Ecto.Adapters.SQL.query!(Pleroma.Repo, "DROP INDEX IF EXISTS objects_fts;")
+
+        Ecto.Adapters.SQL.query!(
+          Pleroma.Repo,
+          "CREATE INDEX objects_fts ON objects USING gin(to_tsvector('#{tsconfig}', data->>'content')); "
+        )
+      end
+
+      shell_info('Done.')
+    end
+  end
+
+  # Rolls back a specific migration (leaving subsequent migrations applied).
+  # WARNING: imposes a risk of unrecoverable data loss — proceed at your own responsibility.
+  # Based on https://stackoverflow.com/a/53825840
+  def run(["rollback", version]) do
+    prompt = "SEVERE WARNING: this operation may result in unrecoverable data loss. Continue?"
+
+    if shell_prompt(prompt, "n") in ~w(Yn Y y) do
+      {_, result, _} =
+        Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
+          version = String.to_integer(version)
+          re = ~r/^#{version}_.*\.exs/
+          path = Ecto.Migrator.migrations_path(repo)
+
+          with {_, "" <> file} <- {:find, Enum.find(File.ls!(path), &String.match?(&1, re))},
+               {_, [{mod, _} | _]} <- {:compile, Code.compile_file(Path.join(path, file))},
+               {_, :ok} <- {:rollback, Ecto.Migrator.down(repo, version, mod)} do
+            {:ok, "Reversed migration: #{file}"}
+          else
+            {:find, _} -> {:error, "No migration found with version prefix: #{version}"}
+            {:compile, e} -> {:error, "Problem compiling migration module: #{inspect(e)}"}
+            {:rollback, e} -> {:error, "Problem reversing migration: #{inspect(e)}"}
+          end
+        end)
+
+      shell_info(inspect(result))
+    end
+  end
 end

lib/mix/tasks/pleroma/digest.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Digest do

lib/mix/tasks/pleroma/docs.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Docs do

lib/mix/tasks/pleroma/ecto.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-onl
 
 defmodule Mix.Tasks.Pleroma.Ecto do

lib/mix/tasks/pleroma/ecto/migrate.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-onl
 
 defmodule Mix.Tasks.Pleroma.Ecto.Migrate do

lib/mix/tasks/pleroma/ecto/rollback.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-onl
 
 defmodule Mix.Tasks.Pleroma.Ecto.Rollback do
@@ -20,7 +20,8 @@ defmodule Mix.Tasks.Pleroma.Ecto.Rollback do
     start: :boolean,
     quiet: :boolean,
     log_sql: :boolean,
-    migrations_path: :string
+    migrations_path: :string,
+    env: :string
   ]
 
   @moduledoc """
@@ -59,7 +60,7 @@ defmodule Mix.Tasks.Pleroma.Ecto.Rollback do
     level = Logger.level()
     Logger.configure(level: :info)
 
-    if Pleroma.Config.get(:env) == :test do
+    if opts[:env] == "test" do
       Logger.info("Rollback succesfully")
     else
       {:ok, _, _} =

lib/mix/tasks/pleroma/email.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Email do
@@ -33,12 +33,12 @@ defmodule Mix.Tasks.Pleroma.Email do
 
     Pleroma.User.Query.build(%{
       local: true,
-      deactivated: false,
+      is_active: true,
-      confirmation_pending: true,
+      is_confirmed: false,
       invisible: false
     })
     |> Pleroma.Repo.chunk_stream(500)
-    |> Stream.each(&Pleroma.User.try_send_confirmation_email(&1))
+    |> Stream.each(&Pleroma.User.maybe_send_confirmation_email(&1))
     |> Stream.run()
   end
 end

lib/mix/tasks/pleroma/emoji.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Emoji do

lib/mix/tasks/pleroma/frontend.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Frontend do

lib/mix/tasks/pleroma/instance.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Instance do
@@ -161,12 +161,21 @@ defmodule Mix.Tasks.Pleroma.Instance do
         )
         |> Path.expand()
 
+      {strip_uploads_message, strip_uploads_default} =
+        if Pleroma.Utils.command_available?("exiftool") do
+          {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as installed. (y/n)",
+           "y"}
+        else
+          {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
+           "n"}
+        end
+
       strip_uploads =
         get_option(
           options,
           :strip_uploads,
-          "Do you want to strip location (GPS) data from uploaded images? (y/n)",
+          strip_uploads_message,
-          "y"
+          strip_uploads_default
         ) === "y"
 
       anonymize_uploads =
@@ -233,6 +242,13 @@ defmodule Mix.Tasks.Pleroma.Instance do
           rum_enabled: rum_enabled
         )
 
+      config_dir = Path.dirname(config_path)
+      psql_dir = Path.dirname(psql_path)
+
+      [config_dir, psql_dir, static_dir, uploads_dir]
+      |> Enum.reject(&File.exists?/1)
+      |> Enum.map(&File.mkdir_p!/1)
+
       shell_info("Writing config to #{config_path}.")
 
       File.write(config_path, result_config)
@@ -253,7 +269,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
     else
       shell_error(
         "The task would have overwritten the following files:\n" <>
-          (Enum.map(paths, &"- #{&1}\n") |> Enum.join("")) <>
+          (Enum.map(will_overwrite, &"- #{&1}\n") |> Enum.join("")) <>
           "Rerun with `--force` to overwrite them."
       )
     end
@@ -266,10 +282,6 @@ defmodule Mix.Tasks.Pleroma.Instance do
         indexable: indexable
       )
 
-    unless File.exists?(static_dir) do
-      File.mkdir_p!(static_dir)
-    end
-
     robots_txt_path = Path.join(static_dir, "robots.txt")
 
     if File.exists?(robots_txt_path) do

lib/mix/tasks/pleroma/notification_settings.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.NotificationSettings do

lib/mix/tasks/pleroma/openapi_spec.ex

@@ -0,0 +1,8 @@
+defmodule Mix.Tasks.Pleroma.OpenapiSpec do
+  def run([path]) do
+    # Load Pleroma application to get version info
+    Application.load(:pleroma)
+    spec = Pleroma.Web.ApiSpec.spec(server_specific: false) |> Jason.encode!()
+    File.write(path, spec)
+  end
+end

lib/mix/tasks/pleroma/refresh_counter_cache.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.RefreshCounterCache do

lib/mix/tasks/pleroma/relay.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Relay do

lib/mix/tasks/pleroma/robots_txt.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.RobotsTxt do

lib/mix/tasks/pleroma/uploads.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Uploads do

lib/mix/tasks/pleroma/user.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.User do
@@ -74,7 +74,7 @@ defmodule Mix.Tasks.Pleroma.User do
         bio: bio
       }
 
-      changeset = User.register_changeset(%User{}, params, need_confirmation: false)
+      changeset = User.register_changeset(%User{}, params, is_confirmed: true)
       {:ok, _user} = User.register(changeset)
 
       shell_info("User #{nickname} created")
@@ -107,21 +107,6 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
-  def run(["toggle_activated", nickname]) do
-    start_pleroma()
-
-    with %User{} = user <- User.get_cached_by_nickname(nickname) do
-      {:ok, user} = User.deactivate(user, !user.deactivated)
-
-      shell_info(
-        "Activation status of #{nickname}: #{if(user.deactivated, do: "de", else: "")}activated"
-      )
-    else
-      _ ->
-        shell_error("No user #{nickname}")
-    end
-  end
-
   def run(["reset_password", nickname]) do
     start_pleroma()
 
@@ -156,20 +141,41 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
+  def run(["activate", nickname]) do
+    start_pleroma()
+
+    with %User{} = user <- User.get_cached_by_nickname(nickname),
+         false <- user.is_active do
+      User.set_activation(user, true)
+      :timer.sleep(500)
+
+      shell_info("Successfully activated #{nickname}")
+    else
+      true ->
+        shell_info("User #{nickname} already activated")
+
+      _ ->
+        shell_error("No user #{nickname}")
+    end
+  end
+
   def run(["deactivate", nickname]) do
     start_pleroma()
 
-    with %User{} = user <- User.get_cached_by_nickname(nickname) do
+    with %User{} = user <- User.get_cached_by_nickname(nickname),
-      shell_info("Deactivating #{user.nickname}")
+         true <- user.is_active do
-      User.deactivate(user)
+      User.set_activation(user, false)
       :timer.sleep(500)
 
       user = User.get_cached_by_id(user.id)
 
       if Enum.empty?(Enum.filter(User.get_friends(user), & &1.local)) do
-        shell_info("Successfully unsubscribed all local followers from #{user.nickname}")
+        shell_info("Successfully deactivated #{nickname} and unsubscribed all local followers")
       end
     else
+      false ->
+        shell_info("User #{nickname} already deactivated")
+
       _ ->
         shell_error("No user #{nickname}")
     end
@@ -213,7 +219,7 @@ defmodule Mix.Tasks.Pleroma.User do
       user =
         case Keyword.get(options, :confirmed) do
           nil -> user
-          value -> set_confirmed(user, value)
+          value -> set_confirmation(user, value)
         end
 
       user =
@@ -345,13 +351,13 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
-  def run(["toggle_confirmed", nickname]) do
+  def run(["confirm", nickname]) do
     start_pleroma()
 
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
-      {:ok, user} = User.toggle_confirmation(user)
+      {:ok, user} = User.confirm(user)
 
-      message = if user.confirmation_pending, do: "needs", else: "doesn't need"
+      message = if !user.is_confirmed, do: "needs", else: "doesn't need"
 
       shell_info("#{nickname} #{message} confirmation.")
     else
@@ -365,7 +371,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
     Pleroma.User.Query.build(%{
       local: true,
-      deactivated: false,
+      is_active: true,
       is_moderator: false,
       is_admin: false,
       invisible: false
@@ -373,7 +379,7 @@ defmodule Mix.Tasks.Pleroma.User do
     |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
-      |> Enum.each(fn user -> User.need_confirmation(user, false) end)
+      |> Enum.each(fn user -> User.set_confirmation(user, true) end)
     end)
     |> Stream.run()
   end
@@ -383,7 +389,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
     Pleroma.User.Query.build(%{
       local: true,
-      deactivated: false,
+      is_active: true,
       is_moderator: false,
       is_admin: false,
       invisible: false
@@ -391,7 +397,7 @@ defmodule Mix.Tasks.Pleroma.User do
     |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
-      |> Enum.each(fn user -> User.need_confirmation(user, true) end)
+      |> Enum.each(fn user -> User.set_confirmation(user, false) end)
     end)
     |> Stream.run()
   end
@@ -420,7 +426,7 @@ defmodule Mix.Tasks.Pleroma.User do
         shell_info(
           "#{user.nickname} moderator: #{user.is_moderator}, admin: #{user.is_admin}, locked: #{
             user.is_locked
-          }, deactivated: #{user.deactivated}"
+          }, is_active: #{user.is_active}"
         )
       end)
     end)
@@ -454,10 +460,10 @@ defmodule Mix.Tasks.Pleroma.User do
     user
   end
 
-  defp set_confirmed(user, value) do
+  defp set_confirmation(user, value) do
-    {:ok, user} = User.need_confirmation(user, !value)
+    {:ok, user} = User.set_confirmation(user, value)
 
-    shell_info("Confirmation pending status of #{user.nickname}: #{user.confirmation_pending}")
+    shell_info("Confirmation status of #{user.nickname}: #{user.is_confirmed}")
     user
   end
 end

lib/phoenix/transports/web_socket/raw.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Phoenix.Transports.WebSocket.Raw do

lib/pleroma/activity.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Activity do
@@ -24,6 +24,8 @@ defmodule Pleroma.Activity do
 
   @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   schema "activities" do
     field(:data, :map)
     field(:local, :boolean, default: true)
@@ -111,6 +113,7 @@ defmodule Pleroma.Activity do
     from([a] in query,
       left_join: b in Bookmark,
       on: b.user_id == ^user.id and b.activity_id == a.id,
+      as: :bookmark,
       preload: [bookmark: b]
     )
   end
@@ -121,6 +124,7 @@ defmodule Pleroma.Activity do
     from([a] in query,
       left_join: r in ReportNote,
       on: a.id == r.activity_id,
+      as: :report_note,
       preload: [report_notes: r]
     )
   end
@@ -180,38 +184,46 @@ defmodule Pleroma.Activity do
     |> Repo.one()
   end
 
-  @spec get_by_id(String.t()) :: Activity.t() | nil
+  @doc """
-  def get_by_id(id) do
+  Gets activity by ID, doesn't load activities from deactivated actors by default.
-    case FlakeId.flake_id?(id) do
+  """
-      true ->
+  @spec get_by_id(String.t(), keyword()) :: t() | nil
-        Activity
+  def get_by_id(id, opts \\ [filter: [:restrict_deactivated]]), do: get_by_id_with_opts(id, opts)
-        |> where([a], a.id == ^id)
-        |> restrict_deactivated_users()
-        |> Repo.one()
 
-      _ ->
+  @spec get_by_id_with_user_actor(String.t()) :: t() | nil
-        nil
+  def get_by_id_with_user_actor(id), do: get_by_id_with_opts(id, preload: [:user_actor])
-    end
+
+  @spec get_by_id_with_object(String.t()) :: t() | nil
+  def get_by_id_with_object(id), do: get_by_id_with_opts(id, preload: [:object])
+
+  defp get_by_id_with_opts(id, opts) do
+    if FlakeId.flake_id?(id) do
+      query = Queries.by_id(id)
+
+      with_filters_query =
+        if is_list(opts[:filter]) do
+          Enum.reduce(opts[:filter], query, fn
+            {:type, type}, acc -> Queries.by_type(acc, type)
+            :restrict_deactivated, acc -> restrict_deactivated_users(acc)
+            _, acc -> acc
+          end)
+        else
+          query
         end
 
-  def get_by_id_with_user_actor(id) do
+      with_preloads_query =
-    case FlakeId.flake_id?(id) do
+        if is_list(opts[:preload]) do
-      true ->
+          Enum.reduce(opts[:preload], with_filters_query, fn
-        Activity
+            :user_actor, acc -> with_preloaded_user_actor(acc)
-        |> where([a], a.id == ^id)
+            :object, acc -> with_preloaded_object(acc)
-        |> with_preloaded_user_actor()
+            _, acc -> acc
-        |> Repo.one()
+          end)
-
+        else
-      _ ->
+          with_filters_query
-        nil
-    end
         end
 
-  def get_by_id_with_object(id) do
+      Repo.one(with_preloads_query)
-    Activity
+    end
-    |> where(id: ^id)
-    |> with_preloaded_object()
-    |> Repo.one()
   end
 
   def all_by_ids_with_object(ids) do
@@ -265,6 +277,11 @@ defmodule Pleroma.Activity do
 
   def get_create_by_object_ap_id_with_object(_), do: nil
 
+  @spec create_by_id_with_object(String.t()) :: t() | nil
+  def create_by_id_with_object(id) do
+    get_by_id_with_opts(id, preload: [:object], filter: [type: "Create"])
+  end
+
   defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do
     get_create_by_object_ap_id_with_object(ap_id)
   end
@@ -272,7 +289,7 @@ defmodule Pleroma.Activity do
   defp get_in_reply_to_activity_from_object(_), do: nil
 
   def get_in_reply_to_activity(%Activity{} = activity) do
-    get_in_reply_to_activity_from_object(Object.normalize(activity))
+    get_in_reply_to_activity_from_object(Object.normalize(activity, fetch: false))
   end
 
   def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"])
@@ -298,7 +315,7 @@ defmodule Pleroma.Activity do
 
   defp purge_web_resp_cache(%Activity{} = activity) do
     %{path: path} = URI.parse(activity.data["id"])
-    Cachex.del(:web_resp_cache, path)
+    @cachex.del(:web_resp_cache, path)
     activity
   end
 
@@ -364,12 +381,6 @@ defmodule Pleroma.Activity do
     end
   end
 
-  @spec pinned_by_actor?(Activity.t()) :: boolean()
-  def pinned_by_actor?(%Activity{} = activity) do
-    actor = user_actor(activity)
-    activity.id in actor.pinned_activities
-  end
-
   @spec get_by_object_ap_id_with_object(String.t()) :: t() | nil
   def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
     ap_id
@@ -380,4 +391,13 @@ defmodule Pleroma.Activity do
   end
 
   def get_by_object_ap_id_with_object(_), do: nil
+
+  @spec add_by_params_query(String.t(), String.t(), String.t()) :: Ecto.Query.t()
+  def add_by_params_query(object_id, actor, target) do
+    object_id
+    |> Queries.by_object_id()
+    |> Queries.by_type("Add")
+    |> Queries.by_actor(actor)
+    |> where([a], fragment("?->>'target' = ?", a.data, ^target))
+  end
 end

lib/pleroma/activity/ir/topics.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Activity.Ir.Topics do
@@ -8,7 +8,7 @@ defmodule Pleroma.Activity.Ir.Topics do
 
   def get_activity_topics(activity) do
     activity
-    |> Object.normalize()
+    |> Object.normalize(fetch: false)
     |> generate_topics(activity)
     |> List.flatten()
   end
@@ -48,14 +48,12 @@ defmodule Pleroma.Activity.Ir.Topics do
     tags
   end
 
-  defp hashtags_to_topics(%{data: %{"tag" => tags}}) do
+  defp hashtags_to_topics(object) do
-    tags
+    object
-    |> Enum.filter(&is_bitstring(&1))
+    |> Object.hashtags()
-    |> Enum.map(fn tag -> "hashtag:" <> tag end)
+    |> Enum.map(fn hashtag -> "hashtag:" <> hashtag end)
   end
 
-  defp hashtags_to_topics(_), do: []
-
   defp remote_topics(%{local: true}), do: []
 
   defp remote_topics(%{actor: actor}) when is_binary(actor),

lib/pleroma/activity/queries.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Activity.Queries do
@@ -14,6 +14,11 @@ defmodule Pleroma.Activity.Queries do
   alias Pleroma.Activity
   alias Pleroma.User
 
+  @spec by_id(query(), String.t()) :: query()
+  def by_id(query \\ Activity, id) do
+    from(a in query, where: a.id == ^id)
+  end
+
   @spec by_ap_id(query, String.t()) :: query
   def by_ap_id(query \\ Activity, ap_id) do
     from(

lib/pleroma/activity/search.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Activity.Search do
@@ -64,7 +64,7 @@ defmodule Pleroma.Activity.Search do
     from([a, o] in q,
       where:
         fragment(
-          "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
+          "to_tsvector(?->>'content') @@ plainto_tsquery(?)",
           o.data,
           ^search_query
         )
@@ -75,7 +75,7 @@ defmodule Pleroma.Activity.Search do
     from([a, o] in q,
       where:
         fragment(
-          "to_tsvector('english', ?->>'content') @@ websearch_to_tsquery('english', ?)",
+          "to_tsvector(?->>'content') @@ websearch_to_tsquery(?)",
           o.data,
           ^search_query
         )
@@ -86,7 +86,7 @@ defmodule Pleroma.Activity.Search do
     from([a, o] in q,
       where:
         fragment(
-          "? @@ plainto_tsquery('english', ?)",
+          "? @@ plainto_tsquery(?)",
           o.fts_content,
           ^search_query
         ),
@@ -98,7 +98,7 @@ defmodule Pleroma.Activity.Search do
     from([a, o] in q,
       where:
         fragment(
-          "? @@ websearch_to_tsquery('english', ?)",
+          "? @@ websearch_to_tsquery(?)",
           o.fts_content,
           ^search_query
         ),

lib/pleroma/application.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Application do
@@ -14,7 +14,7 @@ defmodule Pleroma.Application do
   @name Mix.Project.config()[:name]
   @version Mix.Project.config()[:version]
   @repository Mix.Project.config()[:source_url]
-  @env Mix.env()
+  @mix_env Mix.env()
 
   def name, do: @name
   def version, do: @version
@@ -92,20 +92,18 @@ defmodule Pleroma.Application do
         Pleroma.Web.Plugs.RateLimiter.Supervisor
       ] ++
         cachex_children() ++
-        http_children(adapter, @env) ++
+        http_children(adapter, @mix_env) ++
         [
           Pleroma.Stats,
           Pleroma.JobQueueMonitor,
           {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]},
-          {Oban, Config.get(Oban)}
+          {Oban, Config.get(Oban)},
+          Pleroma.Web.Endpoint
         ] ++
-        task_children(@env) ++
+        task_children(@mix_env) ++
-        dont_run_in_test(@env) ++
+        dont_run_in_test(@mix_env) ++
         chat_child(chat_enabled?()) ++
-        [
+        [Pleroma.Gopher.Server]
-          Pleroma.Web.Endpoint,
-          Pleroma.Gopher.Server
-        ]
 
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
     # for other strategies and supported options
@@ -145,7 +143,7 @@ defmodule Pleroma.Application do
           raise "Invalid custom modules"
 
         {:ok, modules, _warnings} ->
-          if @env != :test do
+          if @mix_env != :test do
             Enum.each(modules, fn mod ->
               Logger.info("Custom module loaded: #{inspect(mod)}")
             end)
@@ -230,6 +228,12 @@ defmodule Pleroma.Application do
          keys: :duplicate,
          partitions: System.schedulers_online()
        ]}
+    ] ++ background_migrators()
+  end
+
+  defp background_migrators do
+    [
+      Pleroma.Migrators.HashtagsTableMigrator
     ]
   end
 
@@ -297,7 +301,16 @@ defmodule Pleroma.Application do
 
   @spec limiters_setup() :: :ok
   def limiters_setup do
-    [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy]
+    config = Config.get(ConcurrentLimiter, [])
-    |> Enum.each(&ConcurrentLimiter.new(&1, 1, 0))
+
+    [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy]
+    |> Enum.each(fn module ->
+      mod_config = Keyword.get(config, module, [])
+
+      max_running = Keyword.get(mod_config, :max_running, 5)
+      max_waiting = Keyword.get(mod_config, :max_waiting, 5)
+
+      ConcurrentLimiter.new(module, max_running, max_waiting)
+    end)
   end
 end

lib/pleroma/application_requirements.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ApplicationRequirements do

lib/pleroma/bbs/authenticator.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.BBS.Authenticator do

lib/pleroma/bbs/handler.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.BBS.Handler do

lib/pleroma/bookmark.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Bookmark do

lib/pleroma/caching.ex

@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Caching do
+  @callback get!(Cachex.cache(), any()) :: any()
+  @callback get(Cachex.cache(), any()) :: {atom(), any()}
+  @callback put(Cachex.cache(), any(), any(), Keyword.t()) :: {Cachex.status(), boolean()}
+  @callback put(Cachex.cache(), any(), any()) :: {Cachex.status(), boolean()}
+  @callback fetch!(Cachex.cache(), any(), function() | nil) :: any()
+  # @callback del(Cachex.cache(), any(), Keyword.t()) :: {Cachex.status(), boolean()}
+  @callback del(Cachex.cache(), any()) :: {Cachex.status(), boolean()}
+  @callback stream!(Cachex.cache(), any()) :: Enumerable.t()
+  @callback expire_at(Cachex.cache(), binary(), number()) :: {Cachex.status(), boolean()}
+  @callback exists?(Cachex.cache(), any()) :: {Cachex.status(), boolean()}
+  @callback execute!(Cachex.cache(), function()) :: any()
+  @callback get_and_update(Cachex.cache(), any(), function()) ::
+              {:commit | :ignore, any()}
+end

lib/pleroma/captcha.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha do
@@ -7,6 +7,8 @@ defmodule Pleroma.Captcha do
   alias Plug.Crypto.KeyGenerator
   alias Plug.Crypto.MessageEncryptor
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   @doc """
   Ask the configured captcha service for a new captcha
   """
@@ -86,7 +88,7 @@ defmodule Pleroma.Captcha do
   end
 
   defp validate_usage(token) do
-    if is_nil(Cachex.get!(:used_captcha_cache, token)) do
+    if is_nil(@cachex.get!(:used_captcha_cache, token)) do
       :ok
     else
       {:error, :already_used}
@@ -95,7 +97,7 @@ defmodule Pleroma.Captcha do
 
   defp mark_captcha_as_used(token) do
     ttl = seconds_valid() |> :timer.seconds()
-    Cachex.put(:used_captcha_cache, token, true, ttl: ttl)
+    @cachex.put(:used_captcha_cache, token, true, ttl: ttl)
   end
 
   defp method, do: Pleroma.Config.get!([__MODULE__, :method])

lib/pleroma/captcha/kocaptcha.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha.Kocaptcha do

lib/pleroma/captcha/native.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha.Native do

lib/pleroma/captcha/service.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha.Service do

lib/pleroma/chat.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Chat do

lib/pleroma/chat/message_reference.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Chat.MessageReference do

lib/pleroma/clippy.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Clippy do

lib/pleroma/config.ex

@@ -1,16 +1,20 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Config do
+  @behaviour Pleroma.Config.Getting
   defmodule Error do
     defexception [:message]
   end
 
+  @impl true
   def get(key), do: get(key, nil)
 
+  @impl true
   def get([key], default), do: get(key, default)
 
+  @impl true
   def get([_ | _] = path, default) do
     case fetch(path) do
       {:ok, value} -> value
@@ -18,6 +22,7 @@ defmodule Pleroma.Config do
     end
   end
 
+  @impl true
   def get(key, default) do
     Application.get_env(:pleroma, key, default)
   end
@@ -95,15 +100,7 @@ defmodule Pleroma.Config do
 
   def oauth_consumer_enabled?, do: oauth_consumer_strategies() != []
 
-  def enforce_oauth_admin_scope_usage?, do: !!get([:auth, :enforce_oauth_admin_scope_usage])
+  def feature_enabled?(feature_name) do
-
+    get([:features, feature_name]) not in [nil, false, :disabled, :auto]
-  def oauth_admin_scopes(scopes) when is_list(scopes) do
-    Enum.flat_map(
-      scopes,
-      fn scope ->
-        ["admin:#{scope}"] ++
-          if enforce_oauth_admin_scope_usage?(), do: [], else: [scope]
-      end
-    )
   end
 end

lib/pleroma/config/deprecation_warnings.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Config.DeprecationWarnings do
@@ -40,7 +40,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
          :ok <- check_welcome_message_config(),
          :ok <- check_gun_pool_options(),
          :ok <- check_activity_expiration_config(),
-         :ok <- check_remote_ip_plug_name() do
+         :ok <- check_remote_ip_plug_name(),
+         :ok <- check_uploders_s3_public_endpoint() do
       :ok
     else
       _ ->
@@ -193,4 +194,25 @@ defmodule Pleroma.Config.DeprecationWarnings do
       warning_preface
     )
   end
+
+  @spec check_uploders_s3_public_endpoint() :: :ok | nil
+  def check_uploders_s3_public_endpoint do
+    s3_config = Pleroma.Config.get([Pleroma.Uploaders.S3])
+
+    use_old_config = Keyword.has_key?(s3_config, :public_endpoint)
+
+    if use_old_config do
+      Logger.error("""
+      !!!DEPRECATION WARNING!!!
+      Your config is using the old setting for controlling the URL of media uploaded to your S3 bucket.\n
+      Please make the following change at your earliest convenience.\n
+      \n* `config :pleroma, Pleroma.Uploaders.S3, public_endpoint` is now equal to:
+      \n* `config :pleroma, Pleroma.Upload, base_url`
+      """)
+
+      :error
+    else
+      :ok
+    end
+  end
 end

lib/pleroma/config/getting.ex

@@ -0,0 +1,8 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Config.Getting do
+  @callback get(any()) :: any()
+  @callback get(any(), any()) :: any()
+end

lib/pleroma/config/helpers.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Config.Helpers do

lib/pleroma/config/holder.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Config.Holder do

lib/pleroma/config/loader.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Config.Loader do

lib/pleroma/config/oban.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Config.Oban do

lib/pleroma/config/release_runtime_provider.ex

@@ -1,6 +1,6 @@
 defmodule Pleroma.Config.ReleaseRuntimeProvider do
   @moduledoc """
-  Imports `runtime.exs` and `{env}.exported_from_db.secret.exs` for elixir releases.
+  Imports runtime config and `{env}.exported_from_db.secret.exs` for releases.
   """
   @behaviour Config.Provider
 
@@ -8,10 +8,11 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do
   def init(opts), do: opts
 
   @impl true
-  def load(config, _opts) do
+  def load(config, opts) do
     with_defaults = Config.Reader.merge(config, Pleroma.Config.Holder.release_defaults())
 
-    config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
+    config_path =
+      opts[:config_path] || System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
 
     with_runtime_config =
       if File.exists?(config_path) do
@@ -24,7 +25,7 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do
         warning = [
           IO.ANSI.red(),
           IO.ANSI.bright(),
-          "!!! #{config_path} not found! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file",
+          "!!! Config path is not declared! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file",
           IO.ANSI.reset()
         ]
 
@@ -33,13 +34,14 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do
       end
 
     exported_config_path =
+      opts[:exported_config_path] ||
         config_path
         |> Path.dirname()
-      |> Path.join("prod.exported_from_db.secret.exs")
+        |> Path.join("#{Pleroma.Config.get(:env)}.exported_from_db.secret.exs")
 
     with_exported =
       if File.exists?(exported_config_path) do
-        exported_config = Config.Reader.read!(with_runtime_config)
+        exported_config = Config.Reader.read!(exported_config_path)
         Config.Reader.merge(with_runtime_config, exported_config)
       else
         with_runtime_config

lib/pleroma/config/transfer_task.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Config.TransferTask do

lib/pleroma/config_db.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ConfigDB do
@@ -387,6 +387,6 @@ defmodule Pleroma.ConfigDB do
   @spec module_name?(String.t()) :: boolean()
   def module_name?(string) do
     Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or
-      string in ["Oban", "Ueberauth", "ExSyslogger"]
+      string in ["Oban", "Ueberauth", "ExSyslogger", "ConcurrentLimiter"]
   end
 end

lib/pleroma/constants.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Constants do
@@ -18,7 +18,8 @@ defmodule Pleroma.Constants do
       "emoji",
       "context_id",
       "deleted_activity_id",
-      "pleroma_internal"
+      "pleroma_internal",
+      "generator"
     ]
   )
 

lib/pleroma/conversation.ex

@@ -1,10 +1,11 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Conversation do
   alias Pleroma.Conversation.Participation
   alias Pleroma.Conversation.Participation.RecipientShip
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
   use Ecto.Schema
@@ -58,11 +59,10 @@ defmodule Pleroma.Conversation do
   def create_or_bump_for(activity, opts \\ []) do
     with true <- Pleroma.Web.ActivityPub.Visibility.is_direct?(activity),
          "Create" <- activity.data["type"],
-         object <- Pleroma.Object.normalize(activity),
+         %Object{} = object <- Object.normalize(activity, fetch: false),
          true <- object.data["type"] in ["Note", "Question"],
-         ap_id when is_binary(ap_id) and byte_size(ap_id) > 0 <- object.data["context"] do
+         ap_id when is_binary(ap_id) and byte_size(ap_id) > 0 <- object.data["context"],
-      {:ok, conversation} = create_for_ap_id(ap_id)
+         {:ok, conversation} <- create_for_ap_id(ap_id) do
-
       users = User.get_users_from_set(activity.recipients, local_only: false)
 
       participations =

lib/pleroma/conversation/participation.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Conversation.Participation do
@@ -220,4 +220,8 @@ defmodule Pleroma.Conversation.Participation do
       select: %{count: count(p.id)}
     )
   end
+
+  def delete(%__MODULE__{} = participation) do
+    Repo.delete(participation)
+  end
 end

lib/pleroma/conversation/participation/recipient_ship.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Conversation.Participation.RecipientShip do