diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dab52e4c6..21d7b2242 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,8 +1,8 @@
-image: git.pleroma.social:5050/pleroma/pleroma/ci-base
+image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24
variables: &global_variables
# Only used for the release
- ELIXIR_VER: 1.12.3
+ ELIXIR_VER: 1.13.4
POSTGRES_DB: pleroma_test
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
@@ -72,7 +72,7 @@ check-changelog:
tags:
- amd64
-build-1.12.3:
+build-1.13.4:
extends:
- .build_changes_policy
- .using-ci-base
@@ -85,7 +85,7 @@ build-1.15.7-otp-25:
- .build_changes_policy
- .using-ci-base
stage: build
- image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15
+ image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25
allow_failure: true
script:
- mix compile --force
diff --git a/Dockerfile b/Dockerfile
index 69c3509de..72461305c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,7 @@
ARG ELIXIR_IMG=hexpm/elixir
-ARG ELIXIR_VER=1.12.3
-ARG ERLANG_VER=24.2.1
-ARG ALPINE_VER=3.17.0
+ARG ELIXIR_VER=1.13.4
+ARG ERLANG_VER=24.3.4.15
+ARG ALPINE_VER=3.17.5
FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} as build
diff --git a/changelog.d/api-docs-2.skip b/changelog.d/api-docs-2.skip
new file mode 100644
index 000000000..e69de29bb
diff --git a/changelog.d/bump-elixir.change b/changelog.d/bump-elixir.change
new file mode 100644
index 000000000..afb25d4e7
--- /dev/null
+++ b/changelog.d/bump-elixir.change
@@ -0,0 +1 @@
+Elixir 1.13 is the minimum required version.
diff --git a/changelog.d/realpath-over-readlink.fix b/changelog.d/realpath-over-readlink.fix
new file mode 100644
index 000000000..479561b95
--- /dev/null
+++ b/changelog.d/realpath-over-readlink.fix
@@ -0,0 +1 @@
+pleroma_ctl: Use realpath(1) instead of readlink(1)
diff --git a/changelog.d/richmediattl.fix b/changelog.d/richmediattl.fix
new file mode 100644
index 000000000..98de63015
--- /dev/null
+++ b/changelog.d/richmediattl.fix
@@ -0,0 +1 @@
+Parsing of RichMedia TTLs for Amazon URLs when query parameters are nil
diff --git a/changelog.d/search-healthcheck.add b/changelog.d/search-healthcheck.add
new file mode 100644
index 000000000..4974925e7
--- /dev/null
+++ b/changelog.d/search-healthcheck.add
@@ -0,0 +1 @@
+Monitoring of search backend health to control the processing of jobs in the search indexing Oban queue
diff --git a/ci/elixir-1.13/Dockerfile b/ci/elixir-1.13/Dockerfile
new file mode 100644
index 000000000..b8bceb3d9
--- /dev/null
+++ b/ci/elixir-1.13/Dockerfile
@@ -0,0 +1,8 @@
+FROM elixir:1.13.4-otp-24
+
+# Single RUN statement, otherwise intermediate images are created
+# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run
+RUN apt-get update &&\
+ apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\
+ mix local.hex --force &&\
+ mix local.rebar --force
diff --git a/ci/elixir-1.13/build_and_push.sh b/ci/elixir-1.13/build_and_push.sh
new file mode 100755
index 000000000..64e1856db
--- /dev/null
+++ b/ci/elixir-1.13/build_and_push.sh
@@ -0,0 +1 @@
+docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24 --push .
diff --git a/ci/elixir-1.15-otp25/build_and_push.sh b/ci/elixir-1.15-otp25/build_and_push.sh
index 06fe74f34..a28e0d33c 100755
--- a/ci/elixir-1.15-otp25/build_and_push.sh
+++ b/ci/elixir-1.15-otp25/build_and_push.sh
@@ -1 +1 @@
-docker buildx build --platform linux/amd64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push .
+docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push .
diff --git a/config/config.exs b/config/config.exs
index c3b20947d..d891a5218 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -579,7 +579,7 @@ config :pleroma, Oban,
attachments_cleanup: 1,
new_users_digest: 1,
mute_expire: 5,
- search_indexing: 10,
+ search_indexing: [limit: 10, paused: true],
rich_media_expiration: 2
],
plugins: [Oban.Plugins.Pruner],
diff --git a/docs/development/API/pleroma_api.md b/docs/development/API/pleroma_api.md
index 267dfc1ec..57d333ffe 100644
--- a/docs/development/API/pleroma_api.md
+++ b/docs/development/API/pleroma_api.md
@@ -295,9 +295,7 @@ See [Admin-API](admin_api.md)
"id": "9umDrYheeY451cQnEe",
"name": "Read later",
"emoji": "🕓",
- "source": {
- "emoji": "🕓"
- }
+ "emoji_url": null
}
]
```
diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md
index b6b5c9c07..5a0823a63 100644
--- a/docs/installation/debian_based_jp.md
+++ b/docs/installation/debian_based_jp.md
@@ -14,7 +14,7 @@ Note: This article is potentially outdated because at this time we may not have
- PostgreSQL 11.0以上 (Ubuntu16.04では9.5しか提供されていないので,[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください)
- `postgresql-contrib` 11.0以上 (同上)
-- Elixir 1.8 以上 ([Debianのリポジトリからインストールしないこと!!! ここからインストールすること!](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
+- Elixir 1.13 以上 ([Debianのリポジトリからインストールしないこと!!! ここからインストールすること!](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
- `erlang-dev`
- `erlang-nox`
- `git`
diff --git a/docs/installation/generic_dependencies.include b/docs/installation/generic_dependencies.include
index 6572716ed..666f49fbb 100644
--- a/docs/installation/generic_dependencies.include
+++ b/docs/installation/generic_dependencies.include
@@ -1,7 +1,7 @@
## Required dependencies
* PostgreSQL >=11.0
-* Elixir >=1.11.0 <1.15
+* Elixir >=1.13.0 <1.15
* Erlang OTP >=22.2.0 (supported: <27)
* git
* file / libmagic
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 649bb11c8..d266d1836 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -109,7 +109,8 @@ defmodule Pleroma.Application do
streamer_registry() ++
background_migrators() ++
shout_child(shout_enabled?()) ++
- [Pleroma.Gopher.Server]
+ [Pleroma.Gopher.Server] ++
+ [Pleroma.Search.Healthcheck]
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
# for other strategies and supported options
diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex
index 3b266e59b..fd0218cb8 100644
--- a/lib/pleroma/search.ex
+++ b/lib/pleroma/search.ex
@@ -10,8 +10,12 @@ defmodule Pleroma.Search do
end
def search(query, options) do
- search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
-
+ search_module = Pleroma.Config.get([Pleroma.Search, :module])
search_module.search(options[:for_user], query, options)
end
+
+ def healthcheck_endpoints do
+ search_module = Pleroma.Config.get([Pleroma.Search, :module])
+ search_module.healthcheck_endpoints
+ end
end
diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex
index 24a1ff431..c6fe8a9bd 100644
--- a/lib/pleroma/search/database_search.ex
+++ b/lib/pleroma/search/database_search.ex
@@ -54,6 +54,9 @@ defmodule Pleroma.Search.DatabaseSearch do
@impl true
def drop_index, do: :ok
+ @impl true
+ def healthcheck_endpoints, do: nil
+
def maybe_restrict_author(query, %User{} = author) do
Activity.Queries.by_author(query, author)
end
diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex
new file mode 100644
index 000000000..e562c8478
--- /dev/null
+++ b/lib/pleroma/search/healthcheck.ex
@@ -0,0 +1,86 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+defmodule Pleroma.Search.Healthcheck do
+ @doc """
+ Monitors health of search backend to control processing of events based on health and availability.
+ """
+ use GenServer
+ require Logger
+
+ @queue :search_indexing
+ @tick :timer.seconds(5)
+ @timeout :timer.seconds(2)
+
+ def start_link(_) do
+ GenServer.start_link(__MODULE__, [], name: __MODULE__)
+ end
+
+ @impl true
+ def init(_) do
+ state = %{healthy: false}
+ {:ok, state, {:continue, :start}}
+ end
+
+ @impl true
+ def handle_continue(:start, state) do
+ tick()
+ {:noreply, state}
+ end
+
+ @impl true
+ def handle_info(:check, state) do
+ urls = Pleroma.Search.healthcheck_endpoints()
+
+ new_state =
+ if check(urls) do
+ Oban.resume_queue(queue: @queue)
+ Map.put(state, :healthy, true)
+ else
+ Oban.pause_queue(queue: @queue)
+ Map.put(state, :healthy, false)
+ end
+
+ maybe_log_state_change(state, new_state)
+
+ tick()
+ {:noreply, new_state}
+ end
+
+ @impl true
+ def handle_call(:state, _from, state) do
+ {:reply, state, state, :hibernate}
+ end
+
+ def state, do: GenServer.call(__MODULE__, :state)
+
+ def check([]), do: true
+
+ def check(urls) when is_list(urls) do
+ Enum.all?(
+ urls,
+ fn url ->
+ case Pleroma.HTTP.get(url, [], recv_timeout: @timeout) do
+ {:ok, %{status: 200}} -> true
+ _ -> false
+ end
+ end
+ )
+ end
+
+ def check(_), do: true
+
+ defp tick do
+ Process.send_after(self(), :check, @tick)
+ end
+
+ defp maybe_log_state_change(%{healthy: true}, %{healthy: false}) do
+ Logger.error("Pausing Oban queue #{@queue} due to search backend healthcheck failure")
+ end
+
+ defp maybe_log_state_change(%{healthy: false}, %{healthy: true}) do
+ Logger.info("Resuming Oban queue #{@queue} due to search backend healthcheck pass")
+ end
+
+ defp maybe_log_state_change(_, _), do: :ok
+end
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 50f5984d6..9bba5b30f 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -184,4 +184,15 @@ defmodule Pleroma.Search.Meilisearch do
def remove_from_index(object) do
meili_delete("/indexes/objects/documents/#{object.id}")
end
+
+ @impl true
+ def healthcheck_endpoints do
+ endpoint =
+ Config.get([Pleroma.Search.Meilisearch, :url])
+ |> URI.parse()
+ |> Map.put(:path, "/health")
+ |> URI.to_string()
+
+ [endpoint]
+ end
end
diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex
index 9735ab3f4..f4ed13c36 100644
--- a/lib/pleroma/search/search_backend.ex
+++ b/lib/pleroma/search/search_backend.ex
@@ -31,4 +31,12 @@ defmodule Pleroma.Search.SearchBackend do
Drop the index
"""
@callback drop_index() :: :ok | {:error, any()}
+
+ @doc """
+ Healthcheck endpoints of search backend infrastructure to monitor for controlling
+ processing of jobs in the Oban queue.
+
+ It is expected a 200 response is healthy and other responses are unhealthy.
+ """
+ @callback healthcheck_endpoints :: list() | nil
end
diff --git a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex
index 948c727e1..1172a120a 100644
--- a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex
+++ b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex
@@ -23,7 +23,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
%URI{host: host, query: query} = URI.parse(image)
is_binary(host) and String.contains?(host, "amazonaws.com") and
- String.contains?(query, "X-Amz-Expires")
+ is_binary(query) and String.contains?(query, "X-Amz-Expires")
end
defp aws_signed_url?(_), do: nil
diff --git a/mix.exs b/mix.exs
index 3edae5046..fe50139ef 100644
--- a/mix.exs
+++ b/mix.exs
@@ -5,7 +5,7 @@ defmodule Pleroma.Mixfile do
[
app: :pleroma,
version: version("2.6.52"),
- elixir: "~> 1.11",
+ elixir: "~> 1.13",
elixirc_paths: elixirc_paths(Mix.env()),
compilers: Mix.compilers(),
elixirc_options: [warnings_as_errors: warnings_as_errors()],
diff --git a/rel/files/bin/pleroma_ctl b/rel/files/bin/pleroma_ctl
index 87c486514..6f0dba3a8 100755
--- a/rel/files/bin/pleroma_ctl
+++ b/rel/files/bin/pleroma_ctl
@@ -134,7 +134,7 @@ if [ -z "$1" ] || [ "$1" = "help" ]; then
"
else
- SCRIPT=$(readlink -f "$0")
+ SCRIPT=$(realpath "$0")
SCRIPTPATH=$(dirname "$SCRIPT")
FULL_ARGS="$*"
diff --git a/test/pleroma/search/healthcheck_test.exs b/test/pleroma/search/healthcheck_test.exs
new file mode 100644
index 000000000..e7649d949
--- /dev/null
+++ b/test/pleroma/search/healthcheck_test.exs
@@ -0,0 +1,49 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Search.HealthcheckTest do
+ use Pleroma.DataCase
+
+ import Tesla.Mock
+
+ alias Pleroma.Search.Healthcheck
+
+ @good1 "http://good1.example.com/healthz"
+ @good2 "http://good2.example.com/health"
+ @bad "http://bad.example.com/healthy"
+
+ setup do
+ mock(fn
+ %{method: :get, url: @good1} ->
+ %Tesla.Env{
+ status: 200,
+ body: ""
+ }
+
+ %{method: :get, url: @good2} ->
+ %Tesla.Env{
+ status: 200,
+ body: ""
+ }
+
+ %{method: :get, url: @bad} ->
+ %Tesla.Env{
+ status: 503,
+ body: ""
+ }
+ end)
+
+ :ok
+ end
+
+ test "true for 200 responses" do
+ assert Healthcheck.check([@good1])
+ assert Healthcheck.check([@good1, @good2])
+ end
+
+ test "false if any response is not a 200" do
+ refute Healthcheck.check([@bad])
+ refute Healthcheck.check([@good1, @bad])
+ end
+end
diff --git a/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs b/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs
index cd8be8675..cc28aa7f3 100644
--- a/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs
+++ b/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do
alias Pleroma.UnstubbedConfigMock, as: ConfigMock
alias Pleroma.Web.RichMedia.Card
+ alias Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl
setup do
ConfigMock
@@ -82,6 +83,12 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do
assert DateTime.diff(scheduled_at, timestamp_dt) == valid_till
end
+ test "AWS URL for an image without expiration works" do
+ og_data = %{"image" => "https://amazonaws.com/image.png"}
+
+ assert is_nil(AwsSignedUrl.ttl(og_data, ""))
+ end
+
defp construct_s3_url(timestamp, valid_till) do
"https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{timestamp}&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host"
end