From 7e3bbdded5dea73a0bad3a8905839e42d476e506 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 20 Dec 2023 23:39:12 +0000 Subject: [PATCH 01/15] Elixir 1.13 is the minimum required version --- .gitlab-ci.yml | 2 +- Dockerfile | 6 +++--- changelog.d/bump-elixir.change | 1 + mix.exs | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 changelog.d/bump-elixir.change diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eb31a8086..8f10790da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ image: git.pleroma.social:5050/pleroma/pleroma/ci-base variables: &global_variables # Only used for the release - ELIXIR_VER: 1.12.3 + ELIXIR_VER: 1.13.4 POSTGRES_DB: pleroma_test POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres diff --git a/Dockerfile b/Dockerfile index 69c3509de..72461305c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ARG ELIXIR_IMG=hexpm/elixir -ARG ELIXIR_VER=1.12.3 -ARG ERLANG_VER=24.2.1 -ARG ALPINE_VER=3.17.0 +ARG ELIXIR_VER=1.13.4 +ARG ERLANG_VER=24.3.4.15 +ARG ALPINE_VER=3.17.5 FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} as build diff --git a/changelog.d/bump-elixir.change b/changelog.d/bump-elixir.change new file mode 100644 index 000000000..afb25d4e7 --- /dev/null +++ b/changelog.d/bump-elixir.change @@ -0,0 +1 @@ +Elixir 1.13 is the minimum required version. diff --git a/mix.exs b/mix.exs index b4b77b161..2056e591d 100644 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ defmodule Pleroma.Mixfile do [ app: :pleroma, version: version("2.6.51"), - elixir: "~> 1.11", + elixir: "~> 1.13", elixirc_paths: elixirc_paths(Mix.env()), compilers: Mix.compilers(), elixirc_options: [warnings_as_errors: warnings_as_errors()], From ddb9e90c405369496fdf9e6dfed593eff8d5dc5c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Dec 2023 15:59:25 -0500 Subject: [PATCH 02/15] Update minimum elixir version found in various docs --- docs/installation/debian_based_jp.md | 2 +- docs/installation/generic_dependencies.include | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index 1424ad7f4..502eefaf8 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -14,7 +14,7 @@ Note: This article is potentially outdated because at this time we may not have - PostgreSQL 9.6以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください) - `postgresql-contrib` 9.6以上 (同上) -- Elixir 1.8 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) +- Elixir 1.13 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) - `erlang-dev` - `erlang-nox` - `git` diff --git a/docs/installation/generic_dependencies.include b/docs/installation/generic_dependencies.include index 3365a36a8..e0cfd3264 100644 --- a/docs/installation/generic_dependencies.include +++ b/docs/installation/generic_dependencies.include @@ -1,7 +1,7 @@ ## Required dependencies * PostgreSQL >=9.6 -* Elixir >=1.11.0 <1.15 +* Elixir >=1.13.0 <1.15 * Erlang OTP >=22.2.0 <26 * git * file / libmagic From 226874c9d603be72699d5aa5434616efffe3f239 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 20 May 2024 13:12:12 +0400 Subject: [PATCH 03/15] CI: Add new builders for base images --- .gitlab-ci.yml | 6 +++--- ci/elixir-1.13/Dockerfile | 8 ++++++++ ci/elixir-1.13/build_and_push.sh | 1 + ci/elixir-1.15-otp25/build_and_push.sh | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 ci/elixir-1.13/Dockerfile create mode 100755 ci/elixir-1.13/build_and_push.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2e321c978..eba769af8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: git.pleroma.social:5050/pleroma/pleroma/ci-base +image: git.pleroma.social:5050/pleroma/pleroma/ci-base:1.13.4-otp24 variables: &global_variables # Only used for the release @@ -72,7 +72,7 @@ check-changelog: tags: - amd64 -build-1.12.3: +build-1.13.4: extends: - .build_changes_policy - .using-ci-base @@ -85,7 +85,7 @@ build-1.15.7-otp-25: - .build_changes_policy - .using-ci-base stage: build - image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15 + image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 allow_failure: true script: - mix compile --force diff --git a/ci/elixir-1.13/Dockerfile b/ci/elixir-1.13/Dockerfile new file mode 100644 index 000000000..b8bceb3d9 --- /dev/null +++ b/ci/elixir-1.13/Dockerfile @@ -0,0 +1,8 @@ +FROM elixir:1.13.4-otp-24 + +# Single RUN statement, otherwise intermediate images are created +# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run +RUN apt-get update &&\ + apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\ + mix local.hex --force &&\ + mix local.rebar --force diff --git a/ci/elixir-1.13/build_and_push.sh b/ci/elixir-1.13/build_and_push.sh new file mode 100755 index 000000000..53af4245f --- /dev/null +++ b/ci/elixir-1.13/build_and_push.sh @@ -0,0 +1 @@ +docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13-otp24 --push . diff --git a/ci/elixir-1.15-otp25/build_and_push.sh b/ci/elixir-1.15-otp25/build_and_push.sh index 06fe74f34..a28e0d33c 100755 --- a/ci/elixir-1.15-otp25/build_and_push.sh +++ b/ci/elixir-1.15-otp25/build_and_push.sh @@ -1 +1 @@ -docker buildx build --platform linux/amd64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push . +docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push . From f8411a351de07f14fdc9c9eca30109feaadf6f93 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 20 May 2024 13:30:31 +0400 Subject: [PATCH 04/15] CI: Specify version fully in base image tag --- ci/elixir-1.13/build_and_push.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/elixir-1.13/build_and_push.sh b/ci/elixir-1.13/build_and_push.sh index 53af4245f..d848344a3 100755 --- a/ci/elixir-1.13/build_and_push.sh +++ b/ci/elixir-1.13/build_and_push.sh @@ -1 +1 @@ -docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13-otp24 --push . +docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp24 --push . From f5c029524752e1820ea29f6557647823ae89ecf1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 20 May 2024 13:32:25 +0400 Subject: [PATCH 05/15] CI: Specify correct image name. --- .gitlab-ci.yml | 2 +- ci/elixir-1.13/build_and_push.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eba769af8..21d7b2242 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: git.pleroma.social:5050/pleroma/pleroma/ci-base:1.13.4-otp24 +image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24 variables: &global_variables # Only used for the release diff --git a/ci/elixir-1.13/build_and_push.sh b/ci/elixir-1.13/build_and_push.sh index d848344a3..64e1856db 100755 --- a/ci/elixir-1.13/build_and_push.sh +++ b/ci/elixir-1.13/build_and_push.sh @@ -1 +1 @@ -docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp24 --push . +docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24 --push . From 818712f99f165011aaaad5fd82c40304004ace23 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 23 May 2024 00:35:38 +0200 Subject: [PATCH 06/15] pleroma_ctl: Use realpath(1) instead of readlink(1) From realpath(1) in POSIX 202x Draft 4.1: > If file does not name a symbolic link, readlink shall write a diagnostic > message to standard error and exit with non-zero status. Which also doesn't includes `-f`, in preference of `realpath`. --- changelog.d/realpath-over-readlink.fix | 1 + rel/files/bin/pleroma_ctl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/realpath-over-readlink.fix diff --git a/changelog.d/realpath-over-readlink.fix b/changelog.d/realpath-over-readlink.fix new file mode 100644 index 000000000..479561b95 --- /dev/null +++ b/changelog.d/realpath-over-readlink.fix @@ -0,0 +1 @@ +pleroma_ctl: Use realpath(1) instead of readlink(1) diff --git a/rel/files/bin/pleroma_ctl b/rel/files/bin/pleroma_ctl index 87c486514..6f0dba3a8 100755 --- a/rel/files/bin/pleroma_ctl +++ b/rel/files/bin/pleroma_ctl @@ -134,7 +134,7 @@ if [ -z "$1" ] || [ "$1" = "help" ]; then " else - SCRIPT=$(readlink -f "$0") + SCRIPT=$(realpath "$0") SCRIPTPATH=$(dirname "$SCRIPT") FULL_ARGS="$*" From 618b77071afb480b763a493bfcd9b376effedaaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 25 May 2024 09:10:11 +0200 Subject: [PATCH 07/15] Update pleroma_api.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/api-docs-2.skip | 0 docs/development/API/pleroma_api.md | 4 +--- 2 files changed, 1 insertion(+), 3 deletions(-) create mode 100644 changelog.d/api-docs-2.skip diff --git a/changelog.d/api-docs-2.skip b/changelog.d/api-docs-2.skip new file mode 100644 index 000000000..e69de29bb diff --git a/docs/development/API/pleroma_api.md b/docs/development/API/pleroma_api.md index 267dfc1ec..57d333ffe 100644 --- a/docs/development/API/pleroma_api.md +++ b/docs/development/API/pleroma_api.md @@ -295,9 +295,7 @@ See [Admin-API](admin_api.md) "id": "9umDrYheeY451cQnEe", "name": "Read later", "emoji": "🕓", - "source": { - "emoji": "🕓" - } + "emoji_url": null } ] ``` From 61a3b793165e92d6f23a2e59fb9b95e06737cf25 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 25 May 2024 14:20:47 -0400 Subject: [PATCH 08/15] Search backend healthcheck process --- changelog.d/search-healthcheck.add | 1 + config/config.exs | 2 +- lib/pleroma/application.ex | 3 +- lib/pleroma/search.ex | 5 ++ lib/pleroma/search/database_search.ex | 3 + lib/pleroma/search/healthcheck.ex | 85 +++++++++++++++++++++++++++ lib/pleroma/search/meilisearch.ex | 11 ++++ lib/pleroma/search/search_backend.ex | 8 +++ 8 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 changelog.d/search-healthcheck.add create mode 100644 lib/pleroma/search/healthcheck.ex diff --git a/changelog.d/search-healthcheck.add b/changelog.d/search-healthcheck.add new file mode 100644 index 000000000..4974925e7 --- /dev/null +++ b/changelog.d/search-healthcheck.add @@ -0,0 +1 @@ +Monitoring of search backend health to control the processing of jobs in the search indexing Oban queue diff --git a/config/config.exs b/config/config.exs index b69044a2b..8b9a588b7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -579,7 +579,7 @@ config :pleroma, Oban, attachments_cleanup: 1, new_users_digest: 1, mute_expire: 5, - search_indexing: 10, + search_indexing: [limit: 10, paused: true], rich_media_expiration: 2 ], plugins: [Oban.Plugins.Pruner], diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 649bb11c8..d266d1836 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -109,7 +109,8 @@ defmodule Pleroma.Application do streamer_registry() ++ background_migrators() ++ shout_child(shout_enabled?()) ++ - [Pleroma.Gopher.Server] + [Pleroma.Gopher.Server] ++ + [Pleroma.Search.Healthcheck] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index 3b266e59b..e8dbcca1f 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -14,4 +14,9 @@ defmodule Pleroma.Search do search_module.search(options[:for_user], query, options) end + + def healthcheck_endpoints do + search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + search_module.healthcheck_endpoints + end end diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 31bfc7e33..11e99e7f1 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -48,6 +48,9 @@ defmodule Pleroma.Search.DatabaseSearch do @impl true def remove_from_index(_object), do: :ok + @impl true + def healthcheck_endpoints, do: nil + def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) end diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex new file mode 100644 index 000000000..495aee930 --- /dev/null +++ b/lib/pleroma/search/healthcheck.ex @@ -0,0 +1,85 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only +defmodule Pleroma.Search.Healthcheck do + @doc """ + Monitors health of search backend to control processing of events based on health and availability. + """ + use GenServer + require Logger + + @tick :timer.seconds(60) + @queue :search_indexing + + def start_link(_) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + @impl true + def init(_) do + state = %{healthy: false} + {:ok, state, {:continue, :start}} + end + + @impl true + def handle_continue(:start, state) do + tick() + {:noreply, state} + end + + @impl true + def handle_info(:check, state) do + urls = Pleroma.Search.healthcheck_endpoints() + + new_state = + if healthy?(urls) do + Oban.resume_queue(queue: @queue) + Map.put(state, :healthy, true) + else + Oban.pause_queue(queue: @queue) + Map.put(state, :healthy, false) + end + + maybe_log_state_change(state, new_state) + + tick() + {:noreply, new_state} + end + + @impl true + def handle_call(:check, _from, state) do + status = Map.get(state, :healthy) + + {:reply, status, state, :hibernate} + end + + defp healthy?([]), do: true + + defp healthy?(urls) when is_list(urls) do + Enum.all?( + urls, + fn url -> + case Pleroma.HTTP.get(url) do + {:ok, %{status: 200}} -> true + _ -> false + end + end + ) + end + + defp healthy?(_), do: true + + defp tick do + Process.send_after(self(), :check, @tick) + end + + defp maybe_log_state_change(%{healthy: true}, %{healthy: false}) do + Logger.error("Pausing Oban queue #{@queue} due to search backend healthcheck failure") + end + + defp maybe_log_state_change(%{healthy: false}, %{healthy: true}) do + Logger.info("Resuming Oban queue #{@queue} due to search backend healthcheck pass") + end + + defp maybe_log_state_change(_, _), do: :ok +end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 2bff663e8..08c2f3d86 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -178,4 +178,15 @@ defmodule Pleroma.Search.Meilisearch do def remove_from_index(object) do meili_delete("/indexes/objects/documents/#{object.id}") end + + @impl true + def healthcheck_endpoints do + endpoint = + Config.get([Pleroma.Search.Meilisearch, :url]) + |> URI.parse() + |> Map.put(:path, "/health") + |> URI.to_string() + + [endpoint] + end end diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex index 68bc48cec..13c887bc2 100644 --- a/lib/pleroma/search/search_backend.ex +++ b/lib/pleroma/search/search_backend.ex @@ -21,4 +21,12 @@ defmodule Pleroma.Search.SearchBackend do from index. """ @callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()} + + @doc """ + Healthcheck endpoints of search backend infrastructure to monitor for controlling + processing of jobs in the Oban queue. + + It is expected a 200 response is healthy and other responses are unhealthy. + """ + @callback healthcheck_endpoints :: list() | nil end From 3474b42ce396150b21f26ed35bea46ad61f57d5f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 25 May 2024 16:55:29 -0400 Subject: [PATCH 09/15] Drop TTL to 5 seconds --- lib/pleroma/search/healthcheck.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex index 495aee930..9a2d9fdd6 100644 --- a/lib/pleroma/search/healthcheck.ex +++ b/lib/pleroma/search/healthcheck.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Search.Healthcheck do use GenServer require Logger - @tick :timer.seconds(60) + @tick :timer.seconds(5) @queue :search_indexing def start_link(_) do From 354b700bedf8ad6e9187245977165ebd7bc2fa1c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 26 May 2024 14:01:00 -0400 Subject: [PATCH 10/15] Assert that AWS URLs without query parameters do not crash --- .../web/rich_media/parser/ttl/aws_signed_url_test.exs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs b/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs index cd8be8675..cc28aa7f3 100644 --- a/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs +++ b/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs @@ -10,6 +10,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do alias Pleroma.UnstubbedConfigMock, as: ConfigMock alias Pleroma.Web.RichMedia.Card + alias Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl setup do ConfigMock @@ -82,6 +83,12 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do assert DateTime.diff(scheduled_at, timestamp_dt) == valid_till end + test "AWS URL for an image without expiration works" do + og_data = %{"image" => "https://amazonaws.com/image.png"} + + assert is_nil(AwsSignedUrl.ttl(og_data, "")) + end + defp construct_s3_url(timestamp, valid_till) do "https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{timestamp}&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host" end From 807782b7f96ee0e053ad59b464766d750f8a8800 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 25 May 2024 16:27:59 -0400 Subject: [PATCH 11/15] Fix rich media parsing some Amazon URLs --- changelog.d/richmediattl.fix | 1 + lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/richmediattl.fix diff --git a/changelog.d/richmediattl.fix b/changelog.d/richmediattl.fix new file mode 100644 index 000000000..98de63015 --- /dev/null +++ b/changelog.d/richmediattl.fix @@ -0,0 +1 @@ +Parsing of RichMedia TTLs for Amazon URLs when query parameters are nil diff --git a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex index 948c727e1..1172a120a 100644 --- a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex +++ b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do %URI{host: host, query: query} = URI.parse(image) is_binary(host) and String.contains?(host, "amazonaws.com") and - String.contains?(query, "X-Amz-Expires") + is_binary(query) and String.contains?(query, "X-Amz-Expires") end defp aws_signed_url?(_), do: nil From f2b0d5f1d02e243a7a1a6f339b59e5abcb8e1bd8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 26 May 2024 14:11:41 -0400 Subject: [PATCH 12/15] Make it easier to read the state for debugging purposes and expose functions for testing --- lib/pleroma/search/healthcheck.ex | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex index 9a2d9fdd6..170f29344 100644 --- a/lib/pleroma/search/healthcheck.ex +++ b/lib/pleroma/search/healthcheck.ex @@ -32,7 +32,7 @@ defmodule Pleroma.Search.Healthcheck do urls = Pleroma.Search.healthcheck_endpoints() new_state = - if healthy?(urls) do + if check(urls) do Oban.resume_queue(queue: @queue) Map.put(state, :healthy, true) else @@ -47,15 +47,15 @@ defmodule Pleroma.Search.Healthcheck do end @impl true - def handle_call(:check, _from, state) do - status = Map.get(state, :healthy) - - {:reply, status, state, :hibernate} + def handle_call(:state, _from, state) do + {:reply, state, state, :hibernate} end - defp healthy?([]), do: true + def state, do: GenServer.call(__MODULE__, :state) - defp healthy?(urls) when is_list(urls) do + def check([]), do: true + + def check(urls) when is_list(urls) do Enum.all?( urls, fn url -> @@ -67,7 +67,7 @@ defmodule Pleroma.Search.Healthcheck do ) end - defp healthy?(_), do: true + def check(_), do: true defp tick do Process.send_after(self(), :check, @tick) From 03f4b461895802259c895c81462a3e9d0d31c1e5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 26 May 2024 14:21:24 -0400 Subject: [PATCH 13/15] Test that healthchecks behave correctly for the expected HTTP responses --- test/pleroma/search/healthcheck_test.exs | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 test/pleroma/search/healthcheck_test.exs diff --git a/test/pleroma/search/healthcheck_test.exs b/test/pleroma/search/healthcheck_test.exs new file mode 100644 index 000000000..e7649d949 --- /dev/null +++ b/test/pleroma/search/healthcheck_test.exs @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.HealthcheckTest do + use Pleroma.DataCase + + import Tesla.Mock + + alias Pleroma.Search.Healthcheck + + @good1 "http://good1.example.com/healthz" + @good2 "http://good2.example.com/health" + @bad "http://bad.example.com/healthy" + + setup do + mock(fn + %{method: :get, url: @good1} -> + %Tesla.Env{ + status: 200, + body: "" + } + + %{method: :get, url: @good2} -> + %Tesla.Env{ + status: 200, + body: "" + } + + %{method: :get, url: @bad} -> + %Tesla.Env{ + status: 503, + body: "" + } + end) + + :ok + end + + test "true for 200 responses" do + assert Healthcheck.check([@good1]) + assert Healthcheck.check([@good1, @good2]) + end + + test "false if any response is not a 200" do + refute Healthcheck.check([@bad]) + refute Healthcheck.check([@good1, @bad]) + end +end From d9b82255b9cf49176f8ef1d5a87abf7d80769a47 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 26 May 2024 15:23:12 -0400 Subject: [PATCH 14/15] Add an HTTP timeout for the healthcheck --- lib/pleroma/search/healthcheck.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex index 170f29344..e562c8478 100644 --- a/lib/pleroma/search/healthcheck.ex +++ b/lib/pleroma/search/healthcheck.ex @@ -8,8 +8,9 @@ defmodule Pleroma.Search.Healthcheck do use GenServer require Logger - @tick :timer.seconds(5) @queue :search_indexing + @tick :timer.seconds(5) + @timeout :timer.seconds(2) def start_link(_) do GenServer.start_link(__MODULE__, [], name: __MODULE__) @@ -59,7 +60,7 @@ defmodule Pleroma.Search.Healthcheck do Enum.all?( urls, fn url -> - case Pleroma.HTTP.get(url) do + case Pleroma.HTTP.get(url, [], recv_timeout: @timeout) do {:ok, %{status: 200}} -> true _ -> false end From d35b69d2686e62cc5076bd7a33449f98f8a11a85 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 13:18:02 +0400 Subject: [PATCH 15/15] Pleroma.Search: Remove wrong (but irrelevant) results --- lib/pleroma/search.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index e8dbcca1f..fd0218cb8 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -10,13 +10,12 @@ defmodule Pleroma.Search do end def search(query, options) do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) - + search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.search(options[:for_user], query, options) end def healthcheck_endpoints do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.healthcheck_endpoints end end