Custom mix task to retry failed tests once in CI pipeline

This will be temporary* as we hunt down the cause of the random test failures

* gonna regret this

.gitlab-ci.yml

@@ -135,7 +135,7 @@ unit-testing-1.13.4-otp-25:
   script: &testing_script
     - mix ecto.create
     - mix ecto.migrate
-    - mix test --cover --preload-modules
+    - mix pleroma.test_runner --cover --preload-modules
   coverage: '/^Line total: ([^ ]*%)$/'
   artifacts:
     reports:
@@ -165,7 +165,7 @@ unit-testing-1.13.4-otp-25-erratic:
   script:
     - mix ecto.create
     - mix ecto.migrate
-    - mix test --only=erratic
+    - mix pleroma.test_runner --only=erratic
 
 formatting-1.15:
   extends: .build_changes_policy

lib/mix/tasks/pleroma/test_runner.ex

@@ -0,0 +1,25 @@
+defmodule Mix.Tasks.Pleroma.TestRunner do
+  @shortdoc "Retries tests once if they fail"
+
+  use Mix.Task
+
+  def run(args \\ []) do
+    case System.cmd("mix", ["test"] ++ args, into: IO.stream(:stdio, :line)) do
+      {_, 0} ->
+        :ok
+
+      _ ->
+        retry(args)
+    end
+  end
+
+  def retry(args) do
+    case System.cmd("mix", ["test", "--failed"] ++ args, into: IO.stream(:stdio, :line)) do
+      {_, 0} ->
+        :ok
+
+      _ ->
+        exit(1)
+    end
+  end
+end