From a0518a4ee171598dddfc6ae9202074cfe13945d5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 22:35:08 -0500 Subject: [PATCH 01/10] Add a build and test pipeline for elixir 1.15 with a new naming convention --- .gitlab-ci.yml | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eb31a8086..1b581d50f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -71,7 +71,7 @@ check-changelog: tags: - amd64 -build: +build-1.12.3: extends: - .build_changes_policy - .using-ci-base @@ -79,6 +79,16 @@ build: script: - mix compile --force +build-1.15.7-otp-25: + extends: + - .build_changes_policy + - .using-ci-base + stage: build + image: elixir:1.15.7-otp-25 + allow_failure: true + script: + - mix compile --force + spec-build: extends: - .using-ci-base @@ -110,20 +120,17 @@ benchmark: - mix ecto.migrate - mix pleroma.load_testing -unit-testing: +unit-testing-1.12.3: extends: - .build_changes_policy - .using-ci-base stage: test cache: &testing_cache_policy - <<: *global_cache_policy - policy: pull - - services: + services: &testing_services - name: postgres:13-alpine alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - script: + script: &testing_script - mix ecto.create - mix ecto.migrate - mix test --cover --preload-modules @@ -134,6 +141,17 @@ unit-testing: coverage_format: cobertura path: coverage.xml +unit-testing-1.15.7-otp-25: + extends: + - .build_changes_policy + - .using-ci-base + stage: test + image: elixir:1.15.7-otp-25 + allow_failure: true + cache: *testing_cache_policy + services: *testing_services + script: *testing_script + unit-testing-erratic: extends: - .build_changes_policy @@ -141,14 +159,8 @@ unit-testing-erratic: stage: test retry: 2 allow_failure: true - cache: &testing_cache_policy - <<: *global_cache_policy - policy: pull - - services: - - name: postgres:13-alpine - alias: postgres - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + cache: *testing_cache_policy + services: *testing_services script: - mix ecto.create - mix ecto.migrate From df31ec0d52e74ed04021c2b6f0166083117cc7b0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 22:43:41 -0500 Subject: [PATCH 02/10] Linting as a separate stage --- .gitlab-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b581d50f..4bf29b5a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,7 @@ cache: &global_cache_policy stages: - check-changelog - build + - lint - test - benchmark - deploy @@ -185,10 +186,10 @@ unit-testing-rum: - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" - mix test --preload-modules -lint: +formatting: extends: .build_changes_policy image: ¤t_elixir elixir:1.13-alpine - stage: test + stage: lint cache: *testing_cache_policy before_script: ¤t_bfr_script - apk update @@ -203,7 +204,7 @@ analysis: extends: - .build_changes_policy - .using-ci-base - stage: test + stage: lint cache: *testing_cache_policy script: - mix credo --strict --only=warnings,todo,fixme,consistency,readability @@ -211,7 +212,7 @@ analysis: cycles: extends: .build_changes_policy image: *current_elixir - stage: test + stage: lint cache: {} before_script: *current_bfr_script script: From 06ac829eb4cc5cc9b9c34ec711be134be93e97cc Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 22:45:29 -0500 Subject: [PATCH 03/10] Spec building should be in build stage --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4bf29b5a5..635d6a6a8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,7 +93,7 @@ build-1.15.7-otp-25: spec-build: extends: - .using-ci-base - stage: test + stage: build rules: - changes: - ".gitlab-ci.yml" From 17904003139f72780fcb151491e706f3cdd2374d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 22:49:37 -0500 Subject: [PATCH 04/10] Add Dialyxir with manual job execution --- .gitlab-ci.yml | 13 +++++++++++++ mix.exs | 3 ++- mix.lock | 2 ++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 635d6a6a8..fde220020 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -209,6 +209,19 @@ analysis: script: - mix credo --strict --only=warnings,todo,fixme,consistency,readability +dialyzer: + extends: + - .build_changes_policy + - .using-ci-base + stage: lint + allow_failure: true + when: manual + cache: *testing_cache_policy + tags: + - feld + script: + - mix dialyzer + cycles: extends: .build_changes_policy image: *current_elixir diff --git a/mix.exs b/mix.exs index 2e622d9da..541a60555 100644 --- a/mix.exs +++ b/mix.exs @@ -194,7 +194,8 @@ defmodule Pleroma.Mixfile do {:hackney, "~> 1.18.0", override: true}, {:mox, "~> 1.0", only: :test}, {:websockex, "~> 0.4.3", only: :test}, - {:benchee, "~> 1.0", only: :benchmark} + {:benchee, "~> 1.0", only: :benchmark}, + {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false} ] ++ oauth_deps() end diff --git a/mix.lock b/mix.lock index e5e2c0baa..e731a2b98 100644 --- a/mix.lock +++ b/mix.lock @@ -27,6 +27,7 @@ "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, + "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, "earmark": {:hex, :earmark, "1.4.22", "ea3e45c6359446dc308be0a64ce82a03260d973de7d0625a762e6d352ff57958", [:mix], [{:earmark_parser, "~> 1.4.23", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "1caf5145665a42fd76d5317286b0c171861fb1c04f86ab103dde76868814fdfb"}, "earmark_parser": {:hex, :earmark_parser, "1.4.32", "fa739a0ecfa34493de19426681b23f6814573faee95dfd4b4aafe15a7b5b32c6", [:mix], [], "hexpm", "b8b0dd77d60373e77a3d7e8afa598f325e49e8663a51bcc2b88ef41838cca755"}, "eblurhash": {:git, "https://github.com/zotonic/eblurhash.git", "bc37ceb426ef021ee9927fb249bb93f7059194ab", [ref: "bc37ceb426ef021ee9927fb249bb93f7059194ab"]}, @@ -36,6 +37,7 @@ "ecto_sql": {:hex, :ecto_sql, "3.10.2", "6b98b46534b5c2f8b8b5f03f126e75e2a73c64f3c071149d32987a5378b0fdbd", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "68c018debca57cb9235e3889affdaec7a10616a4e3a80c99fa1d01fdafaa9007"}, "eimp": {:hex, :eimp, "1.0.14", "fc297f0c7e2700457a95a60c7010a5f1dcb768a083b6d53f49cd94ab95a28f22", [:rebar3], [{:p1_utils, "1.0.18", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "501133f3112079b92d9e22da8b88bf4f0e13d4d67ae9c15c42c30bd25ceb83b6"}, "elixir_make": {:hex, :elixir_make, "0.7.7", "7128c60c2476019ed978210c245badf08b03dbec4f24d05790ef791da11aa17c", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5bc19fff950fad52bbe5f211b12db9ec82c6b34a9647da0c2224b8b8464c7e6c"}, + "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "esbuild": {:hex, :esbuild, "0.5.0", "d5bb08ff049d7880ee3609ed5c4b864bd2f46445ea40b16b4acead724fb4c4a3", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "f183a0b332d963c4cfaf585477695ea59eef9a6f2204fdd0efa00e099694ffe5"}, "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"}, "ex_aws": {:hex, :ex_aws, "2.1.9", "dc4865ecc20a05190a34a0ac5213e3e5e2b0a75a0c2835e923ae7bfeac5e3c31", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "3e6c776703c9076001fbe1f7c049535f042cb2afa0d2cbd3b47cbc4e92ac0d10"}, From 68f421c203b8b5b178abf0f6dd0e10ee09063d5a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 22:53:07 -0500 Subject: [PATCH 05/10] Use our own 1.15 ci-base image --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fde220020..094e45cf5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -85,7 +85,7 @@ build-1.15.7-otp-25: - .build_changes_policy - .using-ci-base stage: build - image: elixir:1.15.7-otp-25 + image: git.pleroma.social:5050/pleroma/pleroma/ci-base-1.15 allow_failure: true script: - mix compile --force @@ -147,7 +147,7 @@ unit-testing-1.15.7-otp-25: - .build_changes_policy - .using-ci-base stage: test - image: elixir:1.15.7-otp-25 + image: git.pleroma.social:5050/pleroma/pleroma/ci-base-1.15 allow_failure: true cache: *testing_cache_policy services: *testing_services From 06813d4a0eb710fa5ac78333af15ce912a0c3d84 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 21 Jan 2024 03:58:53 +0000 Subject: [PATCH 06/10] Reorganize ci scripts --- ci/build_and_push.sh | 1 - ci/{ => elixir-1.12}/Dockerfile | 0 ci/elixir-1.12/build_and_push.sh | 1 + ci/elixir-1.15/Dockerfile | 8 ++++++++ ci/elixir-1.15/build_and_push.sh | 1 + ci/{postgres_rum => postgres-with-rum-13}/Dockerfile | 0 .../build_and_push.sh | 0 7 files changed, 10 insertions(+), 1 deletion(-) delete mode 100755 ci/build_and_push.sh rename ci/{ => elixir-1.12}/Dockerfile (100%) create mode 100755 ci/elixir-1.12/build_and_push.sh create mode 100644 ci/elixir-1.15/Dockerfile create mode 100755 ci/elixir-1.15/build_and_push.sh rename ci/{postgres_rum => postgres-with-rum-13}/Dockerfile (100%) rename ci/{postgres_rum => postgres-with-rum-13}/build_and_push.sh (100%) diff --git a/ci/build_and_push.sh b/ci/build_and_push.sh deleted file mode 100755 index 484cc2643..000000000 --- a/ci/build_and_push.sh +++ /dev/null @@ -1 +0,0 @@ -docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:latest --push . diff --git a/ci/Dockerfile b/ci/elixir-1.12/Dockerfile similarity index 100% rename from ci/Dockerfile rename to ci/elixir-1.12/Dockerfile diff --git a/ci/elixir-1.12/build_and_push.sh b/ci/elixir-1.12/build_and_push.sh new file mode 100755 index 000000000..508262ed8 --- /dev/null +++ b/ci/elixir-1.12/build_and_push.sh @@ -0,0 +1 @@ +docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.12 --push . diff --git a/ci/elixir-1.15/Dockerfile b/ci/elixir-1.15/Dockerfile new file mode 100644 index 000000000..a2b566873 --- /dev/null +++ b/ci/elixir-1.15/Dockerfile @@ -0,0 +1,8 @@ +FROM elixir:1.12.3 + +# Single RUN statement, otherwise intermediate images are created +# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run +RUN apt-get update &&\ + apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\ + mix local.hex --force &&\ + mix local.rebar --force diff --git a/ci/elixir-1.15/build_and_push.sh b/ci/elixir-1.15/build_and_push.sh new file mode 100755 index 000000000..fd7ffe2de --- /dev/null +++ b/ci/elixir-1.15/build_and_push.sh @@ -0,0 +1 @@ +docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15 --push . diff --git a/ci/postgres_rum/Dockerfile b/ci/postgres-with-rum-13/Dockerfile similarity index 100% rename from ci/postgres_rum/Dockerfile rename to ci/postgres-with-rum-13/Dockerfile diff --git a/ci/postgres_rum/build_and_push.sh b/ci/postgres-with-rum-13/build_and_push.sh similarity index 100% rename from ci/postgres_rum/build_and_push.sh rename to ci/postgres-with-rum-13/build_and_push.sh From aee971bd2654f61729abc34be0401862f1308e88 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 21 Jan 2024 03:59:26 +0000 Subject: [PATCH 07/10] Only need amd64 for now --- ci/elixir-1.15/build_and_push.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/elixir-1.15/build_and_push.sh b/ci/elixir-1.15/build_and_push.sh index fd7ffe2de..79cf89344 100755 --- a/ci/elixir-1.15/build_and_push.sh +++ b/ci/elixir-1.15/build_and_push.sh @@ -1 +1 @@ -docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15 --push . +docker buildx build --platform linux/amd64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15 --push . From 058fa5471a0d0bfe9eb774e9aa34504ab4d9d90a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 21 Jan 2024 04:06:17 +0000 Subject: [PATCH 08/10] Fix the image name --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 094e45cf5..ca17be7c1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -85,7 +85,7 @@ build-1.15.7-otp-25: - .build_changes_policy - .using-ci-base stage: build - image: git.pleroma.social:5050/pleroma/pleroma/ci-base-1.15 + image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15 allow_failure: true script: - mix compile --force @@ -147,7 +147,7 @@ unit-testing-1.15.7-otp-25: - .build_changes_policy - .using-ci-base stage: test - image: git.pleroma.social:5050/pleroma/pleroma/ci-base-1.15 + image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15 allow_failure: true cache: *testing_cache_policy services: *testing_services From 8f0051d73929ad515cc1582f727d21e49af40d6a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 21 Jan 2024 04:10:20 +0000 Subject: [PATCH 09/10] Rename 1.15 image to include otp25, clarify test names --- .gitlab-ci.yml | 6 +++--- ci/{elixir-1.15 => elixir-1.15-otp25}/Dockerfile | 0 ci/{elixir-1.15 => elixir-1.15-otp25}/build_and_push.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename ci/{elixir-1.15 => elixir-1.15-otp25}/Dockerfile (100%) rename ci/{elixir-1.15 => elixir-1.15-otp25}/build_and_push.sh (52%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ca17be7c1..41132864b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -147,13 +147,13 @@ unit-testing-1.15.7-otp-25: - .build_changes_policy - .using-ci-base stage: test - image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15 + image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 allow_failure: true cache: *testing_cache_policy services: *testing_services script: *testing_script -unit-testing-erratic: +unit-testing-1.12-erratic: extends: - .build_changes_policy - .using-ci-base @@ -167,7 +167,7 @@ unit-testing-erratic: - mix ecto.migrate - mix test --only=erratic -unit-testing-rum: +unit-testing-1.12-rum: extends: - .build_changes_policy - .using-ci-base diff --git a/ci/elixir-1.15/Dockerfile b/ci/elixir-1.15-otp25/Dockerfile similarity index 100% rename from ci/elixir-1.15/Dockerfile rename to ci/elixir-1.15-otp25/Dockerfile diff --git a/ci/elixir-1.15/build_and_push.sh b/ci/elixir-1.15-otp25/build_and_push.sh similarity index 52% rename from ci/elixir-1.15/build_and_push.sh rename to ci/elixir-1.15-otp25/build_and_push.sh index 79cf89344..06fe74f34 100755 --- a/ci/elixir-1.15/build_and_push.sh +++ b/ci/elixir-1.15-otp25/build_and_push.sh @@ -1 +1 @@ -docker buildx build --platform linux/amd64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15 --push . +docker buildx build --platform linux/amd64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push . From 518ddd458c1c111decf55d3ed7d00e4d3d6f65cd Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 21 Jan 2024 04:14:41 +0000 Subject: [PATCH 10/10] Clarify formatting and cycles versions --- .gitlab-ci.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 41132864b..b6880a34c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -186,9 +186,9 @@ unit-testing-1.12-rum: - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" - mix test --preload-modules -formatting: +formatting-1.13: extends: .build_changes_policy - image: ¤t_elixir elixir:1.13-alpine + image: &formatting_elixir elixir:1.13-alpine stage: lint cache: *testing_cache_policy before_script: ¤t_bfr_script @@ -200,6 +200,16 @@ formatting: script: - mix format --check-formatted +cycles-1.13: + extends: .build_changes_policy + image: *formatting_elixir + stage: lint + cache: {} + before_script: *current_bfr_script + script: + - mix compile + - mix xref graph --format cycles --label compile | awk '{print $0} END{exit ($0 != "No cycles found")}' + analysis: extends: - .build_changes_policy @@ -222,16 +232,6 @@ dialyzer: script: - mix dialyzer -cycles: - extends: .build_changes_policy - image: *current_elixir - stage: lint - cache: {} - before_script: *current_bfr_script - script: - - mix compile - - mix xref graph --format cycles --label compile | awk '{print $0} END{exit ($0 != "No cycles found")}' - docs-deploy: stage: deploy cache: *testing_cache_policy