Merge branch 'ipfs_uploader' into 'develop'

feat: simple, but not stupid, uploader for IPFS

See merge request pleroma/pleroma!3654

changelog.d/add-ipfs-upload.add

@@ -0,0 +1 @@
+Uploader: Add support for uploading attachments using IPFS

config/config.exs

@@ -82,6 +82,10 @@ config :ex_aws, :s3,
   # region: "us-east-1", # may be required for Amazon AWS
   scheme: "https://"
 
+config :pleroma, Pleroma.Uploaders.IPFS,
+  post_gateway_url: nil,
+  get_gateway_url: nil
+
 config :pleroma, :emoji,
   shortcode_globs: ["/emoji/custom/**/*.png"],
   pack_extensions: [".png", ".gif"],

config/description.exs

@@ -136,6 +136,31 @@ config :pleroma, :config_description, [
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: Pleroma.Uploaders.IPFS,
+    type: :group,
+    description: "IPFS uploader-related settings",
+    children: [
+      %{
+        key: :get_gateway_url,
+        type: :string,
+        description: "GET Gateway URL",
+        suggestions: [
+          "https://ipfs.mydomain.com/{CID}",
+          "https://{CID}.ipfs.mydomain.com/"
+        ]
+      },
+      %{
+        key: :post_gateway_url,
+        type: :string,
+        description: "POST Gateway URL",
+        suggestions: [
+          "http://localhost:5001/"
+        ]
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: Pleroma.Uploaders.S3,

config/test.exs

@@ -153,6 +153,7 @@ config :pleroma, Pleroma.Uploaders.S3, config_impl: Pleroma.UnstubbedConfigMock
 config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock
 config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock
 config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock
+config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock
 
 peer_module =
   if String.to_integer(System.otp_release()) >= 25 do

docs/configuration/cheatsheet.md

@@ -661,6 +661,19 @@ config :ex_aws, :s3,
   host: "s3.eu-central-1.amazonaws.com"
 ```
 
+#### Pleroma.Uploaders.IPFS
+
+* `post_gateway_url`: URL with port of POST Gateway (unauthenticated)
+* `get_gateway_url`: URL of public GET Gateway
+
+Example:
+
+```elixir
+config :pleroma, Pleroma.Uploaders.IPFS,
+  post_gateway_url: "http://localhost:5001",
+  get_gateway_url: "http://{CID}.ipfs.mydomain.com"
+```
+
 ### Upload filters
 
 #### Pleroma.Upload.Filter.AnonymizeFilename

lib/pleroma/upload.ex

@@ -239,9 +239,13 @@ defmodule Pleroma.Upload do
           ""
         end
 
+    if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do
+      String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path)
+    else
       [base_url, path]
       |> Path.join()
     end
+  end
 
   defp url_from_spec(_upload, _base_url, {:url, url}), do: url
 
@@ -277,6 +281,9 @@ defmodule Pleroma.Upload do
           Path.join([upload_base_url, bucket_with_namespace])
         end
 
+      Pleroma.Uploaders.IPFS ->
+        @config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url])
+
       _ ->
         public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
     end

lib/pleroma/uploaders/ipfs.ex

@@ -0,0 +1,77 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Uploaders.IPFS do
+  @behaviour Pleroma.Uploaders.Uploader
+  require Logger
+
+  alias Tesla.Multipart
+
+  @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
+
+  defp get_final_url(method) do
+    config = @config_impl.get([__MODULE__])
+    post_base_url = Keyword.get(config, :post_gateway_url)
+
+    Path.join([post_base_url, method])
+  end
+
+  def put_file_endpoint do
+    get_final_url("/api/v0/add")
+  end
+
+  def delete_file_endpoint do
+    get_final_url("/api/v0/files/rm")
+  end
+
+  @placeholder "{CID}"
+  def placeholder, do: @placeholder
+
+  @impl true
+  def get_file(file) do
+    b_url = Pleroma.Upload.base_url()
+
+    if String.contains?(b_url, @placeholder) do
+      {:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}}
+    else
+      {:error, "IPFS Get URL doesn't contain 'cid' placeholder"}
+    end
+  end
+
+  @impl true
+  def put_file(%Pleroma.Upload{} = upload) do
+    mp =
+      Multipart.new()
+      |> Multipart.add_content_type_param("charset=utf-8")
+      |> Multipart.add_file(upload.tempfile)
+
+    case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do
+      {:ok, ret} ->
+        case Jason.decode(ret.body) do
+          {:ok, ret} ->
+            if Map.has_key?(ret, "Hash") do
+              {:ok, {:file, ret["Hash"]}}
+            else
+              {:error, "JSON doesn't contain Hash key"}
+            end
+
+          error ->
+            Logger.error("#{__MODULE__}: #{inspect(error)}")
+            {:error, "JSON decode failed"}
+        end
+
+      error ->
+        Logger.error("#{__MODULE__}: #{inspect(error)}")
+        {:error, "IPFS Gateway upload failed"}
+    end
+  end
+
+  @impl true
+  def delete_file(file) do
+    case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do
+      {:ok, %{status: 204}} -> :ok
+      error -> {:error, inspect(error)}
+    end
+  end
+end

test/pleroma/uploaders/ipfs_test.exs

@@ -0,0 +1,158 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Uploaders.IPFSTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Uploaders.IPFS
+  alias Tesla.Multipart
+
+  import ExUnit.CaptureLog
+  import Mock
+  import Mox
+
+  alias Pleroma.UnstubbedConfigMock, as: Config
+
+  describe "get_final_url" do
+    setup do
+      Config
+      |> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
+        [post_gateway_url: "http://localhost:5001"]
+      end)
+
+      :ok
+    end
+
+    test "it returns the final url for put_file" do
+      assert IPFS.put_file_endpoint() == "http://localhost:5001/api/v0/add"
+    end
+
+    test "it returns the final url for delete_file" do
+      assert IPFS.delete_file_endpoint() == "http://localhost:5001/api/v0/files/rm"
+    end
+  end
+
+  describe "get_file/1" do
+    setup do
+      Config
+      |> expect(:get, fn [Pleroma.Upload, :uploader] -> Pleroma.Uploaders.IPFS end)
+      |> expect(:get, fn [Pleroma.Upload, :base_url] -> nil end)
+      |> expect(:get, fn [Pleroma.Uploaders.IPFS, :public_endpoint] -> nil end)
+
+      :ok
+    end
+
+    test "it returns path to ipfs file with cid as subdomain" do
+      Config
+      |> expect(:get, fn [Pleroma.Uploaders.IPFS, :get_gateway_url] ->
+        "https://{CID}.ipfs.mydomain.com"
+      end)
+
+      assert IPFS.get_file("testcid") == {
+               :ok,
+               {:url, "https://testcid.ipfs.mydomain.com"}
+             }
+    end
+
+    test "it returns path to ipfs file with cid as path" do
+      Config
+      |> expect(:get, fn [Pleroma.Uploaders.IPFS, :get_gateway_url] ->
+        "https://ipfs.mydomain.com/ipfs/{CID}"
+      end)
+
+      assert IPFS.get_file("testcid") == {
+               :ok,
+               {:url, "https://ipfs.mydomain.com/ipfs/testcid"}
+             }
+    end
+  end
+
+  describe "put_file/1" do
+    setup do
+      Config
+      |> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
+        [post_gateway_url: "http://localhost:5001"]
+      end)
+
+      file_upload = %Pleroma.Upload{
+        name: "image-tet.jpg",
+        content_type: "image/jpeg",
+        path: "test_folder/image-tet.jpg",
+        tempfile: Path.absname("test/instance_static/add/shortcode.png")
+      }
+
+      mp =
+        Multipart.new()
+        |> Multipart.add_content_type_param("charset=utf-8")
+        |> Multipart.add_file(file_upload.tempfile)
+
+      [file_upload: file_upload, mp: mp]
+    end
+
+    test "save file", %{file_upload: file_upload} do
+      with_mock Pleroma.HTTP,
+        post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
+          {:ok,
+           %Tesla.Env{
+             status: 200,
+             body:
+               "{\"Name\":\"image-tet.jpg\",\"Size\":\"5000\", \"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}"
+           }}
+        end do
+        assert IPFS.put_file(file_upload) ==
+                 {:ok, {:file, "bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi"}}
+      end
+    end
+
+    test "returns error", %{file_upload: file_upload} do
+      with_mock Pleroma.HTTP,
+        post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
+          {:error, "IPFS Gateway upload failed"}
+        end do
+        assert capture_log(fn ->
+                 assert IPFS.put_file(file_upload) == {:error, "IPFS Gateway upload failed"}
+               end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}"
+      end
+    end
+
+    test "returns error if JSON decode fails", %{file_upload: file_upload} do
+      with_mock Pleroma.HTTP, [],
+        post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
+          {:ok, %Tesla.Env{status: 200, body: "invalid"}}
+        end do
+        assert capture_log(fn ->
+                 assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"}
+               end) =~
+                 "Elixir.Pleroma.Uploaders.IPFS: {:error, %Jason.DecodeError"
+      end
+    end
+
+    test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do
+      with_mock Pleroma.HTTP, [],
+        post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
+          {:ok, %Tesla.Env{status: 200, body: "{\"key\": \"value\"}"}}
+        end do
+        assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"}
+      end
+    end
+  end
+
+  describe "delete_file/1" do
+    setup do
+      Config
+      |> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
+        [post_gateway_url: "http://localhost:5001"]
+      end)
+
+      :ok
+    end
+
+    test_with_mock "deletes file", Pleroma.HTTP,
+      post: fn "http://localhost:5001/api/v0/files/rm", "", [], params: [arg: "image.jpg"] ->
+        {:ok, %{status: 204}}
+      end do
+      assert :ok = IPFS.delete_file("image.jpg")
+    end
+  end
+end