Merge branch 'develop' into 'fix/hide-follows-counters'

# Conflicts:
#   CHANGELOG.md

CHANGELOG.md

@@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`)
 - Mastodon API: Embedded relationships not being properly rendered in the Account entity of Status entity
 - Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set
+- Mastodon API: `muted` in the Status entity, using author's account to determine if the tread was muted
 - Mastodon API: Add `account_id`, `type`, `offset`, and `limit` to search API (`/api/v1/search` and `/api/v2/search`)
 - Mastodon API, streaming: Fix filtering of notifications based on blocks/mutes/thread mutes
 - ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set
@@ -73,6 +74,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Admin API: Endpoint for fetching latest user's statuses
 - Pleroma API: Add `/api/v1/pleroma/accounts/confirmation_resend?email=<email>` for resending account confirmation.
 - Relays: Added a task to list relay subscriptions.
+- Mix Tasks: `mix pleroma.database fix_likes_collections`
+- Federation: Remove `likes` from objects.
 
 ### Changed
 - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text

docs/config.md

@@ -18,6 +18,7 @@ Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`.
 
 ## Pleroma.Uploaders.S3
 * `bucket`: S3 bucket name
+* `bucket_namespace`: S3 bucket namespace
 * `public_endpoint`: S3 endpoint that the user finally accesses(ex. "https://s3.dualstack.ap-northeast-1.amazonaws.com")
 * `truncated_namespace`: If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or "" etc.
 For example, when using CDN to S3 virtual host format, set "".

lib/mix/tasks/pleroma/database.ex

@@ -36,6 +36,10 @@ defmodule Mix.Tasks.Pleroma.Database do
   ## Remove duplicated items from following and update followers count for all users
 
       mix pleroma.database update_users_following_followers_counts
+
+  ## Fix the pre-existing "likes" collections for all objects
+
+      mix pleroma.database fix_likes_collections
   """
   def run(["remove_embedded_objects" | args]) do
     {options, [], []} =
@@ -125,4 +129,36 @@ defmodule Mix.Tasks.Pleroma.Database do
       )
     end
   end
+
+  def run(["fix_likes_collections"]) do
+    import Ecto.Query
+
+    start_pleroma()
+
+    from(object in Object,
+      where: fragment("(?)->>'likes' is not null", object.data),
+      select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)}
+    )
+    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Stream.each(fn objects ->
+      ids =
+        objects
+        |> Enum.filter(fn object -> object.likes |> Jason.decode!() |> is_map() end)
+        |> Enum.map(& &1.id)
+
+      Object
+      |> where([object], object.id in ^ids)
+      |> update([object],
+        set: [
+          data:
+            fragment(
+              "jsonb_set(?, '{likes}', '[]'::jsonb, true)",
+              object.data
+            )
+        ]
+      )
+      |> Repo.update_all([], timeout: :infinity)
+    end)
+    |> Stream.run()
+  end
 end

lib/pleroma/uploaders/local.ex

@@ -11,7 +11,7 @@ defmodule Pleroma.Uploaders.Local do
 
   def put_file(upload) do
     {local_path, file} =
-      case Enum.reverse(String.split(upload.path, "/", trim: true)) do
+      case Enum.reverse(Path.split(upload.path)) do
         [file] ->
           {upload_path(), file}
 
@@ -23,7 +23,7 @@ defmodule Pleroma.Uploaders.Local do
 
     result_file = Path.join(local_path, file)
 
-    unless File.exists?(result_file) do
+    if not File.exists?(result_file) do
       File.cp!(upload.tempfile, result_file)
     end
 

lib/pleroma/uploaders/mdii.ex

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Uploaders.MDII do
+  @moduledoc "Represents uploader for https://github.com/hakaba-hitoyo/minimal-digital-image-infrastructure"
+
   alias Pleroma.Config
   alias Pleroma.HTTP
 

lib/pleroma/uploaders/s3.ex

@@ -6,10 +6,12 @@ defmodule Pleroma.Uploaders.S3 do
   @behaviour Pleroma.Uploaders.Uploader
   require Logger
 
+  alias Pleroma.Config
+
   # The file name is re-encoded with S3's constraints here to comply with previous
   # links with less strict filenames
   def get_file(file) do
-    config = Pleroma.Config.get([__MODULE__])
+    config = Config.get([__MODULE__])
     bucket = Keyword.fetch!(config, :bucket)
 
     bucket_with_namespace =
@@ -34,15 +36,15 @@ defmodule Pleroma.Uploaders.S3 do
   end
 
   def put_file(%Pleroma.Upload{} = upload) do
-    config = Pleroma.Config.get([__MODULE__])
+    config = Config.get([__MODULE__])
     bucket = Keyword.get(config, :bucket)
 
-    {:ok, file_data} = File.read(upload.tempfile)
-
     s3_name = strict_encode(upload.path)
 
     op =
-      ExAws.S3.put_object(bucket, s3_name, file_data, [
+      upload.tempfile
+      |> ExAws.S3.Upload.stream_file()
+      |> ExAws.S3.upload(bucket, s3_name, [
         {:acl, :public_read},
         {:content_type, upload.content_type}
       ])

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -26,6 +26,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   """
   def fix_object(object, options \\ []) do
     object
+    |> strip_internal_fields
     |> fix_actor
     |> fix_url
     |> fix_attachments
@@ -34,7 +35,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> fix_emoji
     |> fix_tag
     |> fix_content_map
-    |> fix_likes
     |> fix_addressing
     |> fix_summary
     |> fix_type(options)
@@ -151,20 +151,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> Map.put("actor", Containment.get_actor(%{"actor" => actor}))
   end
 
-  # Check for standardisation
-  # This is what Peertube does
-  # curl -H 'Accept: application/activity+json' $likes | jq .totalItems
-  # Prismo returns only an integer (count) as "likes"
-  def fix_likes(%{"likes" => likes} = object) when not is_map(likes) do
-    object
-    |> Map.put("likes", [])
-    |> Map.put("like_count", 0)
-  end
-
-  def fix_likes(object) do
-    object
-  end
-
   def fix_in_reply_to(object, options \\ [])
 
   def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options)
@@ -784,7 +770,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> add_mention_tags
     |> add_emoji_tags
     |> add_attributed_to
-    |> add_likes
     |> prepare_attachments
     |> set_conversation
     |> set_reply_to_uri
@@ -971,22 +956,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> Map.put("attributedTo", attributed_to)
   end
 
-  def add_likes(%{"id" => id, "like_count" => likes} = object) do
-    likes = %{
-      "id" => "#{id}/likes",
-      "first" => "#{id}/likes?page=1",
-      "type" => "OrderedCollection",
-      "totalItems" => likes
-    }
-
-    object
-    |> Map.put("likes", likes)
-  end
-
-  def add_likes(object) do
-    object
-  end
-
   def prepare_attachments(object) do
     attachments =
       (object["attachment"] || [])
@@ -1002,6 +971,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   defp strip_internal_fields(object) do
     object
     |> Map.drop([
+      "likes",
       "like_count",
       "announcements",
       "announcement_count",

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -168,7 +168,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     thread_muted? =
       case activity.thread_muted? do
         thread_muted? when is_boolean(thread_muted?) -> thread_muted?
-        nil -> CommonAPI.thread_muted?(user, activity)
+        nil -> (opts[:for] && CommonAPI.thread_muted?(opts[:for], activity)) || false
       end
 
     attachment_data = object.data["attachment"] || []

mix.exs

@@ -114,8 +114,9 @@ defmodule Pleroma.Mixfile do
       {:tesla, "~> 1.2"},
       {:jason, "~> 1.0"},
       {:mogrify, "~> 0.6.1"},
-      {:ex_aws, "~> 2.0"},
+      {:ex_aws, "~> 2.1"},
       {:ex_aws_s3, "~> 2.0"},
+      {:sweet_xml, "~> 0.6.6"},
       {:earmark, "~> 1.3"},
       {:bbcode, "~> 0.1.1"},
       {:ex_machina, "~> 2.3", only: :test},

mix.lock

@@ -80,6 +80,7 @@
   "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"},
   "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
+  "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"},
   "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"},
   "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},
   "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},

test/tasks/database_test.exs

@@ -3,8 +3,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.DatabaseTest do
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+
   use Pleroma.DataCase
 
   import Pleroma.Factory
@@ -46,4 +49,37 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do
       assert user.info.follower_count == 0
     end
   end
+
+  describe "running fix_likes_collections" do
+    test "it turns OrderedCollection likes into empty arrays" do
+      [user, user2] = insert_pair(:user)
+
+      {:ok, %{id: id, object: object}} = CommonAPI.post(user, %{"status" => "test"})
+      {:ok, %{object: object2}} = CommonAPI.post(user, %{"status" => "test test"})
+
+      CommonAPI.favorite(id, user2)
+
+      likes = %{
+        "first" =>
+          "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1",
+        "id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes",
+        "totalItems" => 3,
+        "type" => "OrderedCollection"
+      }
+
+      new_data = Map.put(object2.data, "likes", likes)
+
+      object2
+      |> Ecto.Changeset.change(%{data: new_data})
+      |> Repo.update()
+
+      assert length(Object.get_by_id(object.id).data["likes"]) == 1
+      assert is_map(Object.get_by_id(object2.id).data["likes"])
+
+      assert :ok == Mix.Tasks.Pleroma.Database.run(["fix_likes_collections"])
+
+      assert length(Object.get_by_id(object.id).data["likes"]) == 1
+      assert Enum.empty?(Object.get_by_id(object2.id).data["likes"])
+    end
+  end
 end

test/uploaders/local_test.exs

@@ -0,0 +1,32 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Uploaders.LocalTest do
+  use Pleroma.DataCase
+  alias Pleroma.Uploaders.Local
+
+  describe "get_file/1" do
+    test "it returns path to local folder for files" do
+      assert Local.get_file("") == {:ok, {:static_dir, "test/uploads"}}
+    end
+  end
+
+  describe "put_file/1" do
+    test "put file to local folder" do
+      file_path = "local_upload/files/image.jpg"
+
+      file = %Pleroma.Upload{
+        name: "image.jpg",
+        content_type: "image/jpg",
+        path: file_path,
+        tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+      }
+
+      assert Local.put_file(file) == :ok
+
+      assert Path.join([Local.upload_path(), file_path])
+             |> File.exists?()
+    end
+  end
+end

test/uploaders/mdii_test.exs

@@ -0,0 +1,50 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Uploaders.MDIITest do
+  use Pleroma.DataCase
+  alias Pleroma.Uploaders.MDII
+  import Tesla.Mock
+
+  describe "get_file/1" do
+    test "it returns path to local folder for files" do
+      assert MDII.get_file("") == {:ok, {:static_dir, "test/uploads"}}
+    end
+  end
+
+  describe "put_file/1" do
+    setup do
+      file_upload = %Pleroma.Upload{
+        name: "mdii-image.jpg",
+        content_type: "image/jpg",
+        path: "test_folder/mdii-image.jpg",
+        tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+      }
+
+      [file_upload: file_upload]
+    end
+
+    test "save file", %{file_upload: file_upload} do
+      mock(fn
+        %{method: :post, url: "https://mdii.sakura.ne.jp/mdii-post.cgi?jpg"} ->
+          %Tesla.Env{status: 200, body: "mdii-image"}
+      end)
+
+      assert MDII.put_file(file_upload) ==
+               {:ok, {:url, "https://mdii.sakura.ne.jp/mdii-image.jpg"}}
+    end
+
+    test "save file to local if MDII  isn`t available", %{file_upload: file_upload} do
+      mock(fn
+        %{method: :post, url: "https://mdii.sakura.ne.jp/mdii-post.cgi?jpg"} ->
+          %Tesla.Env{status: 500}
+      end)
+
+      assert MDII.put_file(file_upload) == :ok
+
+      assert Path.join([Pleroma.Uploaders.Local.upload_path(), file_upload.path])
+             |> File.exists?()
+    end
+  end
+end

test/uploaders/s3_test.exs

@@ -0,0 +1,90 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Uploaders.S3Test do
+  use Pleroma.DataCase
+
+  alias Pleroma.Config
+  alias Pleroma.Uploaders.S3
+
+  import Mock
+  import ExUnit.CaptureLog
+
+  setup do
+    config = Config.get([Pleroma.Uploaders.S3])
+
+    Config.put([Pleroma.Uploaders.S3],
+      bucket: "test_bucket",
+      public_endpoint: "https://s3.amazonaws.com"
+    )
+
+    on_exit(fn ->
+      Config.put([Pleroma.Uploaders.S3], config)
+    end)
+
+    :ok
+  end
+
+  describe "get_file/1" do
+    test "it returns path to local folder for files" do
+      assert S3.get_file("test_image.jpg") == {
+               :ok,
+               {:url, "https://s3.amazonaws.com/test_bucket/test_image.jpg"}
+             }
+    end
+
+    test "it returns path without bucket when truncated_namespace set to ''" do
+      Config.put([Pleroma.Uploaders.S3],
+        bucket: "test_bucket",
+        public_endpoint: "https://s3.amazonaws.com",
+        truncated_namespace: ""
+      )
+
+      assert S3.get_file("test_image.jpg") == {
+               :ok,
+               {:url, "https://s3.amazonaws.com/test_image.jpg"}
+             }
+    end
+
+    test "it returns path with bucket namespace when namespace is set" do
+      Config.put([Pleroma.Uploaders.S3],
+        bucket: "test_bucket",
+        public_endpoint: "https://s3.amazonaws.com",
+        bucket_namespace: "family"
+      )
+
+      assert S3.get_file("test_image.jpg") == {
+               :ok,
+               {:url, "https://s3.amazonaws.com/family:test_bucket/test_image.jpg"}
+             }
+    end
+  end
+
+  describe "put_file/1" do
+    setup do
+      file_upload = %Pleroma.Upload{
+        name: "image-tet.jpg",
+        content_type: "image/jpg",
+        path: "test_folder/image-tet.jpg",
+        tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+      }
+
+      [file_upload: file_upload]
+    end
+
+    test "save file", %{file_upload: file_upload} do
+      with_mock ExAws, request: fn _ -> {:ok, :ok} end do
+        assert S3.put_file(file_upload) == {:ok, {:file, "test_folder/image-tet.jpg"}}
+      end
+    end
+
+    test "returns error", %{file_upload: file_upload} do
+      with_mock ExAws, request: fn _ -> {:error, "S3 Upload failed"} end do
+        assert capture_log(fn ->
+                 assert S3.put_file(file_upload) == {:error, "S3 Upload failed"}
+               end) =~ "Elixir.Pleroma.Uploaders.S3: {:error, \"S3 Upload failed\"}"
+      end
+    end
+  end
+end

test/web/activity_pub/transmogrifier_test.exs

@@ -450,6 +450,27 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
       assert !is_nil(data["cc"])
     end
 
+    test "it strips internal likes" do
+      data =
+        File.read!("test/fixtures/mastodon-post-activity.json")
+        |> Poison.decode!()
+
+      likes = %{
+        "first" =>
+          "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1",
+        "id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes",
+        "totalItems" => 3,
+        "type" => "OrderedCollection"
+      }
+
+      object = Map.put(data["object"], "likes", likes)
+      data = Map.put(data, "object", object)
+
+      {:ok, %Activity{object: object}} = Transmogrifier.handle_incoming(data)
+
+      refute Map.has_key?(object.data, "likes")
+    end
+
     test "it works for incoming update activities" do
       data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!()
 
@@ -1061,14 +1082,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
       assert is_nil(modified["object"]["announcements"])
       assert is_nil(modified["object"]["announcement_count"])
       assert is_nil(modified["object"]["context_id"])
-    end
-
-    test "it adds like collection to object" do
-      activity = insert(:note_activity)
-      {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
-
-      assert modified["object"]["likes"]["type"] == "OrderedCollection"
-      assert modified["object"]["likes"]["totalItems"] == 0
+      assert is_nil(modified["object"]["likes"])
     end
 
     test "the directMessage flag is present" do

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -2901,8 +2901,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
 
   describe "conversation muting" do
     setup do
+      post_user = insert(:user)
       user = insert(:user)
-      {:ok, activity} = CommonAPI.post(user, %{"status" => "HIE"})
+
+      {:ok, activity} = CommonAPI.post(post_user, %{"status" => "HIE"})
 
       [user: user, activity: activity]
     end