From 7bcd7a959519ea023b075142aa36005f80503ba4 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 11 Jun 2020 20:23:10 +0200
Subject: [PATCH 01/57] QuestionValidator: Create

---
 lib/pleroma/web/activity_pub/activity_pub.ex  |  2 +-
 .../web/activity_pub/object_validator.ex      | 41 +++++++-
 .../object_validators/common_validations.ex   |  4 +-
 .../create_question_validator.ex              | 94 +++++++++++++++++++
 .../object_validators/note_validator.ex       |  2 +-
 .../question_options_validator.ex             | 47 ++++++++++
 .../object_validators/question_validator.ex   | 89 ++++++++++++++++++
 lib/pleroma/web/activity_pub/side_effects.ex  | 10 +-
 .../web/activity_pub/transmogrifier.ex        | 17 +++-
 test/web/activity_pub/transmogrifier_test.exs |  2 +-
 10 files changed, 297 insertions(+), 11 deletions(-)
 create mode 100644 lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex
 create mode 100644 lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex
 create mode 100644 lib/pleroma/web/activity_pub/object_validators/question_validator.ex

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index bc7b5d95a..462aa57a6 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -95,7 +95,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp increase_poll_votes_if_vote(_create_data), do: :noop
 
-  @object_types ["ChatMessage"]
+  @object_types ["ChatMessage", "Question"]
   @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index df926829c..5cc66d7bd 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -16,10 +16,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CreateQuestionValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator
 
@@ -105,17 +107,30 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     end
   end
 
+  def validate(%{"type" => "Question"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> QuestionValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
   def validate(%{"type" => "EmojiReact"} = object, meta) do
     with {:ok, object} <-
            object
            |> EmojiReactValidator.cast_and_validate()
            |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object |> Map.from_struct())
+      object = stringify_keys(object)
       {:ok, object, meta}
     end
   end
 
-  def validate(%{"type" => "Create", "object" => object} = create_activity, meta) do
+  def validate(
+        %{"type" => "Create", "object" => %{"type" => "ChatMessage"} = object} = create_activity,
+        meta
+      ) do
     with {:ok, object_data} <- cast_and_apply(object),
          meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
          {:ok, create_activity} <-
@@ -127,12 +142,27 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     end
   end
 
+  def validate(
+        %{"type" => "Create", "object" => %{"type" => "Question"} = object} = create_activity,
+        meta
+      ) do
+    with {:ok, object_data} <- cast_and_apply(object),
+         meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
+         {:ok, create_activity} <-
+           create_activity
+           |> CreateQuestionValidator.cast_and_validate(meta)
+           |> Ecto.Changeset.apply_action(:insert) do
+      create_activity = stringify_keys(create_activity)
+      {:ok, create_activity, meta}
+    end
+  end
+
   def validate(%{"type" => "Announce"} = object, meta) do
     with {:ok, object} <-
            object
            |> AnnounceValidator.cast_and_validate()
            |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object |> Map.from_struct())
+      object = stringify_keys(object)
       {:ok, object, meta}
     end
   end
@@ -141,8 +171,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     ChatMessageValidator.cast_and_apply(object)
   end
 
+  def cast_and_apply(%{"type" => "Question"} = object) do
+    QuestionValidator.cast_and_apply(object)
+  end
+
   def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
 
+  # is_struct/1 isn't present in Elixir 1.8.x
   def stringify_keys(%{__struct__: _} = object) do
     object
     |> Map.from_struct()
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
index aeef31945..e746b9360 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
   alias Pleroma.Object
   alias Pleroma.User
 
-  def validate_recipients_presence(cng, fields \\ [:to, :cc]) do
+  def validate_any_presence(cng, fields) do
     non_empty =
       fields
       |> Enum.map(fn field -> get_field(cng, field) end)
@@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
       fields
       |> Enum.reduce(cng, fn field, cng ->
         cng
-        |> add_error(field, "no recipients in any field")
+        |> add_error(field, "none of #{inspect(fields)} present")
       end)
     end
   end
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex
new file mode 100644
index 000000000..f09207418
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex
@@ -0,0 +1,94 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# Code based on CreateChatMessageValidator
+# NOTES
+# - Can probably be a generic create validator
+# - doesn't embed, will only get the object id
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateQuestionValidator do
+  use Ecto.Schema
+
+  alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+
+  import Ecto.Changeset
+  import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+
+  @primary_key false
+
+  embedded_schema do
+    field(:id, Types.ObjectID, primary_key: true)
+    field(:actor, Types.ObjectID)
+    field(:type, :string)
+    field(:to, Types.Recipients, default: [])
+    field(:cc, Types.Recipients, default: [])
+    field(:object, Types.ObjectID)
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_data(data) do
+    cast(%__MODULE__{}, data, __schema__(:fields))
+  end
+
+  def cast_and_validate(data, meta \\ []) do
+    cast_data(data)
+    |> validate_data(meta)
+  end
+
+  def validate_data(cng, meta \\ []) do
+    cng
+    |> validate_required([:actor, :type, :object])
+    |> validate_inclusion(:type, ["Create"])
+    |> validate_actor_presence()
+    |> validate_any_presence([:to, :cc])
+    |> validate_recipients_match(meta)
+    |> validate_actors_match(meta)
+    |> validate_object_nonexistence()
+  end
+
+  def validate_object_nonexistence(cng) do
+    cng
+    |> validate_change(:object, fn :object, object_id ->
+      if Object.get_cached_by_ap_id(object_id) do
+        [{:object, "The object to create already exists"}]
+      else
+        []
+      end
+    end)
+  end
+
+  def validate_actors_match(cng, meta) do
+    object_actor = meta[:object_data]["actor"]
+
+    cng
+    |> validate_change(:actor, fn :actor, actor ->
+      if actor == object_actor do
+        []
+      else
+        [{:actor, "Actor doesn't match with object actor"}]
+      end
+    end)
+  end
+
+  def validate_recipients_match(cng, meta) do
+    object_recipients = meta[:object_data]["to"] || []
+
+    cng
+    |> validate_change(:to, fn :to, recipients ->
+      activity_set = MapSet.new(recipients)
+      object_set = MapSet.new(object_recipients)
+
+      if MapSet.equal?(activity_set, object_set) do
+        []
+      else
+        [{:to, "Recipients don't match with object recipients"}]
+      end
+    end)
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
index 56b93dde8..a65fe2354 100644
--- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
@@ -34,7 +34,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
     field(:replies_count, :integer, default: 0)
     field(:like_count, :integer, default: 0)
     field(:announcement_count, :integer, default: 0)
-    field(:inRepyTo, :string)
+    field(:inReplyTo, :string)
     field(:uri, ObjectValidators.Uri)
 
     field(:likes, {:array, :string}, default: [])
diff --git a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex
new file mode 100644
index 000000000..9bc7e0cc0
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex
@@ -0,0 +1,47 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator do
+  use Ecto.Schema
+
+  alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsRepliesValidator
+
+  import Ecto.Changeset
+
+  @primary_key false
+
+  embedded_schema do
+    field(:name, :string)
+    embeds_one(:replies, QuestionOptionsRepliesValidator)
+    field(:type, :string)
+  end
+
+  def changeset(struct, data) do
+    struct
+    |> cast(data, [:name, :type])
+    |> cast_embed(:replies)
+    |> validate_inclusion(:type, ["Note"])
+    |> validate_required([:name, :type])
+  end
+end
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsRepliesValidator do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+
+  @primary_key false
+
+  embedded_schema do
+    field(:totalItems, :integer)
+    field(:type, :string)
+  end
+
+  def changeset(struct, data) do
+    struct
+    |> cast(data, __schema__(:fields))
+    |> validate_inclusion(:type, ["Collection"])
+    |> validate_required([:type])
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
new file mode 100644
index 000000000..f94d79352
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -0,0 +1,89 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
+  use Ecto.Schema
+
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+
+  import Ecto.Changeset
+
+  @primary_key false
+  @derive Jason.Encoder
+
+  # Extends from NoteValidator
+  embedded_schema do
+    field(:id, Types.ObjectID, primary_key: true)
+    field(:to, {:array, :string}, default: [])
+    field(:cc, {:array, :string}, default: [])
+    field(:bto, {:array, :string}, default: [])
+    field(:bcc, {:array, :string}, default: [])
+    # TODO: Write type
+    field(:tag, {:array, :map}, default: [])
+    field(:type, :string)
+    field(:content, :string)
+    field(:context, :string)
+    field(:actor, Types.ObjectID)
+    field(:attributedTo, Types.ObjectID)
+    field(:summary, :string)
+    field(:published, Types.DateTime)
+    # TODO: Write type
+    field(:emoji, :map, default: %{})
+    field(:sensitive, :boolean, default: false)
+    # TODO: Write type
+    field(:attachment, {:array, :map}, default: [])
+    field(:replies_count, :integer, default: 0)
+    field(:like_count, :integer, default: 0)
+    field(:announcement_count, :integer, default: 0)
+    field(:inReplyTo, :string)
+    field(:uri, Types.Uri)
+
+    field(:likes, {:array, :string}, default: [])
+    field(:announcements, {:array, :string}, default: [])
+
+    # see if needed
+    field(:conversation, :string)
+    field(:context_id, :string)
+
+    field(:closed, Types.DateTime)
+    field(:voters, {:array, Types.ObjectID}, default: [])
+    embeds_many(:anyOf, QuestionOptionsValidator)
+    embeds_many(:oneOf, QuestionOptionsValidator)
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  def changeset(struct, data) do
+    struct
+    |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf])
+    |> cast_embed(:anyOf)
+    |> cast_embed(:oneOf)
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["Question"])
+    |> validate_required([:id, :actor, :type, :content, :context])
+    |> CommonValidations.validate_any_presence([:cc, :to])
+    |> CommonValidations.validate_actor_presence()
+    |> CommonValidations.validate_any_presence([:oneOf, :anyOf])
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 1d2c296a5..a78ec411f 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -268,9 +268,15 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
     end
   end
 
+  def handle_object_creation(%{"type" => "Question"} = object, meta) do
+    with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
+      {:ok, object, meta}
+    end
+  end
+
   # Nothing to do
-  def handle_object_creation(object) do
-    {:ok, object}
+  def handle_object_creation(object, meta) do
+    {:ok, object, meta}
   end
 
   defp undo_like(nil, object), do: delete_object(object)
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index f37bcab3e..da5dc23bc 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -457,7 +457,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
         options
       )
-      when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer", "Audio"] do
+      when objtype in ["Article", "Event", "Note", "Video", "Page", "Answer", "Audio"] do
     actor = Containment.get_actor(data)
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
@@ -613,6 +613,21 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> handle_incoming(options)
   end
 
+  def handle_incoming(
+        %{"type" => "Create", "object" => %{"type" => "Question"} = object} = data,
+        _options
+      ) do
+    data =
+      data
+      |> Map.put("object", fix_object(object))
+      |> fix_addressing()
+
+    with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
+         {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
+      {:ok, activity}
+    end
+  end
+
   def handle_incoming(
         %{"type" => "Create", "object" => %{"type" => "ChatMessage"}} = data,
         _options
diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs
index 248b410c6..73949b558 100644
--- a/test/web/activity_pub/transmogrifier_test.exs
+++ b/test/web/activity_pub/transmogrifier_test.exs
@@ -222,7 +222,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
 
       {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, false)
 
       assert Enum.all?(object.data["oneOf"], fn choice ->
                choice["name"] in [

From c5efaf6b00bf582a6eef7b5728fe5042f5fcc702 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 11 Jun 2020 20:43:01 +0200
Subject: [PATCH 02/57] AnswerValidator: Create

---
 lib/pleroma/web/activity_pub/activity_pub.ex  |  2 +-
 .../web/activity_pub/object_validator.ex      | 20 ++++++-
 .../object_validators/answer_validator.ex     | 58 +++++++++++++++++++
 .../web/activity_pub/transmogrifier.ex        |  7 ++-
 4 files changed, 81 insertions(+), 6 deletions(-)
 create mode 100644 lib/pleroma/web/activity_pub/object_validators/answer_validator.ex

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 462aa57a6..d8cc8d24f 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -95,7 +95,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp increase_poll_votes_if_vote(_create_data), do: :noop
 
-  @object_types ["ChatMessage", "Question"]
+  @object_types ["ChatMessage", "Question", "Answer"]
   @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index 5cc66d7bd..c89311187 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -13,6 +13,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
@@ -117,6 +118,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     end
   end
 
+  def validate(%{"type" => "Answer"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> AnswerValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
   def validate(%{"type" => "EmojiReact"} = object, meta) do
     with {:ok, object} <-
            object
@@ -143,9 +154,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   end
 
   def validate(
-        %{"type" => "Create", "object" => %{"type" => "Question"} = object} = create_activity,
+        %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
         meta
-      ) do
+      )
+      when objtype in ["Question", "Answer"] do
     with {:ok, object_data} <- cast_and_apply(object),
          meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
          {:ok, create_activity} <-
@@ -175,6 +187,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     QuestionValidator.cast_and_apply(object)
   end
 
+  def cast_and_apply(%{"type" => "Answer"} = object) do
+    QuestionValidator.cast_and_apply(object)
+  end
+
   def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
 
   # is_struct/1 isn't present in Elixir 1.8.x
diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
new file mode 100644
index 000000000..0b51eccfa
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
@@ -0,0 +1,58 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
+  use Ecto.Schema
+
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+
+  import Ecto.Changeset
+
+  @primary_key false
+  @derive Jason.Encoder
+
+  # Extends from NoteValidator
+  embedded_schema do
+    field(:id, Types.ObjectID, primary_key: true)
+    field(:to, {:array, :string}, default: [])
+    field(:cc, {:array, :string}, default: [])
+    field(:bto, {:array, :string}, default: [])
+    field(:bcc, {:array, :string}, default: [])
+    field(:type, :string)
+    field(:name, :string)
+    field(:inReplyTo, :string)
+    field(:attributedTo, Types.ObjectID)
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  def changeset(struct, data) do
+    struct
+    |> cast(data, __schema__(:fields))
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["Answer"])
+    |> validate_required([:id, :inReplyTo, :name])
+    |> CommonValidations.validate_any_presence([:cc, :to])
+    |> CommonValidations.validate_actor_presence()
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index da5dc23bc..9900602e4 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -457,7 +457,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
         options
       )
-      when objtype in ["Article", "Event", "Note", "Video", "Page", "Answer", "Audio"] do
+      when objtype in ["Article", "Event", "Note", "Video", "Page", "Audio"] do
     actor = Containment.get_actor(data)
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
@@ -614,9 +614,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def handle_incoming(
-        %{"type" => "Create", "object" => %{"type" => "Question"} = object} = data,
+        %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
         _options
-      ) do
+      )
+      when objtype in ["Question", "Answer"] do
     data =
       data
       |> Map.put("object", fix_object(object))

From 89a2433154b05c378f9c5b3224375049f3dbc8af Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 12 Jun 2020 21:22:05 +0200
Subject: [PATCH 03/57] QuestionOptionsValidator: inline schema for replies

---
 .../question_options_validator.ex             | 28 ++++++-------------
 1 file changed, 9 insertions(+), 19 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex
index 9bc7e0cc0..8291d7b9f 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex
@@ -5,42 +5,32 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator do
   use Ecto.Schema
 
-  alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsRepliesValidator
-
   import Ecto.Changeset
 
   @primary_key false
 
   embedded_schema do
     field(:name, :string)
-    embeds_one(:replies, QuestionOptionsRepliesValidator)
+
+    embeds_one :replies, Replies do
+      field(:totalItems, :integer)
+      field(:type, :string)
+    end
+
     field(:type, :string)
   end
 
   def changeset(struct, data) do
     struct
     |> cast(data, [:name, :type])
-    |> cast_embed(:replies)
+    |> cast_embed(:replies, with: &replies_changeset/2)
     |> validate_inclusion(:type, ["Note"])
     |> validate_required([:name, :type])
   end
-end
 
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsRepliesValidator do
-  use Ecto.Schema
-
-  import Ecto.Changeset
-
-  @primary_key false
-
-  embedded_schema do
-    field(:totalItems, :integer)
-    field(:type, :string)
-  end
-
-  def changeset(struct, data) do
+  def replies_changeset(struct, data) do
     struct
-    |> cast(data, __schema__(:fields))
+    |> cast(data, [:totalItems, :type])
     |> validate_inclusion(:type, ["Collection"])
     |> validate_required([:type])
   end

From 10bd08ef07bd91995589ad37cb25e6889dac59b3 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sun, 14 Jun 2020 22:25:04 +0200
Subject: [PATCH 04/57] transmogrifier_test: test date, anyOf and oneOf
 completely

---
 .../question_options_validator.ex             |  2 +-
 test/web/activity_pub/transmogrifier_test.exs | 35 ++++++++++++++-----
 2 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex
index 8291d7b9f..478b3b5cf 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex
@@ -12,7 +12,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator do
   embedded_schema do
     field(:name, :string)
 
-    embeds_one :replies, Replies do
+    embeds_one :replies, Replies, primary_key: false do
       field(:totalItems, :integer)
       field(:type, :string)
     end
diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs
index 73949b558..4184b93ce 100644
--- a/test/web/activity_pub/transmogrifier_test.exs
+++ b/test/web/activity_pub/transmogrifier_test.exs
@@ -224,14 +224,33 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
 
       object = Object.normalize(activity, false)
 
-      assert Enum.all?(object.data["oneOf"], fn choice ->
-               choice["name"] in [
-                 "Dunno",
-                 "Everyone knows that!",
-                 "25 char limit is dumb",
-                 "I can't even fit a funny"
-               ]
-             end)
+      assert object.data["closed"] == "2019-05-11T09:03:36Z"
+
+      assert object.data["anyOf"] == []
+
+      assert Enum.sort(object.data["oneOf"]) ==
+               Enum.sort([
+                 %{
+                   "name" => "25 char limit is dumb",
+                   "replies" => %{"totalItems" => 0, "type" => "Collection"},
+                   "type" => "Note"
+                 },
+                 %{
+                   "name" => "Dunno",
+                   "replies" => %{"totalItems" => 0, "type" => "Collection"},
+                   "type" => "Note"
+                 },
+                 %{
+                   "name" => "Everyone knows that!",
+                   "replies" => %{"totalItems" => 1, "type" => "Collection"},
+                   "type" => "Note"
+                 },
+                 %{
+                   "name" => "I can't even fit a funny",
+                   "replies" => %{"totalItems" => 1, "type" => "Collection"},
+                   "type" => "Note"
+                 }
+               ])
     end
 
     test "it works for incoming listens" do

From 4644a8bd109faf6c684767fc60c37e298b8c5c07 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 15 Jun 2020 00:30:45 +0200
Subject: [PATCH 05/57] Fix multiple-choice poll detection

---
 lib/pleroma/object.ex                           | 17 ++++++++++-------
 lib/pleroma/web/common_api/common_api.ex        |  9 +++++++--
 lib/pleroma/web/mastodon_api/views/poll_view.ex |  4 ++--
 3 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index 546c4ea01..4dd929cfd 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -255,6 +255,12 @@ defmodule Pleroma.Object do
     end
   end
 
+  defp poll_is_multiple?(%Object{data: %{"anyOf" => anyOf}}) do
+    !Enum.empty?(anyOf)
+  end
+
+  defp poll_is_multiple?(_), do: false
+
   def decrease_replies_count(ap_id) do
     Object
     |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id)))
@@ -281,10 +287,10 @@ defmodule Pleroma.Object do
   def increase_vote_count(ap_id, name, actor) do
     with %Object{} = object <- Object.normalize(ap_id),
          "Question" <- object.data["type"] do
-      multiple = Map.has_key?(object.data, "anyOf")
+      key = if poll_is_multiple?(object), do: "anyOf", else: "oneOf"
 
       options =
-        (object.data["anyOf"] || object.data["oneOf"] || [])
+        object.data[key]
         |> Enum.map(fn
           %{"name" => ^name} = option ->
             Kernel.update_in(option["replies"]["totalItems"], &(&1 + 1))
@@ -296,11 +302,8 @@ defmodule Pleroma.Object do
       voters = [actor | object.data["voters"] || []] |> Enum.uniq()
 
       data =
-        if multiple do
-          Map.put(object.data, "anyOf", options)
-        else
-          Map.put(object.data, "oneOf", options)
-        end
+        object.data
+        |> Map.put(key, options)
         |> Map.put("voters", voters)
 
       object
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 4d5b0decf..692ceab1e 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -340,8 +340,13 @@ defmodule Pleroma.Web.CommonAPI do
     end
   end
 
-  defp get_options_and_max_count(%{data: %{"anyOf" => any_of}}), do: {any_of, Enum.count(any_of)}
-  defp get_options_and_max_count(%{data: %{"oneOf" => one_of}}), do: {one_of, 1}
+  defp get_options_and_max_count(%{data: %{"anyOf" => any_of}})
+       when is_list(any_of) and any_of != [],
+       do: {any_of, Enum.count(any_of)}
+
+  defp get_options_and_max_count(%{data: %{"oneOf" => one_of}})
+       when is_list(one_of) and one_of != [],
+       do: {one_of, 1}
 
   defp normalize_and_validate_choices(choices, object) do
     choices = Enum.map(choices, fn i -> if is_binary(i), do: String.to_integer(i), else: i end)
diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex
index 59a5deb28..73c990e2e 100644
--- a/lib/pleroma/web/mastodon_api/views/poll_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex
@@ -28,10 +28,10 @@ defmodule Pleroma.Web.MastodonAPI.PollView do
 
   def render("show.json", %{object: object} = params) do
     case object.data do
-      %{"anyOf" => options} when is_list(options) ->
+      %{"anyOf" => options} when is_list(options) and options != [] ->
         render(__MODULE__, "show.json", Map.merge(params, %{multiple: true, options: options}))
 
-      %{"oneOf" => options} when is_list(options) ->
+      %{"oneOf" => options} when is_list(options) and options != [] ->
         render(__MODULE__, "show.json", Map.merge(params, %{multiple: false, options: options}))
 
       _ ->

From 6b9c4bc1f1e5013cdfc084603cdf6cfa83ac2778 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sun, 14 Jun 2020 22:24:00 +0200
Subject: [PATCH 06/57] fetcher: more descriptive variable names

---
 lib/pleroma/object/fetcher.ex | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 3e2949ee2..e1ab4ef8b 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -23,21 +23,21 @@ defmodule Pleroma.Object.Fetcher do
     Ecto.Changeset.put_change(changeset, :updated_at, updated_at)
   end
 
-  defp maybe_reinject_internal_fields(data, %{data: %{} = old_data}) do
+  defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do
     internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields())
 
-    Map.merge(data, internal_fields)
+    Map.merge(new_data, internal_fields)
   end
 
-  defp maybe_reinject_internal_fields(data, _), do: data
+  defp maybe_reinject_internal_fields(_, new_data), do: new_data
 
   @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()}
-  defp reinject_object(struct, data) do
-    Logger.debug("Reinjecting object #{data["id"]}")
+  defp reinject_object(%Object{} = object, new_data) do
+    Logger.debug("Reinjecting object #{new_data["id"]}")
 
-    with data <- Transmogrifier.fix_object(data),
-         data <- maybe_reinject_internal_fields(data, struct),
-         changeset <- Object.change(struct, %{data: data}),
+    with new_data <- Transmogrifier.fix_object(new_data),
+         data <- maybe_reinject_internal_fields(object, new_data),
+         changeset <- Object.change(object, %{data: data}),
          changeset <- touch_changeset(changeset),
          {:ok, object} <- Repo.insert_or_update(changeset),
          {:ok, object} <- Object.set_cache(object) do
@@ -51,8 +51,8 @@ defmodule Pleroma.Object.Fetcher do
 
   def refetch_object(%Object{data: %{"id" => id}} = object) do
     with {:local, false} <- {:local, Object.local?(object)},
-         {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
-         {:ok, object} <- reinject_object(object, data) do
+         {:ok, new_data} <- fetch_and_contain_remote_object_from_id(id),
+         {:ok, object} <- reinject_object(object, new_data) do
       {:ok, object}
     else
       {:local, true} -> {:ok, object}

From ad867ccfa18afe2a6104cad4d3035834c5a264c8 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sun, 14 Jun 2020 22:01:14 +0200
Subject: [PATCH 07/57] fetcher: Reinject Question through validator

---
 lib/pleroma/object/fetcher.ex | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index e1ab4ef8b..3956bb727 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.Repo
   alias Pleroma.Signature
   alias Pleroma.Web.ActivityPub.InternalFetchActor
+  alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
 
@@ -32,6 +33,24 @@ defmodule Pleroma.Object.Fetcher do
   defp maybe_reinject_internal_fields(_, new_data), do: new_data
 
   @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()}
+  defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do
+    Logger.debug("Reinjecting object #{new_data["id"]}")
+
+    with new_data <- Transmogrifier.fix_object(new_data),
+         data <- maybe_reinject_internal_fields(object, new_data),
+         {:ok, data, _} <- ObjectValidator.validate(data, %{}),
+         changeset <- Object.change(object, %{data: data}),
+         changeset <- touch_changeset(changeset),
+         {:ok, object} <- Repo.insert_or_update(changeset),
+         {:ok, object} <- Object.set_cache(object) do
+      {:ok, object}
+    else
+      e ->
+        Logger.error("Error while processing object: #{inspect(e)}")
+        {:error, e}
+    end
+  end
+
   defp reinject_object(%Object{} = object, new_data) do
     Logger.debug("Reinjecting object #{new_data["id"]}")
 

From 47ba796f415740c443cd8477c121280656b13032 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 15 Jun 2020 02:20:18 +0200
Subject: [PATCH 08/57] create_question_validator: remove
 validate_recipients_match

---
 .../create_question_validator.ex                | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex
index f09207418..6d3f71566 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex
@@ -47,7 +47,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateQuestionValidator do
     |> validate_inclusion(:type, ["Create"])
     |> validate_actor_presence()
     |> validate_any_presence([:to, :cc])
-    |> validate_recipients_match(meta)
     |> validate_actors_match(meta)
     |> validate_object_nonexistence()
   end
@@ -75,20 +74,4 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateQuestionValidator do
       end
     end)
   end
-
-  def validate_recipients_match(cng, meta) do
-    object_recipients = meta[:object_data]["to"] || []
-
-    cng
-    |> validate_change(:to, fn :to, recipients ->
-      activity_set = MapSet.new(recipients)
-      object_set = MapSet.new(object_recipients)
-
-      if MapSet.equal?(activity_set, object_set) do
-        []
-      else
-        [{:to, "Recipients don't match with object recipients"}]
-      end
-    end)
-  end
 end

From 173f69c854adf966d52b3767c4de43a0b1ce5b00 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 15 Jun 2020 05:18:30 +0200
Subject: [PATCH 09/57] question_validator: fix for mastodon poll expiration

Mastodon activities do not have a "closed" field, this could be seen on
https://pouet.it/users/lanodan_tmp/statuses/104345126997708380
which runs Mastodon 3.1.4 (SDF runs 3.1.2)
---
 .../object_validators/question_validator.ex        | 10 ++++++++++
 lib/pleroma/web/mastodon_api/views/poll_view.ex    | 14 ++++++--------
 test/fixtures/mastodon-question-activity.json      |  1 -
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
index f94d79352..605cb56f8 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -71,7 +71,17 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     |> changeset(data)
   end
 
+  def fix(data) do
+    cond do
+      is_binary(data["closed"]) -> data
+      is_binary(data["endTime"]) -> Map.put(data, "closed", data["endTime"])
+      true -> Map.drop(data, ["closed"])
+    end
+  end
+
   def changeset(struct, data) do
+    data = fix(data)
+
     struct
     |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf])
     |> cast_embed(:anyOf)
diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex
index 73c990e2e..ce595ae8a 100644
--- a/lib/pleroma/web/mastodon_api/views/poll_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex
@@ -40,15 +40,13 @@ defmodule Pleroma.Web.MastodonAPI.PollView do
   end
 
   defp end_time_and_expired(object) do
-    case object.data["closed"] || object.data["endTime"] do
-      end_time when is_binary(end_time) ->
-        end_time = NaiveDateTime.from_iso8601!(end_time)
-        expired = NaiveDateTime.compare(end_time, NaiveDateTime.utc_now()) == :lt
+    if object.data["closed"] do
+      end_time = NaiveDateTime.from_iso8601!(object.data["closed"])
+      expired = NaiveDateTime.compare(end_time, NaiveDateTime.utc_now()) == :lt
 
-        {Utils.to_masto_date(end_time), expired}
-
-      _ ->
-        {nil, false}
+      {Utils.to_masto_date(end_time), expired}
+    else
+      {nil, false}
     end
   end
 
diff --git a/test/fixtures/mastodon-question-activity.json b/test/fixtures/mastodon-question-activity.json
index ac329c7d5..3648b9f90 100644
--- a/test/fixtures/mastodon-question-activity.json
+++ b/test/fixtures/mastodon-question-activity.json
@@ -49,7 +49,6 @@
       "en": "<p>Why is Tenshi eating a corndog so cute?</p>"
     },
     "endTime": "2019-05-11T09:03:36Z",
-    "closed": "2019-05-11T09:03:36Z",
     "attachment": [],
     "tag": [],
     "replies": {

From 4f70fd4105e90c8fc06a1eb6bd70084874bae3a5 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 18 Jun 2020 19:54:56 +0200
Subject: [PATCH 10/57] question_validator: remove conversation field

---
 .../web/activity_pub/object_validators/question_validator.ex     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
index 605cb56f8..211f520c4 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -45,7 +45,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     field(:announcements, {:array, :string}, default: [])
 
     # see if needed
-    field(:conversation, :string)
     field(:context_id, :string)
 
     field(:closed, Types.DateTime)

From 82895a40123ca24258a95b9ac7e117dd8b1e698e Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 18 Jun 2020 04:05:42 +0200
Subject: [PATCH 11/57] SideEffects: port ones from ActivityPub.do_create and
 ActivityPub.insert

---
 lib/pleroma/object/containment.ex             |  2 +-
 lib/pleroma/web/activity_pub/activity_pub.ex  | 13 +----
 lib/pleroma/web/activity_pub/builder.ex       | 15 ++++++
 .../web/activity_pub/object_validator.ex      |  6 +--
 .../object_validators/answer_validator.ex     |  7 ++-
 .../object_validators/common_validations.ex   | 38 ++++++++++++++
 ...lidator.ex => create_generic_validator.ex} | 39 ++++++++++++---
 .../object_validators/question_validator.ex   | 28 +++++++++--
 lib/pleroma/web/activity_pub/side_effects.ex  | 30 +++++++++++-
 .../web/activity_pub/transmogrifier.ex        | 49 ++++++++++++-------
 lib/pleroma/web/common_api/common_api.ex      | 21 ++++----
 lib/pleroma/web/common_api/utils.ex           | 11 -----
 test/web/activity_pub/transmogrifier_test.exs |  2 +-
 13 files changed, 188 insertions(+), 73 deletions(-)
 rename lib/pleroma/web/activity_pub/object_validators/{create_question_validator.ex => create_generic_validator.ex} (70%)

diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex
index 99608b8a5..bc88e8a0c 100644
--- a/lib/pleroma/object/containment.ex
+++ b/lib/pleroma/object/containment.ex
@@ -55,7 +55,7 @@ defmodule Pleroma.Object.Containment do
   defp compare_uris(_id_uri, _other_uri), do: :error
 
   @doc """
-  Checks that an imported AP object's actor matches the domain it came from.
+  Checks that an imported AP object's actor matches the host it came from.
   """
   def contain_origin(_id, %{"actor" => nil}), do: :error
 
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index d8cc8d24f..9d13a06c4 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -66,7 +66,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp check_remote_limit(_), do: true
 
-  defp increase_note_count_if_public(actor, object) do
+  def increase_note_count_if_public(actor, object) do
     if is_public?(object), do: User.increase_note_count(actor), else: {:ok, actor}
   end
 
@@ -85,16 +85,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
-  defp increase_poll_votes_if_vote(%{
-         "object" => %{"inReplyTo" => reply_ap_id, "name" => name},
-         "type" => "Create",
-         "actor" => actor
-       }) do
-    Object.increase_vote_count(reply_ap_id, name, actor)
-  end
-
-  defp increase_poll_votes_if_vote(_create_data), do: :noop
-
   @object_types ["ChatMessage", "Question", "Answer"]
   @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
   def persist(%{"type" => type} = object, meta) when type in @object_types do
@@ -258,7 +248,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     with {:ok, activity} <- insert(create_data, local, fake),
          {:fake, false, activity} <- {:fake, fake, activity},
          _ <- increase_replies_count_if_reply(create_data),
-         _ <- increase_poll_votes_if_vote(create_data),
          {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity},
          {:ok, _actor} <- increase_note_count_if_public(actor, activity),
          _ <- notify_and_stream(activity),
diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex
index d5f3610ed..e97381954 100644
--- a/lib/pleroma/web/activity_pub/builder.ex
+++ b/lib/pleroma/web/activity_pub/builder.ex
@@ -115,6 +115,21 @@ defmodule Pleroma.Web.ActivityPub.Builder do
     end
   end
 
+  def answer(user, object, name) do
+    {:ok,
+     %{
+       "type" => "Answer",
+       "actor" => user.ap_id,
+       "cc" => [object.data["actor"]],
+       "to" => [],
+       "name" => name,
+       "inReplyTo" => object.data["id"],
+       "context" => object.data["context"],
+       "published" => DateTime.utc_now() |> DateTime.to_iso8601(),
+       "id" => Utils.generate_object_id()
+     }, []}
+  end
+
   @spec tombstone(String.t(), String.t()) :: {:ok, map(), keyword()}
   def tombstone(actor, id) do
     {:ok,
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index c89311187..a24aaf00c 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -17,7 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.CreateQuestionValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator
@@ -162,7 +162,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
          meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
          {:ok, create_activity} <-
            create_activity
-           |> CreateQuestionValidator.cast_and_validate(meta)
+           |> CreateGenericValidator.cast_and_validate(meta)
            |> Ecto.Changeset.apply_action(:insert) do
       create_activity = stringify_keys(create_activity)
       {:ok, create_activity, meta}
@@ -188,7 +188,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   end
 
   def cast_and_apply(%{"type" => "Answer"} = object) do
-    QuestionValidator.cast_and_apply(object)
+    AnswerValidator.cast_and_apply(object)
   end
 
   def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
index 0b51eccfa..8d4c92520 100644
--- a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
@@ -13,22 +13,25 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
   @primary_key false
   @derive Jason.Encoder
 
-  # Extends from NoteValidator
   embedded_schema do
     field(:id, Types.ObjectID, primary_key: true)
     field(:to, {:array, :string}, default: [])
     field(:cc, {:array, :string}, default: [])
+
+    # is this actually needed?
     field(:bto, {:array, :string}, default: [])
     field(:bcc, {:array, :string}, default: [])
+
     field(:type, :string)
     field(:name, :string)
     field(:inReplyTo, :string)
     field(:attributedTo, Types.ObjectID)
+    field(:actor, Types.ObjectID)
   end
 
   def cast_and_apply(data) do
     data
-    |> cast_data
+    |> cast_data()
     |> apply_action(:insert)
   end
 
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
index e746b9360..140555a45 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
@@ -42,6 +42,19 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
     end)
   end
 
+  def validate_actor_is_active(cng, options \\ []) do
+    field_name = Keyword.get(options, :field_name, :actor)
+
+    cng
+    |> validate_change(field_name, fn field_name, actor ->
+      if %User{deactivated: false} = User.get_cached_by_ap_id(actor) do
+        []
+      else
+        [{field_name, "can't find user (or deactivated)"}]
+      end
+    end)
+  end
+
   def validate_object_presence(cng, options \\ []) do
     field_name = Keyword.get(options, :field_name, :object)
     allowed_types = Keyword.get(options, :allowed_types, false)
@@ -77,4 +90,29 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
 
     if actor_cng.valid?, do: actor_cng, else: object_cng
   end
+
+  def validate_host_match(cng, fields \\ [:id, :actor]) do
+    unique_hosts =
+      fields
+      |> Enum.map(fn field ->
+        %URI{host: host} =
+          cng
+          |> get_field(field)
+          |> URI.parse()
+
+        host
+      end)
+      |> Enum.uniq()
+      |> Enum.count()
+
+    if unique_hosts == 1 do
+      cng
+    else
+      fields
+      |> Enum.reduce(cng, fn field, cng ->
+        cng
+        |> add_error(field, "hosts of #{inspect(fields)} aren't matching")
+      end)
+    end
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
similarity index 70%
rename from lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex
rename to lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
index 6d3f71566..4ad4ca0de 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
@@ -4,9 +4,8 @@
 
 # Code based on CreateChatMessageValidator
 # NOTES
-# - Can probably be a generic create validator
 # - doesn't embed, will only get the object id
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateQuestionValidator do
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
   use Ecto.Schema
 
   alias Pleroma.Object
@@ -26,29 +25,53 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateQuestionValidator do
     field(:object, Types.ObjectID)
   end
 
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
   def cast_and_apply(data) do
     data
     |> cast_data
     |> apply_action(:insert)
   end
 
-  def cast_data(data) do
-    cast(%__MODULE__{}, data, __schema__(:fields))
+  def cast_and_validate(data, meta \\ []) do
+    data
+    |> cast_data
+    |> validate_data(meta)
   end
 
-  def cast_and_validate(data, meta \\ []) do
-    cast_data(data)
-    |> validate_data(meta)
+  def changeset(struct, data) do
+    struct
+    |> cast(data, __schema__(:fields))
   end
 
   def validate_data(cng, meta \\ []) do
     cng
     |> validate_required([:actor, :type, :object])
     |> validate_inclusion(:type, ["Create"])
-    |> validate_actor_presence()
+    |> validate_actor_is_active()
     |> validate_any_presence([:to, :cc])
     |> validate_actors_match(meta)
     |> validate_object_nonexistence()
+    |> validate_object_containment()
+  end
+
+  def validate_object_containment(cng) do
+    actor = get_field(cng, :actor)
+
+    cng
+    |> validate_change(:object, fn :object, object_id ->
+      %URI{host: object_id_host} = URI.parse(object_id)
+      %URI{host: actor_host} = URI.parse(actor)
+
+      if object_id_host == actor_host do
+        []
+      else
+        [{:object, "The host of the object id doesn't match with the host of the actor"}]
+      end
+    end)
   end
 
   def validate_object_nonexistence(cng) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
index 211f520c4..f7f3b1354 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
   use Ecto.Schema
 
+  alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
   alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.Types
@@ -40,13 +41,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     field(:announcement_count, :integer, default: 0)
     field(:inReplyTo, :string)
     field(:uri, Types.Uri)
+    # short identifier for PleromaFE to group statuses by context
+    field(:context_id, :integer)
 
     field(:likes, {:array, :string}, default: [])
     field(:announcements, {:array, :string}, default: [])
 
-    # see if needed
-    field(:context_id, :string)
-
     field(:closed, Types.DateTime)
     field(:voters, {:array, Types.ObjectID}, default: [])
     embeds_many(:anyOf, QuestionOptionsValidator)
@@ -70,7 +70,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     |> changeset(data)
   end
 
-  def fix(data) do
+  defp fix_closed(data) do
     cond do
       is_binary(data["closed"]) -> data
       is_binary(data["endTime"]) -> Map.put(data, "closed", data["endTime"])
@@ -78,6 +78,23 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     end
   end
 
+  # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults
+  defp fix_defaults(data) do
+    %{data: %{"id" => context}, id: context_id} = Utils.create_context(data["context"])
+
+    data
+    |> Map.put_new_lazy("id", &Utils.generate_object_id/0)
+    |> Map.put_new_lazy("published", &Utils.make_date/0)
+    |> Map.put_new("context", context)
+    |> Map.put_new("context_id", context_id)
+  end
+
+  defp fix(data) do
+    data
+    |> fix_closed()
+    |> fix_defaults()
+  end
+
   def changeset(struct, data) do
     data = fix(data)
 
@@ -92,7 +109,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     |> validate_inclusion(:type, ["Question"])
     |> validate_required([:id, :actor, :type, :content, :context])
     |> CommonValidations.validate_any_presence([:cc, :to])
-    |> CommonValidations.validate_actor_presence()
+    |> CommonValidations.validate_actor_is_active()
     |> CommonValidations.validate_any_presence([:oneOf, :anyOf])
+    |> CommonValidations.validate_host_match()
   end
 end
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index a78ec411f..c17197bec 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   """
   alias Pleroma.Activity
   alias Pleroma.Activity.Ir.Topics
+  alias Pleroma.ActivityExpiration
   alias Pleroma.Chat
   alias Pleroma.Chat.MessageReference
   alias Pleroma.FollowingRelationship
@@ -19,6 +20,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Push
   alias Pleroma.Web.Streamer
+  alias Pleroma.Workers.BackgroundWorker
 
   def handle(object, meta \\ [])
 
@@ -135,10 +137,24 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   # Tasks this handles
   # - Actually create object
   # - Rollback if we couldn't create it
+  # - Increase the user note count
+  # - Increase the reply count
   # - Set up notifications
   def handle(%{data: %{"type" => "Create"}} = activity, meta) do
-    with {:ok, _object, meta} <- handle_object_creation(meta[:object_data], meta) do
+    with {:ok, object, meta} <- handle_object_creation(meta[:object_data], meta),
+         %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
       {:ok, notifications} = Notification.create_notifications(activity, do_send: false)
+      {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object)
+
+      if in_reply_to = object.data["inReplyTo"] do
+        Object.increase_replies_count(in_reply_to)
+      end
+
+      if expires_at = activity.data["expires_at"] do
+        ActivityExpiration.create(activity, expires_at)
+      end
+
+      BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
 
       meta =
         meta
@@ -268,6 +284,18 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
     end
   end
 
+  def handle_object_creation(%{"type" => "Answer"} = object_map, meta) do
+    with {:ok, object, meta} <- Pipeline.common_pipeline(object_map, meta) do
+      Object.increase_vote_count(
+        object.data["inReplyTo"],
+        object.data["name"],
+        object.data["actor"]
+      )
+
+      {:ok, object, meta}
+    end
+  end
+
   def handle_object_creation(%{"type" => "Question"} = object, meta) do
     with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
       {:ok, object, meta}
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 9900602e4..26325d5de 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -419,6 +419,29 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     end)
   end
 
+  # Compatibility wrapper for Mastodon votes
+  defp handle_create(%{"object" => %{"type" => "Answer"}} = data, _user) do
+    handle_incoming(data)
+  end
+
+  defp handle_create(%{"object" => object} = data, user) do
+    %{
+      to: data["to"],
+      object: object,
+      actor: user,
+      context: object["context"],
+      local: false,
+      published: data["published"],
+      additional:
+        Map.take(data, [
+          "cc",
+          "directMessage",
+          "id"
+        ])
+    }
+    |> ActivityPub.create()
+  end
+
   def handle_incoming(data, options \\ [])
 
   # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them
@@ -461,26 +484,14 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     actor = Containment.get_actor(data)
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
-         {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(actor),
-         data <- Map.put(data, "actor", actor) |> fix_addressing() do
-      object = fix_object(object, options)
+         {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(actor) do
+      data =
+        data
+        |> Map.put("object", fix_object(object, options))
+        |> Map.put("actor", actor)
+        |> fix_addressing()
 
-      params = %{
-        to: data["to"],
-        object: object,
-        actor: user,
-        context: object["context"],
-        local: false,
-        published: data["published"],
-        additional:
-          Map.take(data, [
-            "cc",
-            "directMessage",
-            "id"
-          ])
-      }
-
-      with {:ok, created_activity} <- ActivityPub.create(params) do
+      with {:ok, created_activity} <- handle_create(data, user) do
         reply_depth = (options[:depth] || 0) + 1
 
         if Federator.allowed_thread_distance?(reply_depth) do
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 692ceab1e..c08e0ffeb 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -308,18 +308,19 @@ defmodule Pleroma.Web.CommonAPI do
          {:ok, options, choices} <- normalize_and_validate_choices(choices, object) do
       answer_activities =
         Enum.map(choices, fn index ->
-          answer_data = make_answer_data(user, object, Enum.at(options, index)["name"])
+          {:ok, answer_object, _meta} =
+            Builder.answer(user, object, Enum.at(options, index)["name"])
 
-          {:ok, activity} =
-            ActivityPub.create(%{
-              to: answer_data["to"],
-              actor: user,
-              context: object.data["context"],
-              object: answer_data,
-              additional: %{"cc" => answer_data["cc"]}
-            })
+          {:ok, activity_data, _meta} = Builder.create(user, answer_object, [])
 
-          activity
+          {:ok, activity, _meta} =
+            activity_data
+            |> Map.put("cc", answer_object["cc"])
+            |> Map.put("context", answer_object["context"])
+            |> Pipeline.common_pipeline(local: true)
+
+          # TODO: Do preload of Pleroma.Object in Pipeline
+          Activity.normalize(activity.data)
         end)
 
       object = Object.get_cached_by_ap_id(object.data["id"])
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 9c38b73eb..9d7b24eb2 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -548,17 +548,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     end
   end
 
-  def make_answer_data(%User{ap_id: ap_id}, object, name) do
-    %{
-      "type" => "Answer",
-      "actor" => ap_id,
-      "cc" => [object.data["actor"]],
-      "to" => [],
-      "name" => name,
-      "inReplyTo" => object.data["id"]
-    }
-  end
-
   def validate_character_limit("" = _full_payload, [] = _attachments) do
     {:error, dgettext("errors", "Cannot post an empty status without attachments")}
   end
diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs
index 4184b93ce..62b5b06aa 100644
--- a/test/web/activity_pub/transmogrifier_test.exs
+++ b/test/web/activity_pub/transmogrifier_test.exs
@@ -282,7 +282,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
       assert object.data["length"] == 180_000
     end
 
-    test "it rewrites Note votes to Answers and increments vote counters on question activities" do
+    test "it rewrites Note votes to Answer and increments vote counters on Question activities" do
       user = insert(:user)
 
       {:ok, activity} =

From 39870d99b810940ccce430411c9fc6939f760663 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 19 Jun 2020 23:31:19 +0200
Subject: [PATCH 12/57] transmogrifier tests: Move & enhance in specialised
 modules

---
 .../transmogrifier/answer_handling_test.exs   | 78 ++++++++++++++++
 .../transmogrifier/question_handling_test.exs | 65 +++++++++++++
 test/web/activity_pub/transmogrifier_test.exs | 92 -------------------
 3 files changed, 143 insertions(+), 92 deletions(-)
 create mode 100644 test/web/activity_pub/transmogrifier/answer_handling_test.exs
 create mode 100644 test/web/activity_pub/transmogrifier/question_handling_test.exs

diff --git a/test/web/activity_pub/transmogrifier/answer_handling_test.exs b/test/web/activity_pub/transmogrifier/answer_handling_test.exs
new file mode 100644
index 000000000..0f6605c3f
--- /dev/null
+++ b/test/web/activity_pub/transmogrifier/answer_handling_test.exs
@@ -0,0 +1,78 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.Transmogrifier.AnswerHandlingTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.CommonAPI
+
+  import Pleroma.Factory
+
+  setup_all do
+    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
+  test "incoming, rewrites Note to Answer and increments vote counters" do
+    user = insert(:user)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        status: "suya...",
+        poll: %{options: ["suya", "suya.", "suya.."], expires_in: 10}
+      })
+
+    object = Object.normalize(activity)
+
+    data =
+      File.read!("test/fixtures/mastodon-vote.json")
+      |> Poison.decode!()
+      |> Kernel.put_in(["to"], user.ap_id)
+      |> Kernel.put_in(["object", "inReplyTo"], object.data["id"])
+      |> Kernel.put_in(["object", "to"], user.ap_id)
+
+    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+    answer_object = Object.normalize(activity)
+    assert answer_object.data["type"] == "Answer"
+    assert answer_object.data["inReplyTo"] == object.data["id"]
+
+    new_object = Object.get_by_ap_id(object.data["id"])
+    assert new_object.data["replies_count"] == object.data["replies_count"]
+
+    assert Enum.any?(
+             new_object.data["oneOf"],
+             fn
+               %{"name" => "suya..", "replies" => %{"totalItems" => 1}} -> true
+               _ -> false
+             end
+           )
+  end
+
+  test "outgoing, rewrites Answer to Note" do
+    user = insert(:user)
+
+    {:ok, poll_activity} =
+      CommonAPI.post(user, %{
+        status: "suya...",
+        poll: %{options: ["suya", "suya.", "suya.."], expires_in: 10}
+      })
+
+    poll_object = Object.normalize(poll_activity)
+    # TODO: Replace with CommonAPI vote creation when implemented
+    data =
+      File.read!("test/fixtures/mastodon-vote.json")
+      |> Poison.decode!()
+      |> Kernel.put_in(["to"], user.ap_id)
+      |> Kernel.put_in(["object", "inReplyTo"], poll_object.data["id"])
+      |> Kernel.put_in(["object", "to"], user.ap_id)
+
+    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+    {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
+
+    assert data["object"]["type"] == "Note"
+  end
+end
diff --git a/test/web/activity_pub/transmogrifier/question_handling_test.exs b/test/web/activity_pub/transmogrifier/question_handling_test.exs
new file mode 100644
index 000000000..b7b9a1a7b
--- /dev/null
+++ b/test/web/activity_pub/transmogrifier/question_handling_test.exs
@@ -0,0 +1,65 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
+  setup_all do
+    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
+  test "Mastodon Question activity" do
+    data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!()
+
+    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+
+    object = Object.normalize(activity, false)
+
+    assert object.data["closed"] == "2019-05-11T09:03:36Z"
+
+    assert object.data["context"] ==
+             "tag:mastodon.sdf.org,2019-05-10:objectId=15095122:objectType=Conversation"
+
+    assert object.data["context_id"]
+
+    assert object.data["anyOf"] == []
+
+    assert Enum.sort(object.data["oneOf"]) ==
+             Enum.sort([
+               %{
+                 "name" => "25 char limit is dumb",
+                 "replies" => %{"totalItems" => 0, "type" => "Collection"},
+                 "type" => "Note"
+               },
+               %{
+                 "name" => "Dunno",
+                 "replies" => %{"totalItems" => 0, "type" => "Collection"},
+                 "type" => "Note"
+               },
+               %{
+                 "name" => "Everyone knows that!",
+                 "replies" => %{"totalItems" => 1, "type" => "Collection"},
+                 "type" => "Note"
+               },
+               %{
+                 "name" => "I can't even fit a funny",
+                 "replies" => %{"totalItems" => 1, "type" => "Collection"},
+                 "type" => "Note"
+               }
+             ])
+  end
+
+  test "returns an error if received a second time" do
+    data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!()
+
+    assert {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+
+    assert {:error, {:validate_object, {:error, _}}} = Transmogrifier.handle_incoming(data)
+  end
+end
diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs
index 62b5b06aa..272431135 100644
--- a/test/web/activity_pub/transmogrifier_test.exs
+++ b/test/web/activity_pub/transmogrifier_test.exs
@@ -217,42 +217,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
       assert Enum.at(object.data["tag"], 2) == "moo"
     end
 
-    test "it works for incoming questions" do
-      data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!()
-
-      {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
-
-      object = Object.normalize(activity, false)
-
-      assert object.data["closed"] == "2019-05-11T09:03:36Z"
-
-      assert object.data["anyOf"] == []
-
-      assert Enum.sort(object.data["oneOf"]) ==
-               Enum.sort([
-                 %{
-                   "name" => "25 char limit is dumb",
-                   "replies" => %{"totalItems" => 0, "type" => "Collection"},
-                   "type" => "Note"
-                 },
-                 %{
-                   "name" => "Dunno",
-                   "replies" => %{"totalItems" => 0, "type" => "Collection"},
-                   "type" => "Note"
-                 },
-                 %{
-                   "name" => "Everyone knows that!",
-                   "replies" => %{"totalItems" => 1, "type" => "Collection"},
-                   "type" => "Note"
-                 },
-                 %{
-                   "name" => "I can't even fit a funny",
-                   "replies" => %{"totalItems" => 1, "type" => "Collection"},
-                   "type" => "Note"
-                 }
-               ])
-    end
-
     test "it works for incoming listens" do
       data = %{
         "@context" => "https://www.w3.org/ns/activitystreams",
@@ -282,38 +246,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
       assert object.data["length"] == 180_000
     end
 
-    test "it rewrites Note votes to Answer and increments vote counters on Question activities" do
-      user = insert(:user)
-
-      {:ok, activity} =
-        CommonAPI.post(user, %{
-          status: "suya...",
-          poll: %{options: ["suya", "suya.", "suya.."], expires_in: 10}
-        })
-
-      object = Object.normalize(activity)
-
-      data =
-        File.read!("test/fixtures/mastodon-vote.json")
-        |> Poison.decode!()
-        |> Kernel.put_in(["to"], user.ap_id)
-        |> Kernel.put_in(["object", "inReplyTo"], object.data["id"])
-        |> Kernel.put_in(["object", "to"], user.ap_id)
-
-      {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
-      answer_object = Object.normalize(activity)
-      assert answer_object.data["type"] == "Answer"
-      object = Object.get_by_ap_id(object.data["id"])
-
-      assert Enum.any?(
-               object.data["oneOf"],
-               fn
-                 %{"name" => "suya..", "replies" => %{"totalItems" => 1}} -> true
-                 _ -> false
-               end
-             )
-    end
-
     test "it works for incoming notices with contentMap" do
       data =
         File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!()
@@ -1280,30 +1212,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
     end
   end
 
-  test "Rewrites Answers to Notes" do
-    user = insert(:user)
-
-    {:ok, poll_activity} =
-      CommonAPI.post(user, %{
-        status: "suya...",
-        poll: %{options: ["suya", "suya.", "suya.."], expires_in: 10}
-      })
-
-    poll_object = Object.normalize(poll_activity)
-    # TODO: Replace with CommonAPI vote creation when implemented
-    data =
-      File.read!("test/fixtures/mastodon-vote.json")
-      |> Poison.decode!()
-      |> Kernel.put_in(["to"], user.ap_id)
-      |> Kernel.put_in(["object", "inReplyTo"], poll_object.data["id"])
-      |> Kernel.put_in(["object", "to"], user.ap_id)
-
-    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
-    {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
-
-    assert data["object"]["type"] == "Note"
-  end
-
   describe "fix_explicit_addressing" do
     setup do
       user = insert(:user)

From fe6924d00d710e7e568d0ed40c02d3c516d91460 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 19 Jun 2020 23:43:36 +0200
Subject: [PATCH 13/57] CreateGenericValidator: add expires_at

---
 .../activity_pub/object_validators/create_generic_validator.ex  | 1 +
 lib/pleroma/web/activity_pub/side_effects.ex                    | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
index 4ad4ca0de..f467ccc7c 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
@@ -23,6 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
     field(:to, Types.Recipients, default: [])
     field(:cc, Types.Recipients, default: [])
     field(:object, Types.ObjectID)
+    field(:expires_at, Types.DateTime)
   end
 
   def cast_data(data) do
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index c17197bec..5104d38ee 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -139,6 +139,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   # - Rollback if we couldn't create it
   # - Increase the user note count
   # - Increase the reply count
+  # - Increase replies count
+  # - Set up ActivityExpiration
   # - Set up notifications
   def handle(%{data: %{"type" => "Create"}} = activity, meta) do
     with {:ok, object, meta} <- handle_object_creation(meta[:object_data], meta),

From 435a65b9768b9f2c614ba5e00d83753b5834a695 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 20 Jun 2020 00:07:39 +0200
Subject: [PATCH 14/57] QuestionValidator: Use AttachmentValidator

---
 .../activity_pub/object_validators/question_validator.ex   | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
index f7f3b1354..56f02777d 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
   use Ecto.Schema
 
   alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
   alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.Types
@@ -34,8 +35,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     # TODO: Write type
     field(:emoji, :map, default: %{})
     field(:sensitive, :boolean, default: false)
-    # TODO: Write type
-    field(:attachment, {:array, :map}, default: [])
+    embeds_many(:attachment, AttachmentValidator)
     field(:replies_count, :integer, default: 0)
     field(:like_count, :integer, default: 0)
     field(:announcement_count, :integer, default: 0)
@@ -99,7 +99,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     data = fix(data)
 
     struct
-    |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf])
+    |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment])
+    |> cast_embed(:attachment)
     |> cast_embed(:anyOf)
     |> cast_embed(:oneOf)
   end

From d713930ea7fe78f705f534febe4ceaf0a0216d24 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 20 Jun 2020 00:23:04 +0200
Subject: [PATCH 15/57] Fixup for EctoType module move

---
 .../object_validators/answer_validator.ex      |  8 ++++----
 .../create_generic_validator.ex                | 14 +++++++-------
 .../object_validators/question_validator.ex    | 18 +++++++++---------
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
index 8d4c92520..9861eec7f 100644
--- a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
@@ -5,8 +5,8 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
   use Ecto.Schema
 
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   import Ecto.Changeset
 
@@ -14,7 +14,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
   @derive Jason.Encoder
 
   embedded_schema do
-    field(:id, Types.ObjectID, primary_key: true)
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
     field(:to, {:array, :string}, default: [])
     field(:cc, {:array, :string}, default: [])
 
@@ -25,8 +25,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
     field(:type, :string)
     field(:name, :string)
     field(:inReplyTo, :string)
-    field(:attributedTo, Types.ObjectID)
-    field(:actor, Types.ObjectID)
+    field(:attributedTo, ObjectValidators.ObjectID)
+    field(:actor, ObjectValidators.ObjectID)
   end
 
   def cast_and_apply(data) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
index f467ccc7c..97e2def10 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
@@ -8,8 +8,8 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
   use Ecto.Schema
 
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   import Ecto.Changeset
   import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@@ -17,13 +17,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
   @primary_key false
 
   embedded_schema do
-    field(:id, Types.ObjectID, primary_key: true)
-    field(:actor, Types.ObjectID)
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:actor, ObjectValidators.ObjectID)
     field(:type, :string)
-    field(:to, Types.Recipients, default: [])
-    field(:cc, Types.Recipients, default: [])
-    field(:object, Types.ObjectID)
-    field(:expires_at, Types.DateTime)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
+    field(:object, ObjectValidators.ObjectID)
+    field(:expires_at, ObjectValidators.DateTime)
   end
 
   def cast_data(data) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
index 56f02777d..53cf35d40 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -5,11 +5,11 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
   use Ecto.Schema
 
-  alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
   alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+  alias Pleroma.Web.ActivityPub.Utils
 
   import Ecto.Changeset
 
@@ -18,7 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
 
   # Extends from NoteValidator
   embedded_schema do
-    field(:id, Types.ObjectID, primary_key: true)
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
     field(:to, {:array, :string}, default: [])
     field(:cc, {:array, :string}, default: [])
     field(:bto, {:array, :string}, default: [])
@@ -28,10 +28,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     field(:type, :string)
     field(:content, :string)
     field(:context, :string)
-    field(:actor, Types.ObjectID)
-    field(:attributedTo, Types.ObjectID)
+    field(:actor, ObjectValidators.ObjectID)
+    field(:attributedTo, ObjectValidators.ObjectID)
     field(:summary, :string)
-    field(:published, Types.DateTime)
+    field(:published, ObjectValidators.DateTime)
     # TODO: Write type
     field(:emoji, :map, default: %{})
     field(:sensitive, :boolean, default: false)
@@ -40,15 +40,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     field(:like_count, :integer, default: 0)
     field(:announcement_count, :integer, default: 0)
     field(:inReplyTo, :string)
-    field(:uri, Types.Uri)
+    field(:uri, ObjectValidators.Uri)
     # short identifier for PleromaFE to group statuses by context
     field(:context_id, :integer)
 
     field(:likes, {:array, :string}, default: [])
     field(:announcements, {:array, :string}, default: [])
 
-    field(:closed, Types.DateTime)
-    field(:voters, {:array, Types.ObjectID}, default: [])
+    field(:closed, ObjectValidators.DateTime)
+    field(:voters, {:array, ObjectValidators.ObjectID}, default: [])
     embeds_many(:anyOf, QuestionOptionsValidator)
     embeds_many(:oneOf, QuestionOptionsValidator)
   end

From c19bdc811e526f83a2120c58f858044f4ff96e5f Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 23 Jun 2020 05:30:34 +0200
Subject: [PATCH 16/57] Fix attachments in polls

---
 .../object_validators/url_object_validator.ex |  2 +-
 .../web/activity_pub/transmogrifier.ex        | 12 ++-
 test/fixtures/tesla_mock/poll_attachment.json | 99 +++++++++++++++++++
 test/object/fetcher_test.exs                  |  7 ++
 test/support/http_request_mock.ex             |  8 ++
 test/web/activity_pub/transmogrifier_test.exs | 27 ++++-
 6 files changed, 145 insertions(+), 10 deletions(-)
 create mode 100644 test/fixtures/tesla_mock/poll_attachment.json

diff --git a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex
index f64fac46d..881030f38 100644
--- a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex
@@ -13,7 +13,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do
   embedded_schema do
     field(:type, :string)
     field(:href, ObjectValidators.Uri)
-    field(:mediaType, :string)
+    field(:mediaType, :string, default: "application/octet-stream")
   end
 
   def changeset(struct, data) do
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 26325d5de..0ad982720 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -240,13 +240,17 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
         if href do
           attachment_url =
-            %{"href" => href}
+            %{
+              "href" => href,
+              "type" => Map.get(url || %{}, "type", "Link")
+            }
             |> Maps.put_if_present("mediaType", media_type)
-            |> Maps.put_if_present("type", Map.get(url || %{}, "type"))
 
-          %{"url" => [attachment_url]}
+          %{
+            "url" => [attachment_url],
+            "type" => data["type"] || "Document"
+          }
           |> Maps.put_if_present("mediaType", media_type)
-          |> Maps.put_if_present("type", data["type"])
           |> Maps.put_if_present("name", data["name"])
         else
           nil
diff --git a/test/fixtures/tesla_mock/poll_attachment.json b/test/fixtures/tesla_mock/poll_attachment.json
new file mode 100644
index 000000000..92e822dc8
--- /dev/null
+++ b/test/fixtures/tesla_mock/poll_attachment.json
@@ -0,0 +1,99 @@
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    "https://patch.cx/schemas/litepub-0.1.jsonld",
+    {
+      "@language": "und"
+    }
+  ],
+  "actor": "https://patch.cx/users/rin",
+  "anyOf": [],
+  "attachment": [
+    {
+      "mediaType": "image/jpeg",
+      "name": "screenshot_mpv:Totoro@01:18:44.345.jpg",
+      "type": "Document",
+      "url": "https://shitposter.club/media/3bb4c4d402f8fdcc7f80963c3d7cf6f10f936897fd374922ade33199d2f86d87.jpg?name=screenshot_mpv%3ATotoro%4001%3A18%3A44.345.jpg"
+    }
+  ],
+  "attributedTo": "https://patch.cx/users/rin",
+  "cc": [
+    "https://patch.cx/users/rin/followers"
+  ],
+  "closed": "2020-06-19T23:22:02.754678Z",
+  "content": "<span class=\"h-card\"><a class=\"u-url mention\" data-user=\"9vwjTNzEWEM1TfkBGq\" href=\"https://mastodon.sdf.org/users/rinpatch\" rel=\"ugc\">@<span>rinpatch</span></a></span>",
+  "closed": "2019-09-19T00:32:36.785333",
+  "content": "can you vote on this poll?",
+  "id": "https://patch.cx/objects/tesla_mock/poll_attachment",
+  "oneOf": [
+    {
+      "name": "a",
+      "replies": {
+        "totalItems": 0,
+        "type": "Collection"
+      },
+      "type": "Note"
+    },
+    {
+      "name": "A",
+      "replies": {
+        "totalItems": 0,
+        "type": "Collection"
+      },
+      "type": "Note"
+    },
+    {
+      "name": "Aa",
+      "replies": {
+        "totalItems": 0,
+        "type": "Collection"
+      },
+      "type": "Note"
+    },
+    {
+      "name": "AA",
+      "replies": {
+        "totalItems": 0,
+        "type": "Collection"
+      },
+      "type": "Note"
+    },
+    {
+      "name": "AAa",
+      "replies": {
+        "totalItems": 1,
+        "type": "Collection"
+      },
+      "type": "Note"
+    },
+    {
+      "name": "AAA",
+      "replies": {
+        "totalItems": 3,
+        "type": "Collection"
+      },
+      "type": "Note"
+    }
+  ],
+  "published": "2020-06-19T23:12:02.786113Z",
+  "sensitive": false,
+  "summary": "",
+  "tag": [
+    {
+      "href": "https://mastodon.sdf.org/users/rinpatch",
+      "name": "@rinpatch@mastodon.sdf.org",
+      "type": "Mention"
+    }
+  ],
+  "to": [
+    "https://www.w3.org/ns/activitystreams#Public",
+    "https://mastodon.sdf.org/users/rinpatch"
+  ],
+  "type": "Question",
+  "voters": [
+    "https://shitposter.club/users/moonman",
+    "https://skippers-bin.com/users/7v1w1r8ce6",
+    "https://mastodon.sdf.org/users/rinpatch",
+    "https://mastodon.social/users/emelie"
+  ]
+}
diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs
index d9098ea1b..16cfa7f5c 100644
--- a/test/object/fetcher_test.exs
+++ b/test/object/fetcher_test.exs
@@ -177,6 +177,13 @@ defmodule Pleroma.Object.FetcherTest do
                  "https://mastodon.example.org/users/userisgone404"
                )
     end
+
+    test "it can fetch pleroma polls with attachments" do
+      {:ok, object} =
+        Fetcher.fetch_object_from_id("https://patch.cx/objects/tesla_mock/poll_attachment")
+
+      assert object
+    end
   end
 
   describe "pruning" do
diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex
index 19a202654..eeeba7880 100644
--- a/test/support/http_request_mock.ex
+++ b/test/support/http_request_mock.ex
@@ -82,6 +82,14 @@ defmodule HttpRequestMock do
      }}
   end
 
+  def get("https://patch.cx/objects/tesla_mock/poll_attachment", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/poll_attachment.json")
+     }}
+  end
+
   def get(
         "https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/emelie",
         _,
diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs
index 272431135..7269e81bb 100644
--- a/test/web/activity_pub/transmogrifier_test.exs
+++ b/test/web/activity_pub/transmogrifier_test.exs
@@ -620,7 +620,8 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
                    %{
                      "href" =>
                        "https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4",
-                     "mediaType" => "video/mp4"
+                     "mediaType" => "video/mp4",
+                     "type" => "Link"
                    }
                  ]
                }
@@ -639,7 +640,8 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
                    %{
                      "href" =>
                        "https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4",
-                     "mediaType" => "video/mp4"
+                     "mediaType" => "video/mp4",
+                     "type" => "Link"
                    }
                  ]
                }
@@ -1459,8 +1461,13 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
                "attachment" => [
                  %{
                    "mediaType" => "video/mp4",
+                   "type" => "Document",
                    "url" => [
-                     %{"href" => "https://peertube.moe/stat-480.mp4", "mediaType" => "video/mp4"}
+                     %{
+                       "href" => "https://peertube.moe/stat-480.mp4",
+                       "mediaType" => "video/mp4",
+                       "type" => "Link"
+                     }
                    ]
                  }
                ]
@@ -1477,14 +1484,24 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
                "attachment" => [
                  %{
                    "mediaType" => "video/mp4",
+                   "type" => "Document",
                    "url" => [
-                     %{"href" => "https://pe.er/stat-480.mp4", "mediaType" => "video/mp4"}
+                     %{
+                       "href" => "https://pe.er/stat-480.mp4",
+                       "mediaType" => "video/mp4",
+                       "type" => "Link"
+                     }
                    ]
                  },
                  %{
                    "mediaType" => "video/mp4",
+                   "type" => "Document",
                    "url" => [
-                     %{"href" => "https://pe.er/stat-480.mp4", "mediaType" => "video/mp4"}
+                     %{
+                       "href" => "https://pe.er/stat-480.mp4",
+                       "mediaType" => "video/mp4",
+                       "type" => "Link"
+                     }
                    ]
                  }
                ]

From bfe2dafd398114240fcc2d3472799d6770904f6a Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 26 Jun 2020 00:07:43 +0200
Subject: [PATCH 17/57] {Answer,Question}Validator: Keep both actor and
 attributedTo for now but sync them

---
 lib/pleroma/web/activity_pub/builder.ex        |  1 +
 .../object_validators/answer_validator.ex      |  8 ++++++--
 .../object_validators/common_validations.ex    | 18 ++++++++++++++++++
 .../create_generic_validator.ex                |  6 +++---
 .../object_validators/question_validator.ex    |  6 +++++-
 lib/pleroma/web/activity_pub/transmogrifier.ex |  7 ++++++-
 6 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex
index e97381954..49ce5a938 100644
--- a/lib/pleroma/web/activity_pub/builder.ex
+++ b/lib/pleroma/web/activity_pub/builder.ex
@@ -120,6 +120,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do
      %{
        "type" => "Answer",
        "actor" => user.ap_id,
+       "attributedTo" => user.ap_id,
        "cc" => [object.data["actor"]],
        "to" => [],
        "name" => name,
diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
index 9861eec7f..ebddd5038 100644
--- a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
@@ -26,6 +26,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
     field(:name, :string)
     field(:inReplyTo, :string)
     field(:attributedTo, ObjectValidators.ObjectID)
+
+    # TODO: Remove actor on objects
     field(:actor, ObjectValidators.ObjectID)
   end
 
@@ -54,8 +56,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
   def validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Answer"])
-    |> validate_required([:id, :inReplyTo, :name])
+    |> validate_required([:id, :inReplyTo, :name, :attributedTo, :actor])
     |> CommonValidations.validate_any_presence([:cc, :to])
-    |> CommonValidations.validate_actor_presence()
+    |> CommonValidations.validate_fields_match([:actor, :attributedTo])
+    |> CommonValidations.validate_actor_is_active()
+    |> CommonValidations.validate_host_match()
   end
 end
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
index 140555a45..e981dacaa 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
@@ -115,4 +115,22 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
       end)
     end
   end
+
+  def validate_fields_match(cng, fields) do
+    unique_fields =
+      fields
+      |> Enum.map(fn field -> get_field(cng, field) end)
+      |> Enum.uniq()
+      |> Enum.count()
+
+    if unique_fields == 1 do
+      cng
+    else
+      fields
+      |> Enum.reduce(cng, fn field, cng ->
+        cng
+        |> add_error(field, "Fields #{inspect(fields)} aren't matching")
+      end)
+    end
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
index 97e2def10..54ea14f89 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
@@ -87,14 +87,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
   end
 
   def validate_actors_match(cng, meta) do
-    object_actor = meta[:object_data]["actor"]
+    attributed_to = meta[:object_data]["attributedTo"] || meta[:object_data]["actor"]
 
     cng
     |> validate_change(:actor, fn :actor, actor ->
-      if actor == object_actor do
+      if actor == attributed_to do
         []
       else
-        [{:actor, "Actor doesn't match with object actor"}]
+        [{:actor, "Actor doesn't match with object attributedTo"}]
       end
     end)
   end
diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
index 53cf35d40..466b3e6c2 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -28,7 +28,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     field(:type, :string)
     field(:content, :string)
     field(:context, :string)
+
+    # TODO: Remove actor on objects
     field(:actor, ObjectValidators.ObjectID)
+
     field(:attributedTo, ObjectValidators.ObjectID)
     field(:summary, :string)
     field(:published, ObjectValidators.DateTime)
@@ -108,8 +111,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
   def validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Question"])
-    |> validate_required([:id, :actor, :type, :content, :context])
+    |> validate_required([:id, :actor, :attributedTo, :type, :content, :context])
     |> CommonValidations.validate_any_presence([:cc, :to])
+    |> CommonValidations.validate_fields_match([:actor, :attributedTo])
     |> CommonValidations.validate_actor_is_active()
     |> CommonValidations.validate_any_presence([:oneOf, :anyOf])
     |> CommonValidations.validate_host_match()
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 0ad982720..6ab8a52c1 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -157,7 +157,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def fix_actor(%{"attributedTo" => actor} = object) do
-    Map.put(object, "actor", Containment.get_actor(%{"actor" => actor}))
+    actor = Containment.get_actor(%{"actor" => actor})
+
+    # TODO: Remove actor field for Objects
+    object
+    |> Map.put("actor", actor)
+    |> Map.put("attributedTo", actor)
   end
 
   def fix_in_reply_to(object, options \\ [])

From 922ca232988b90b7a4fb5918bb76c383c90fd770 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 2 Jul 2020 05:47:18 +0200
Subject: [PATCH 18/57] Question: Add tests on HTML tags in options

Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/1362
---
 .../transmogrifier/question_handling_test.exs | 35 +++++++++++++++++++
 .../web/mastodon_api/views/poll_view_test.exs | 29 +++++++++++++++
 2 files changed, 64 insertions(+)

diff --git a/test/web/activity_pub/transmogrifier/question_handling_test.exs b/test/web/activity_pub/transmogrifier/question_handling_test.exs
index b7b9a1a7b..fba8106b5 100644
--- a/test/web/activity_pub/transmogrifier/question_handling_test.exs
+++ b/test/web/activity_pub/transmogrifier/question_handling_test.exs
@@ -55,6 +55,41 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do
              ])
   end
 
+  test "Mastodon Question activity with HTML tags in plaintext" do
+    options = [
+      %{
+        "type" => "Note",
+        "name" => "<input type=\"date\">",
+        "replies" => %{"totalItems" => 0, "type" => "Collection"}
+      },
+      %{
+        "type" => "Note",
+        "name" => "<input type=\"date\"/>",
+        "replies" => %{"totalItems" => 0, "type" => "Collection"}
+      },
+      %{
+        "type" => "Note",
+        "name" => "<input type=\"date\" />",
+        "replies" => %{"totalItems" => 1, "type" => "Collection"}
+      },
+      %{
+        "type" => "Note",
+        "name" => "<input type=\"date\"></input>",
+        "replies" => %{"totalItems" => 1, "type" => "Collection"}
+      }
+    ]
+
+    data =
+      File.read!("test/fixtures/mastodon-question-activity.json")
+      |> Poison.decode!()
+      |> Kernel.put_in(["object", "oneOf"], options)
+
+    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+    object = Object.normalize(activity, false)
+
+    assert Enum.sort(object.data["oneOf"]) == Enum.sort(options)
+  end
+
   test "returns an error if received a second time" do
     data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!()
 
diff --git a/test/web/mastodon_api/views/poll_view_test.exs b/test/web/mastodon_api/views/poll_view_test.exs
index 76672f36c..b7e2f17ef 100644
--- a/test/web/mastodon_api/views/poll_view_test.exs
+++ b/test/web/mastodon_api/views/poll_view_test.exs
@@ -135,4 +135,33 @@ defmodule Pleroma.Web.MastodonAPI.PollViewTest do
     assert result[:expires_at] == nil
     assert result[:expired] == false
   end
+
+  test "doesn't strips HTML tags" do
+    user = insert(:user)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        status: "What's with the smug face?",
+        poll: %{
+          options: [
+            "<input type=\"date\">",
+            "<input type=\"date\" >",
+            "<input type=\"date\"/>",
+            "<input type=\"date\"></input>"
+          ],
+          expires_in: 20
+        }
+      })
+
+    object = Object.normalize(activity)
+
+    assert %{
+             options: [
+               %{title: "<input type=\"date\">", votes_count: 0},
+               %{title: "<input type=\"date\" >", votes_count: 0},
+               %{title: "<input type=\"date\"/>", votes_count: 0},
+               %{title: "<input type=\"date\"></input>", votes_count: 0}
+             ]
+           } = PollView.render("show.json", %{object: object})
+  end
 end

From e4beff90f5670876184b2593c1b4a49f2339d048 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 2 Jul 2020 05:45:19 +0200
Subject: [PATCH 19/57] Create Question: Add context field to create

---
 lib/pleroma/web/activity_pub/builder.ex         | 10 +++++++++-
 .../create_generic_validator.ex                 | 17 +++++++++++++++++
 lib/pleroma/web/activity_pub/transmogrifier.ex  |  2 ++
 .../transmogrifier/question_handling_test.exs   | 14 ++++++++++++++
 4 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex
index 49ce5a938..1b4c421b8 100644
--- a/lib/pleroma/web/activity_pub/builder.ex
+++ b/lib/pleroma/web/activity_pub/builder.ex
@@ -80,6 +80,13 @@ defmodule Pleroma.Web.ActivityPub.Builder do
   end
 
   def create(actor, object, recipients) do
+    context =
+      if is_map(object) do
+        object["context"]
+      else
+        nil
+      end
+
     {:ok,
      %{
        "id" => Utils.generate_activity_id(),
@@ -88,7 +95,8 @@ defmodule Pleroma.Web.ActivityPub.Builder do
        "object" => object,
        "type" => "Create",
        "published" => DateTime.utc_now() |> DateTime.to_iso8601()
-     }, []}
+     }
+     |> Pleroma.Maps.put_if_present("context", context), []}
   end
 
   def chat_message(actor, recipient, content, opts \\ []) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
index 54ea14f89..ff889330e 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
@@ -24,6 +24,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
     field(:cc, ObjectValidators.Recipients, default: [])
     field(:object, ObjectValidators.ObjectID)
     field(:expires_at, ObjectValidators.DateTime)
+
+    # Should be moved to object, done for CommonAPI.Utils.make_context
+    field(:context, :string)
   end
 
   def cast_data(data) do
@@ -55,6 +58,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
     |> validate_actor_is_active()
     |> validate_any_presence([:to, :cc])
     |> validate_actors_match(meta)
+    |> validate_context_match(meta)
     |> validate_object_nonexistence()
     |> validate_object_containment()
   end
@@ -98,4 +102,17 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
       end
     end)
   end
+
+  def validate_context_match(cng, %{object_data: %{"context" => object_context}}) do
+    cng
+    |> validate_change(:context, fn :context, context ->
+      if context == object_context do
+        []
+      else
+        [{:context, "context field not matching between Create and object (#{object_context})"}]
+      end
+    end)
+  end
+
+  def validate_context_match(cng, _), do: cng
 end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 6ab8a52c1..edabe1130 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -643,6 +643,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       |> Map.put("object", fix_object(object))
       |> fix_addressing()
 
+    data = Map.put_new(data, "context", data["object"]["context"])
+
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
          {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
diff --git a/test/web/activity_pub/transmogrifier/question_handling_test.exs b/test/web/activity_pub/transmogrifier/question_handling_test.exs
index fba8106b5..12516c4ab 100644
--- a/test/web/activity_pub/transmogrifier/question_handling_test.exs
+++ b/test/web/activity_pub/transmogrifier/question_handling_test.exs
@@ -8,6 +8,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do
   alias Pleroma.Activity
   alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.CommonAPI
+
+  import Pleroma.Factory
 
   setup_all do
     Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
@@ -23,6 +26,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do
 
     assert object.data["closed"] == "2019-05-11T09:03:36Z"
 
+    assert object.data["context"] == activity.data["context"]
+
     assert object.data["context"] ==
              "tag:mastodon.sdf.org,2019-05-10:objectId=15095122:objectType=Conversation"
 
@@ -53,6 +58,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do
                  "type" => "Note"
                }
              ])
+
+    user = insert(:user)
+
+    {:ok, reply_activity} = CommonAPI.post(user, %{status: "hewwo", in_reply_to_id: activity.id})
+
+    reply_object = Object.normalize(reply_activity, false)
+
+    assert reply_object.data["context"] == object.data["context"]
+    assert reply_object.data["context_id"] == object.data["context_id"]
   end
 
   test "Mastodon Question activity with HTML tags in plaintext" do

From 0f088d8ce35150d7baa0591a25c831fce0181239 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 4 Aug 2020 14:23:35 +0200
Subject: [PATCH 20/57] question_validator: Allow content to be an empty-string
 (blank)

---
 .../activity_pub/object_validators/question_validator.ex | 2 +-
 .../transmogrifier/question_handling_test.exs            | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
index 466b3e6c2..d248c6aec 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -111,7 +111,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
   def validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Question"])
-    |> validate_required([:id, :actor, :attributedTo, :type, :content, :context])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])
     |> CommonValidations.validate_actor_is_active()
diff --git a/test/web/activity_pub/transmogrifier/question_handling_test.exs b/test/web/activity_pub/transmogrifier/question_handling_test.exs
index 12516c4ab..9fb965d7f 100644
--- a/test/web/activity_pub/transmogrifier/question_handling_test.exs
+++ b/test/web/activity_pub/transmogrifier/question_handling_test.exs
@@ -111,4 +111,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do
 
     assert {:error, {:validate_object, {:error, _}}} = Transmogrifier.handle_incoming(data)
   end
+
+  test "accepts a Question with no content" do
+    data =
+      File.read!("test/fixtures/mastodon-question-activity.json")
+      |> Poison.decode!()
+      |> Kernel.put_in(["object", "content"], "")
+
+    assert {:ok, %Activity{local: false}} = Transmogrifier.handle_incoming(data)
+  end
 end

From 00c4c6a382d9965ea42236232094c4352c9ebae1 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 5 Aug 2020 12:24:34 +0200
Subject: [PATCH 21/57] CommonValidations: Remove superfluous function

The `is_active` functionality was integrated into the presence
checker.
---
 .../object_validators/answer_validator.ex           |  2 +-
 .../object_validators/common_validations.ex         | 13 -------------
 .../object_validators/create_generic_validator.ex   |  2 +-
 .../object_validators/question_validator.ex         |  2 +-
 4 files changed, 3 insertions(+), 16 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
index ebddd5038..323367642 100644
--- a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
@@ -59,7 +59,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
     |> validate_required([:id, :inReplyTo, :name, :attributedTo, :actor])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])
-    |> CommonValidations.validate_actor_is_active()
+    |> CommonValidations.validate_actor_presence()
     |> CommonValidations.validate_host_match()
   end
 end
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
index 57d4456aa..67352f801 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
@@ -47,19 +47,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
     end)
   end
 
-  def validate_actor_is_active(cng, options \\ []) do
-    field_name = Keyword.get(options, :field_name, :actor)
-
-    cng
-    |> validate_change(field_name, fn field_name, actor ->
-      if %User{deactivated: false} = User.get_cached_by_ap_id(actor) do
-        []
-      else
-        [{field_name, "can't find user (or deactivated)"}]
-      end
-    end)
-  end
-
   def validate_object_presence(cng, options \\ []) do
     field_name = Keyword.get(options, :field_name, :object)
     allowed_types = Keyword.get(options, :allowed_types, false)
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
index ff889330e..2569df7f6 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
@@ -55,7 +55,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
     cng
     |> validate_required([:actor, :type, :object])
     |> validate_inclusion(:type, ["Create"])
-    |> validate_actor_is_active()
+    |> validate_actor_presence()
     |> validate_any_presence([:to, :cc])
     |> validate_actors_match(meta)
     |> validate_context_match(meta)
diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
index d248c6aec..694cb6730 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -114,7 +114,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     |> validate_required([:id, :actor, :attributedTo, :type, :context])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])
-    |> CommonValidations.validate_actor_is_active()
+    |> CommonValidations.validate_actor_presence()
     |> CommonValidations.validate_any_presence([:oneOf, :anyOf])
     |> CommonValidations.validate_host_match()
   end

From 70522989d9d1119e5b3d86151f633f849d92f307 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 5 Aug 2020 11:14:58 +0000
Subject: [PATCH 22/57] Apply 1 suggestion(s) to 1 file(s)

---
 lib/pleroma/web/mastodon_api/views/poll_view.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex
index ce595ae8a..1bfc99259 100644
--- a/lib/pleroma/web/mastodon_api/views/poll_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex
@@ -28,10 +28,10 @@ defmodule Pleroma.Web.MastodonAPI.PollView do
 
   def render("show.json", %{object: object} = params) do
     case object.data do
-      %{"anyOf" => options} when is_list(options) and options != [] ->
+      %{"anyOf" => [ _ | _] = options} ->
         render(__MODULE__, "show.json", Map.merge(params, %{multiple: true, options: options}))
 
-      %{"oneOf" => options} when is_list(options) and options != [] ->
+      %{"oneOf" => [ _ | _] = options} ->
         render(__MODULE__, "show.json", Map.merge(params, %{multiple: false, options: options}))
 
       _ ->

From b5f0cef156c1d1dd0376a791d8b4be48591f2c27 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 5 Aug 2020 11:33:21 +0000
Subject: [PATCH 23/57] Apply 1 suggestion(s) to 1 file(s)

---
 lib/pleroma/object.ex | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index 4dd929cfd..b3e654857 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -255,9 +255,7 @@ defmodule Pleroma.Object do
     end
   end
 
-  defp poll_is_multiple?(%Object{data: %{"anyOf" => anyOf}}) do
-    !Enum.empty?(anyOf)
-  end
+ defp poll_is_multiple?(%Object{data: %{"anyOf" => [_ | _]}}), do: true
 
   defp poll_is_multiple?(_), do: false
 

From f889400d05e86d8d9509577946a0ab3a55b3eabb Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 5 Aug 2020 14:51:33 +0200
Subject: [PATCH 24/57] Questions: Move fixes to validators.

---
 .../create_generic_validator.ex               | 19 +++++++++++++++++--
 .../object_validators/question_validator.ex   | 10 ++++++++--
 .../web/activity_pub/transmogrifier.ex        | 11 ++---------
 .../web/mastodon_api/views/poll_view.ex       |  4 ++--
 4 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
index 2569df7f6..60868eae0 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
@@ -29,7 +29,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
     field(:context, :string)
   end
 
-  def cast_data(data) do
+  def cast_data(data, meta \\ []) do
+    data = fix(data, meta)
+
     %__MODULE__{}
     |> changeset(data)
   end
@@ -42,7 +44,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
 
   def cast_and_validate(data, meta \\ []) do
     data
-    |> cast_data
+    |> cast_data(meta)
     |> validate_data(meta)
   end
 
@@ -51,6 +53,19 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
     |> cast(data, __schema__(:fields))
   end
 
+  defp fix_context(data, meta) do
+    if object = meta[:object_data] do
+      Map.put_new(data, "context", object["context"])
+    else
+      data
+    end
+  end
+
+  defp fix(data, meta) do
+    data
+    |> fix_context(meta)
+  end
+
   def validate_data(cng, meta \\ []) do
     cng
     |> validate_required([:actor, :type, :object])
diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
index 694cb6730..f47acf606 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -83,17 +83,23 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
 
   # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults
   defp fix_defaults(data) do
-    %{data: %{"id" => context}, id: context_id} = Utils.create_context(data["context"])
+    %{data: %{"id" => context}, id: context_id} =
+      Utils.create_context(data["context"] || data["conversation"])
 
     data
-    |> Map.put_new_lazy("id", &Utils.generate_object_id/0)
     |> Map.put_new_lazy("published", &Utils.make_date/0)
     |> Map.put_new("context", context)
     |> Map.put_new("context_id", context_id)
   end
 
+  defp fix_attribution(data) do
+    data
+    |> Map.put_new("actor", data["attributedTo"])
+  end
+
   defp fix(data) do
     data
+    |> fix_attribution()
     |> fix_closed()
     |> fix_defaults()
   end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index f85a26679..7381d4476 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -634,17 +634,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def handle_incoming(
-        %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
+        %{"type" => "Create", "object" => %{"type" => objtype}} = data,
         _options
       )
-      when objtype in ["Question", "Answer"] do
-    data =
-      data
-      |> Map.put("object", fix_object(object))
-      |> fix_addressing()
-
-    data = Map.put_new(data, "context", data["object"]["context"])
-
+      when objtype in ["Question", "Answer", "ChatMessage"] do
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
          {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex
index 1bfc99259..1208dc9a0 100644
--- a/lib/pleroma/web/mastodon_api/views/poll_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex
@@ -28,10 +28,10 @@ defmodule Pleroma.Web.MastodonAPI.PollView do
 
   def render("show.json", %{object: object} = params) do
     case object.data do
-      %{"anyOf" => [ _ | _] = options} ->
+      %{"anyOf" => [_ | _] = options} ->
         render(__MODULE__, "show.json", Map.merge(params, %{multiple: true, options: options}))
 
-      %{"oneOf" => [ _ | _] = options} ->
+      %{"oneOf" => [_ | _] = options} ->
         render(__MODULE__, "show.json", Map.merge(params, %{multiple: false, options: options}))
 
       _ ->

From f7146583e5f1c2d0e8a198db00dfafced79d0706 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 5 Aug 2020 08:15:57 -0500
Subject: [PATCH 25/57] Remove LDAP mail attribute as a requirement for
 registering an account

---
 lib/pleroma/web/auth/ldap_authenticator.ex | 30 ++++++++--------------
 test/web/oauth/ldap_authorization_test.exs |  4 +--
 2 files changed, 12 insertions(+), 22 deletions(-)

diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex
index f63a66c03..f320ec746 100644
--- a/lib/pleroma/web/auth/ldap_authenticator.ex
+++ b/lib/pleroma/web/auth/ldap_authenticator.ex
@@ -105,29 +105,21 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
            {:base, to_charlist(base)},
            {:filter, :eldap.equalityMatch(to_charlist(uid), to_charlist(name))},
            {:scope, :eldap.wholeSubtree()},
-           {:attributes, ['mail', 'email']},
            {:timeout, @search_timeout}
          ]) do
-      {:ok, {:eldap_search_result, [{:eldap_entry, _, attributes}], _}} ->
-        with {_, [mail]} <- List.keyfind(attributes, 'mail', 0) do
-          params = %{
-            email: :erlang.list_to_binary(mail),
-            name: name,
-            nickname: name,
-            password: password,
-            password_confirmation: password
-          }
+      {:ok, {:eldap_search_result, [{:eldap_entry, _, _}], _}} ->
+        params = %{
+          name: name,
+          nickname: name,
+          password: password,
+          password_confirmation: password
+        }
 
-          changeset = User.register_changeset(%User{}, params)
+        changeset = User.register_changeset(%User{}, params)
 
-          case User.register(changeset) do
-            {:ok, user} -> user
-            error -> error
-          end
-        else
-          _ ->
-            Logger.error("Could not find LDAP attribute mail: #{inspect(attributes)}")
-            {:error, :ldap_registration_missing_attributes}
+        case User.register(changeset) do
+          {:ok, user} -> user
+          error -> error
         end
 
       error ->
diff --git a/test/web/oauth/ldap_authorization_test.exs b/test/web/oauth/ldap_authorization_test.exs
index 011642c08..76ae461c3 100644
--- a/test/web/oauth/ldap_authorization_test.exs
+++ b/test/web/oauth/ldap_authorization_test.exs
@@ -72,9 +72,7 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do
          equalityMatch: fn _type, _value -> :ok end,
          wholeSubtree: fn -> :ok end,
          search: fn _connection, _options ->
-           {:ok,
-            {:eldap_search_result, [{:eldap_entry, '', [{'mail', [to_charlist(user.email)]}]}],
-             []}}
+           {:ok, {:eldap_search_result, [{:eldap_entry, '', []}], []}}
          end,
          close: fn _connection ->
            send(self(), :close_connection)

From 0f9aecbca49c828158d2cb549659a68fb21697df Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 5 Aug 2020 08:18:16 -0500
Subject: [PATCH 26/57] Remove fallback to local database when LDAP is
 unavailable.

In many environments this will not work as the LDAP password and the copy stored in Pleroma will stay synchronized.
---
 lib/pleroma/web/auth/ldap_authenticator.ex |  4 --
 test/web/oauth/ldap_authorization_test.exs | 45 ----------------------
 2 files changed, 49 deletions(-)

diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex
index f320ec746..ec47f6f91 100644
--- a/lib/pleroma/web/auth/ldap_authenticator.ex
+++ b/lib/pleroma/web/auth/ldap_authenticator.ex
@@ -28,10 +28,6 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
          %User{} = user <- ldap_user(name, password) do
       {:ok, user}
     else
-      {:error, {:ldap_connection_error, _}} ->
-        # When LDAP is unavailable, try default authenticator
-        @base.get_user(conn)
-
       {:ldap, _} ->
         @base.get_user(conn)
 
diff --git a/test/web/oauth/ldap_authorization_test.exs b/test/web/oauth/ldap_authorization_test.exs
index 76ae461c3..63b1c0eb8 100644
--- a/test/web/oauth/ldap_authorization_test.exs
+++ b/test/web/oauth/ldap_authorization_test.exs
@@ -7,7 +7,6 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do
   alias Pleroma.Repo
   alias Pleroma.Web.OAuth.Token
   import Pleroma.Factory
-  import ExUnit.CaptureLog
   import Mock
 
   @skip if !Code.ensure_loaded?(:eldap), do: :skip
@@ -99,50 +98,6 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do
     end
   end
 
-  @tag @skip
-  test "falls back to the default authorization when LDAP is unavailable" do
-    password = "testpassword"
-    user = insert(:user, password_hash: Pbkdf2.hash_pwd_salt(password))
-    app = insert(:oauth_app, scopes: ["read", "write"])
-
-    host = Pleroma.Config.get([:ldap, :host]) |> to_charlist
-    port = Pleroma.Config.get([:ldap, :port])
-
-    with_mocks [
-      {:eldap, [],
-       [
-         open: fn [^host], [{:port, ^port}, {:ssl, false} | _] -> {:error, 'connect failed'} end,
-         simple_bind: fn _connection, _dn, ^password -> :ok end,
-         close: fn _connection ->
-           send(self(), :close_connection)
-           :ok
-         end
-       ]}
-    ] do
-      log =
-        capture_log(fn ->
-          conn =
-            build_conn()
-            |> post("/oauth/token", %{
-              "grant_type" => "password",
-              "username" => user.nickname,
-              "password" => password,
-              "client_id" => app.client_id,
-              "client_secret" => app.client_secret
-            })
-
-          assert %{"access_token" => token} = json_response(conn, 200)
-
-          token = Repo.get_by(Token, token: token)
-
-          assert token.user_id == user.id
-        end)
-
-      assert log =~ "Could not open LDAP connection: 'connect failed'"
-      refute_received :close_connection
-    end
-  end
-
   @tag @skip
   test "disallow authorization for wrong LDAP credentials" do
     password = "testpassword"

From 5221879c358a7859d54013597c9ed9ccbb494155 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 5 Aug 2020 15:40:32 +0200
Subject: [PATCH 27/57] Fix linting.

---
 lib/pleroma/object.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index b3e654857..052ad413b 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -255,7 +255,7 @@ defmodule Pleroma.Object do
     end
   end
 
- defp poll_is_multiple?(%Object{data: %{"anyOf" => [_ | _]}}), do: true
+  defp poll_is_multiple?(%Object{data: %{"anyOf" => [_ | _]}}), do: true
 
   defp poll_is_multiple?(_), do: false
 

From d5e4d8a6f3f7b577183809a4b371609aa29fa968 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 5 Aug 2020 09:41:17 -0500
Subject: [PATCH 28/57] Define default authenticator in the config

---
 config/config.exs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/config/config.exs b/config/config.exs
index 933a899ab..257b2e061 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -737,6 +737,8 @@ config :ex_aws, http_client: Pleroma.HTTP.ExAws
 
 config :pleroma, :instances_favicons, enabled: false
 
+config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

From 2192d1e4920e2c6deffe9a205cc2ade27d4dc0b1 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 5 Aug 2020 10:07:31 -0500
Subject: [PATCH 29/57] Permit LDAP users to register without capturing their
 password hash We don't need it, and local auth fallback has been removed.

---
 lib/pleroma/user.ex                        | 19 +++++++++++++++++++
 lib/pleroma/web/auth/ldap_authenticator.ex |  7 +++----
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 09e606b37..df9f34baa 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -638,6 +638,25 @@ defmodule Pleroma.User do
   @spec force_password_reset(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
   def force_password_reset(user), do: update_password_reset_pending(user, true)
 
+  # Used to auto-register LDAP accounts which don't have a password hash
+  def register_changeset(struct, params = %{password: password})
+      when is_nil(password) do
+    params = Map.put_new(params, :accepts_chat_messages, true)
+
+    struct
+    |> cast(params, [
+      :name,
+      :nickname,
+      :accepts_chat_messages
+    ])
+    |> unique_constraint(:nickname)
+    |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames]))
+    |> validate_format(:nickname, local_nickname_regex())
+    |> put_ap_id()
+    |> unique_constraint(:ap_id)
+    |> put_following_and_follower_address()
+  end
+
   def register_changeset(struct, params \\ %{}, opts \\ []) do
     bio_limit = Config.get([:instance, :user_bio_length], 5000)
     name_limit = Config.get([:instance, :user_name_length], 100)
diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex
index ec47f6f91..f667da68b 100644
--- a/lib/pleroma/web/auth/ldap_authenticator.ex
+++ b/lib/pleroma/web/auth/ldap_authenticator.ex
@@ -88,7 +88,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
             user
 
           _ ->
-            register_user(connection, base, uid, name, password)
+            register_user(connection, base, uid, name)
         end
 
       error ->
@@ -96,7 +96,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
     end
   end
 
-  defp register_user(connection, base, uid, name, password) do
+  defp register_user(connection, base, uid, name) do
     case :eldap.search(connection, [
            {:base, to_charlist(base)},
            {:filter, :eldap.equalityMatch(to_charlist(uid), to_charlist(name))},
@@ -107,8 +107,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
         params = %{
           name: name,
           nickname: name,
-          password: password,
-          password_confirmation: password
+          password: nil
         }
 
         changeset = User.register_changeset(%User{}, params)

From 2173945f9012ec0db82a73fc7ed9423899dfd28f Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 5 Aug 2020 17:26:03 +0200
Subject: [PATCH 30/57] MailerTest: Give it some time.

---
 test/emails/mailer_test.exs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/emails/mailer_test.exs b/test/emails/mailer_test.exs
index e6e34cba8..3da45056b 100644
--- a/test/emails/mailer_test.exs
+++ b/test/emails/mailer_test.exs
@@ -19,6 +19,7 @@ defmodule Pleroma.Emails.MailerTest do
   test "not send email when mailer is disabled" do
     Pleroma.Config.put([Pleroma.Emails.Mailer, :enabled], false)
     Mailer.deliver(@email)
+    :timer.sleep(100)
 
     refute_email_sent(
       from: {"Pleroma", "noreply@example.com"},
@@ -30,6 +31,7 @@ defmodule Pleroma.Emails.MailerTest do
 
   test "send email" do
     Mailer.deliver(@email)
+    :timer.sleep(100)
 
     assert_email_sent(
       from: {"Pleroma", "noreply@example.com"},
@@ -41,6 +43,7 @@ defmodule Pleroma.Emails.MailerTest do
 
   test "perform" do
     Mailer.perform(:deliver_async, @email, [])
+    :timer.sleep(100)
 
     assert_email_sent(
       from: {"Pleroma", "noreply@example.com"},

From 9c96fc052a89789b398794761741783eaa86d6a1 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 5 Aug 2020 17:26:53 +0200
Subject: [PATCH 31/57] CommonValidations: Extract modification right checker

---
 .../object_validators/common_validations.ex   | 27 ++++++++++++++++++
 .../object_validators/delete_validator.ex     | 28 +------------------
 .../delete_validation_test.exs                |  2 +-
 3 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
index 67352f801..e4c5d9619 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
@@ -125,4 +125,31 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
       end)
     end
   end
+
+  def same_domain?(cng, field_one \\ :actor, field_two \\ :object) do
+    actor_uri =
+      cng
+      |> get_field(field_one)
+      |> URI.parse()
+
+    object_uri =
+      cng
+      |> get_field(field_two)
+      |> URI.parse()
+
+    object_uri.host == actor_uri.host
+  end
+
+  # This figures out if a user is able to create, delete or modify something
+  # based on the domain and superuser status
+  def validate_modification_rights(cng) do
+    actor = User.get_cached_by_ap_id(get_field(cng, :actor))
+
+    if User.superuser?(actor) || same_domain?(cng) do
+      cng
+    else
+      cng
+      |> add_error(:actor, "is not allowed to modify object")
+    end
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex
index 93a7b0e0b..2634e8d4d 100644
--- a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex
@@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
 
   alias Pleroma.Activity
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.User
 
   import Ecto.Changeset
   import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@@ -59,7 +58,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Delete"])
     |> validate_actor_presence()
-    |> validate_deletion_rights()
+    |> validate_modification_rights()
     |> validate_object_or_user_presence(allowed_types: @deletable_types)
     |> add_deleted_activity_id()
   end
@@ -68,31 +67,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
     !same_domain?(cng)
   end
 
-  defp same_domain?(cng) do
-    actor_uri =
-      cng
-      |> get_field(:actor)
-      |> URI.parse()
-
-    object_uri =
-      cng
-      |> get_field(:object)
-      |> URI.parse()
-
-    object_uri.host == actor_uri.host
-  end
-
-  def validate_deletion_rights(cng) do
-    actor = User.get_cached_by_ap_id(get_field(cng, :actor))
-
-    if User.superuser?(actor) || same_domain?(cng) do
-      cng
-    else
-      cng
-      |> add_error(:actor, "is not allowed to delete object")
-    end
-  end
-
   def cast_and_validate(data) do
     data
     |> cast_data
diff --git a/test/web/activity_pub/object_validators/delete_validation_test.exs b/test/web/activity_pub/object_validators/delete_validation_test.exs
index 42cd18298..02683b899 100644
--- a/test/web/activity_pub/object_validators/delete_validation_test.exs
+++ b/test/web/activity_pub/object_validators/delete_validation_test.exs
@@ -87,7 +87,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidationTest do
 
       {:error, cng} = ObjectValidator.validate(invalid_other_actor, [])
 
-      assert {:actor, {"is not allowed to delete object", []}} in cng.errors
+      assert {:actor, {"is not allowed to modify object", []}} in cng.errors
     end
 
     test "it's valid if the actor of the object is a local superuser",

From 3655175639a004976ef8296a0838e72642ba0b11 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 5 Aug 2020 17:36:27 +0200
Subject: [PATCH 32/57] CommonValidations: Refactor `same_domain?`

---
 .../object_validators/common_validations.ex   | 23 +++++++++++--------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
index e4c5d9619..82a9d39b5 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
@@ -126,18 +126,21 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
     end
   end
 
-  def same_domain?(cng, field_one \\ :actor, field_two \\ :object) do
-    actor_uri =
-      cng
-      |> get_field(field_one)
-      |> URI.parse()
+  def same_domain?(cng, fields \\ [:actor, :object]) do
+    unique_domains =
+      fields
+      |> Enum.map(fn field ->
+        %URI{host: host} =
+          cng
+          |> get_field(field)
+          |> URI.parse()
 
-    object_uri =
-      cng
-      |> get_field(field_two)
-      |> URI.parse()
+        host
+      end)
+      |> Enum.uniq()
+      |> Enum.count()
 
-    object_uri.host == actor_uri.host
+    unique_domains == 1
   end
 
   # This figures out if a user is able to create, delete or modify something

From 9d7ce1a6d014499eb4d55190b81e55da849b5ad0 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 5 Aug 2020 17:56:12 +0200
Subject: [PATCH 33/57] CommonValidations: More refactors.

---
 .../object_validators/common_validations.ex   | 53 +++++++------------
 1 file changed, 18 insertions(+), 35 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
index 82a9d39b5..603d87b8e 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
@@ -84,20 +84,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
   end
 
   def validate_host_match(cng, fields \\ [:id, :actor]) do
-    unique_hosts =
-      fields
-      |> Enum.map(fn field ->
-        %URI{host: host} =
-          cng
-          |> get_field(field)
-          |> URI.parse()
-
-        host
-      end)
-      |> Enum.uniq()
-      |> Enum.count()
-
-    if unique_hosts == 1 do
+    if same_domain?(cng, fields) do
       cng
     else
       fields
@@ -109,13 +96,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
   end
 
   def validate_fields_match(cng, fields) do
-    unique_fields =
-      fields
-      |> Enum.map(fn field -> get_field(cng, field) end)
-      |> Enum.uniq()
-      |> Enum.count()
-
-    if unique_fields == 1 do
+    if map_unique?(cng, fields) do
       cng
     else
       fields
@@ -126,21 +107,23 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
     end
   end
 
+  defp map_unique?(cng, fields, func \\ & &1) do
+    Enum.reduce_while(fields, nil, fn field, acc ->
+      value =
+        cng
+        |> get_field(field)
+        |> func.()
+
+      case {value, acc} do
+        {value, nil} -> {:cont, value}
+        {value, value} -> {:cont, value}
+        _ -> {:halt, false}
+      end
+    end)
+  end
+
   def same_domain?(cng, fields \\ [:actor, :object]) do
-    unique_domains =
-      fields
-      |> Enum.map(fn field ->
-        %URI{host: host} =
-          cng
-          |> get_field(field)
-          |> URI.parse()
-
-        host
-      end)
-      |> Enum.uniq()
-      |> Enum.count()
-
-    unique_domains == 1
+    map_unique?(cng, fields, fn value -> URI.parse(value).host end)
   end
 
   # This figures out if a user is able to create, delete or modify something

From 81126b0142ec54c785952d0c84a2bdef76965fc7 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 5 Aug 2020 11:36:12 -0500
Subject: [PATCH 34/57] Add email to user account only if it exists in LDAP

---
 lib/pleroma/user.ex                        | 9 +++++++++
 lib/pleroma/web/auth/ldap_authenticator.ex | 8 +++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index df9f34baa..6d39c9d1b 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -643,12 +643,21 @@ defmodule Pleroma.User do
       when is_nil(password) do
     params = Map.put_new(params, :accepts_chat_messages, true)
 
+    params =
+      if Map.has_key?(params, :email) do
+        Map.put_new(params, :email, params[:email])
+      else
+        params
+      end
+
     struct
     |> cast(params, [
       :name,
       :nickname,
+      :email,
       :accepts_chat_messages
     ])
+    |> validate_required([:name, :nickname])
     |> unique_constraint(:nickname)
     |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames]))
     |> validate_format(:nickname, local_nickname_regex())
diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex
index f667da68b..b1645a359 100644
--- a/lib/pleroma/web/auth/ldap_authenticator.ex
+++ b/lib/pleroma/web/auth/ldap_authenticator.ex
@@ -103,13 +103,19 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
            {:scope, :eldap.wholeSubtree()},
            {:timeout, @search_timeout}
          ]) do
-      {:ok, {:eldap_search_result, [{:eldap_entry, _, _}], _}} ->
+      {:ok, {:eldap_search_result, [{:eldap_entry, _, attributes}], _}} ->
         params = %{
           name: name,
           nickname: name,
           password: nil
         }
 
+        params =
+          case List.keyfind(attributes, 'mail', 0) do
+            {_, [mail]} -> Map.put_new(params, :email, :erlang.list_to_binary(mail))
+            _ -> params
+          end
+
         changeset = User.register_changeset(%User{}, params)
 
         case User.register(changeset) do

From 2a4bca5bd7e33388193d252f9f956d10ce38ad77 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 5 Aug 2020 11:40:09 -0500
Subject: [PATCH 35/57] Comments are good when they're precise...

---
 lib/pleroma/user.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 6d39c9d1b..69b0e1781 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -638,7 +638,7 @@ defmodule Pleroma.User do
   @spec force_password_reset(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
   def force_password_reset(user), do: update_password_reset_pending(user, true)
 
-  # Used to auto-register LDAP accounts which don't have a password hash
+  # Used to auto-register LDAP accounts which won't have a password hash stored locally
   def register_changeset(struct, params = %{password: password})
       when is_nil(password) do
     params = Map.put_new(params, :accepts_chat_messages, true)

From cb7879c7c148cfc318a176d19b1402e370c509e7 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 5 Aug 2020 11:53:57 -0500
Subject: [PATCH 36/57] Add note about removal of
 Pleroma.Web.Auth.LDAPAuthenticator fallback to
 Pleroma.Web.Auth.PleromaAuthenticator

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index de017e30a..c0d0fe269 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `:instance, rewrite_policy` moved to `:mrf, policies`, `:instance, :mrf_transparency` moved to `:mrf, :transparency`, `:instance, :mrf_transparency_exclusions` moved to `:mrf, :transparency_exclusions`. Old config namespace is deprecated.
 - Configuration: `:media_proxy, whitelist` format changed to host with scheme (e.g. `http://example.com` instead of `example.com`). Domain format is deprecated.
 - **Breaking:** Configuration: `:instance, welcome_user_nickname` moved to `:welcome, :direct_message, :sender_nickname`, `:instance, :welcome_message` moved to `:welcome, :direct_message, :message`. Old config namespace is deprecated.
+- **Breaking:** LDAP: Fallback to local database authentication has been removed for security reasons and lack of a mechanism to ensure the passwords are synchronized when LDAP passwords are updated.
 
 <details>
   <summary>API Changes</summary>

From e639eee82e1e0136bf6e64e571f2b05b5b7b948c Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Thu, 6 Aug 2020 18:01:29 -0500
Subject: [PATCH 37/57] restricted_nicknames: Add names from MastoAPI endpoints

---
 config/config.exs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 933a899ab..393f74372 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -515,7 +515,13 @@ config :pleroma, Pleroma.User,
     "user-search",
     "user_exists",
     "users",
-    "web"
+    "web",
+    "verify_credentials",
+    "update_credentials",
+    "relationships",
+    "search",
+    "confirmation_resend",
+    "mfa"
   ],
   email_blacklist: []
 

From 325c7c924bf05d240fcf535a37d32edf15370a0c Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 11 Feb 2020 00:29:25 +0300
Subject: [PATCH 38/57] Make Floki use fast_html

---
 config/config.exs                 | 2 ++
 test/web/metadata/rel_me_test.exs | 6 ++----
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 933a899ab..78f3232e6 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -737,6 +737,8 @@ config :ex_aws, http_client: Pleroma.HTTP.ExAws
 
 config :pleroma, :instances_favicons, enabled: false
 
+config :floki, :html_parser, Floki.HTMLParser.FastHtml
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"
diff --git a/test/web/metadata/rel_me_test.exs b/test/web/metadata/rel_me_test.exs
index 4107a8459..ac7558014 100644
--- a/test/web/metadata/rel_me_test.exs
+++ b/test/web/metadata/rel_me_test.exs
@@ -9,13 +9,11 @@ defmodule Pleroma.Web.Metadata.Providers.RelMeTest do
 
   test "it renders all links with rel='me' from user bio" do
     bio =
-      ~s(<a href="https://some-link.com">https://some-link.com</a> <a rel="me" href="https://another-link.com">https://another-link.com</a>
-    <link href="http://some.com"> <link rel="me" href="http://some3.com>")
-
+      ~s(<a href="https://some-link.com">https://some-link.com</a> <a rel="me" href="https://another-link.com">https://another-link.com</a> <link href="http://some.com"> <link rel="me" href="http://some3.com">)
     user = insert(:user, %{bio: bio})
 
     assert RelMe.build_tags(%{user: user}) == [
-             {:link, [rel: "me", href: "http://some3.com>"], []},
+             {:link, [rel: "me", href: "http://some3.com"], []},
              {:link, [rel: "me", href: "https://another-link.com"], []}
            ]
   end

From 7e23a48d38da50f1653f37c65610623f585ada9e Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 16 Jul 2020 21:09:48 +0300
Subject: [PATCH 39/57] rel me test: fix HTML so broken browsers (and therefore
 lexbor) refuse to parse it like mochiweb does

---
 test/web/metadata/rel_me_test.exs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/web/metadata/rel_me_test.exs b/test/web/metadata/rel_me_test.exs
index ac7558014..2293d6e13 100644
--- a/test/web/metadata/rel_me_test.exs
+++ b/test/web/metadata/rel_me_test.exs
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.Metadata.Providers.RelMeTest do
   test "it renders all links with rel='me' from user bio" do
     bio =
       ~s(<a href="https://some-link.com">https://some-link.com</a> <a rel="me" href="https://another-link.com">https://another-link.com</a> <link href="http://some.com"> <link rel="me" href="http://some3.com">)
+
     user = insert(:user, %{bio: bio})
 
     assert RelMe.build_tags(%{user: user}) == [

From c662b09eee7f1e8a598c595f66e5b38b5dcbad45 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 7 Aug 2020 16:45:04 +0300
Subject: [PATCH 40/57] mix.exs: update fast_sanitize to 0.2.0

---
 mix.exs  | 2 +-
 mix.lock | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/mix.exs b/mix.exs
index aab833c5e..11fdb1670 100644
--- a/mix.exs
+++ b/mix.exs
@@ -127,7 +127,7 @@ defmodule Pleroma.Mixfile do
       {:pbkdf2_elixir, "~> 1.2"},
       {:bcrypt_elixir, "~> 2.2"},
       {:trailing_format_plug, "~> 0.0.7"},
-      {:fast_sanitize, "~> 0.1"},
+      {:fast_sanitize, "~> 0.2.0"},
       {:html_entities, "~> 0.5", override: true},
       {:phoenix_html, "~> 2.14"},
       {:calendar, "~> 1.0"},
diff --git a/mix.lock b/mix.lock
index 55c3c59c6..7ec3a0b28 100644
--- a/mix.lock
+++ b/mix.lock
@@ -42,8 +42,8 @@
   "ex_machina": {:hex, :ex_machina, "2.4.0", "09a34c5d371bfb5f78399029194a8ff67aff340ebe8ba19040181af35315eabb", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "a20bc9ddc721b33ea913b93666c5d0bdca5cbad7a67540784ae277228832d72c"},
   "ex_syslogger": {:hex, :ex_syslogger, "1.5.2", "72b6aa2d47a236e999171f2e1ec18698740f40af0bd02c8c650bf5f1fd1bac79", [:mix], [{:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:syslog, "~> 1.1.0", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm", "ab9fab4136dbc62651ec6f16fa4842f10cf02ab4433fa3d0976c01be99398399"},
   "excoveralls": {:hex, :excoveralls, "0.13.1", "b9f1697f7c9e0cfe15d1a1d737fb169c398803ffcbc57e672aa007e9fd42864c", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "b4bb550e045def1b4d531a37fb766cbbe1307f7628bf8f0414168b3f52021cce"},
-  "fast_html": {:hex, :fast_html, "1.0.3", "2cc0d4b68496266a1530e0c852cafeaede0bd10cfdee26fda50dc696c203162f", [:make, :mix], [], "hexpm", "ab3d782b639d3c4655fbaec0f9d032c91f8cab8dd791ac7469c2381bc7c32f85"},
-  "fast_sanitize": {:hex, :fast_sanitize, "0.1.7", "2a7cd8734c88a2de6de55022104f8a3b87f1fdbe8bbf131d9049764b53d50d0d", [:mix], [{:fast_html, "~> 1.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f39fe8ea08fbac17487c30bf09b7d9f3e12472e51fb07a88ffeb8fd17da8ab67"},
+  "fast_html": {:hex, :fast_html, "2.0.1", "e126c74d287768ae78c48938da6711164517300d108a78f8a38993df8d588335", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "bdd6f8525c95ad391a4f10d9a1b3da4cea94078ec8638487aa8c24015ad9393a"},
+  "fast_sanitize": {:hex, :fast_sanitize, "0.2.0", "004b40d5bbecda182b6fdba762a51fffd3501e689e8eafe196e1a97eb0caf733", [:mix], [{:fast_html, "~> 2.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "11fcb37f26d272a3a2aff861872bf100be4eeacea69505908b8cdbcea5b0813a"},
   "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"},
   "floki": {:hex, :floki, "0.27.0", "6b29a14283f1e2e8fad824bc930eaa9477c462022075df6bea8f0ad811c13599", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "583b8c13697c37179f1f82443bcc7ad2f76fbc0bf4c186606eebd658f7f2631b"},
   "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"},
@@ -76,6 +76,7 @@
   "mox": {:hex, :mox, "0.5.2", "55a0a5ba9ccc671518d068c8dddd20eeb436909ea79d1799e2209df7eaa98b6c", [:mix], [], "hexpm", "df4310628cd628ee181df93f50ddfd07be3e5ecc30232d3b6aadf30bdfe6092b"},
   "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]},
   "nimble_parsec": {:hex, :nimble_parsec, "0.6.0", "32111b3bf39137144abd7ba1cce0914533b2d16ef35e8abc5ec8be6122944263", [:mix], [], "hexpm", "27eac315a94909d4dc68bc07a4a83e06c8379237c5ea528a9acff4ca1c873c52"},
+  "nimble_pool": {:hex, :nimble_pool, "0.1.0", "ffa9d5be27eee2b00b0c634eb649aa27f97b39186fec3c493716c2a33e784ec6", [:mix], [], "hexpm", "343a1eaa620ddcf3430a83f39f2af499fe2370390d4f785cd475b4df5acaf3f9"},
   "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]},
   "oban": {:hex, :oban, "2.0.0", "e6ce70d94dd46815ec0882a1ffb7356df9a9d5b8a40a64ce5c2536617a447379", [:mix], [{:ecto_sql, ">= 3.4.3", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cf574813bd048b98a698aa587c21367d2e06842d4e1b1993dcd6a696e9e633bd"},
   "open_api_spex": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", "f296ac0924ba3cf79c7a588c4c252889df4c2edd", [ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"]},

From ebb30128af653d146091fa2317418103fd1e46a3 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 7 Aug 2020 16:20:13 +0200
Subject: [PATCH 41/57] Changelog: Add information about the object age policy

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 572f9e84b..e2a855bef 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [unreleased]
 
 ### Changed
+- **Breaking:** Added the ObjectAgePolicy to the default set of MRFs. This will delist and strip the follower collection of any message received that is older than 7 days. This will stop users from seeing very old messages in the timelines. The messages can still be viewed on the user's page and in conversations. They also still trigger notifications.
 - **Breaking:** Elixir >=1.9 is now required (was >= 1.8)
 - **Breaking:** Configuration: `:auto_linker, :opts` moved to `:pleroma, Pleroma.Formatter`. Old config namespace is deprecated.
 - In Conversations, return only direct messages as `last_status`

From 6ddea8ebe8794a9626f3907a5d0e0db9604bf949 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 7 Aug 2020 09:42:10 -0500
Subject: [PATCH 42/57] Add a note about the proper value for uid

---
 docs/configuration/cheatsheet.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index f23cf4fe4..d9115a958 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -890,6 +890,9 @@ Pleroma account will be created with the same name as the LDAP user name.
 * `base`: LDAP base, e.g. "dc=example,dc=com"
 * `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base"
 
+Note, if your LDAP server is an Active Directory server the correct value is commonly `uid: "cn"`, but if you use an
+OpenLDAP server the value may be `uid: "uid"`.
+
 ### OAuth consumer mode
 
 OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.).

From cb376c4c4ca6491f2b58a9a916986998312640f5 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 7 Aug 2020 17:33:59 +0300
Subject: [PATCH 43/57] CI: install cmake since fast_html now requires it

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c9ab84892..66813c814 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,6 +22,7 @@ stages:
   - docker
 
 before_script:
+  - apt-get update && apt-get install -y cmake
   - mix local.hex --force
   - mix local.rebar --force
 

From 60fe0a08f0ed4b83847995f4e0a5ff10dcf9d336 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 7 Aug 2020 17:59:55 +0200
Subject: [PATCH 44/57] Docs: Remove wrong / confusing auth docs.

---
 docs/configuration/cheatsheet.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index f23cf4fe4..89036ded0 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -858,9 +858,6 @@ Warning: it's discouraged to use this feature because of the associated security
 
 ### :auth
 
-* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator.
-* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication.
-
 Authentication / authorization settings.
 
 * `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`.

From 673e8e3ac154c4ce5801077234cf2bdee99e78c9 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Fri, 7 Aug 2020 13:02:39 -0500
Subject: [PATCH 45/57] Force 204 responses to be empty, fixes #2029

---
 lib/pleroma/web/controller_helper.ex                   |  6 ++++++
 test/support/conn_case.ex                              |  9 ++++++++-
 .../controllers/admin_api_controller_test.exs          | 10 +++++-----
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex
index 69946fb81..6445966e0 100644
--- a/lib/pleroma/web/controller_helper.ex
+++ b/lib/pleroma/web/controller_helper.ex
@@ -18,6 +18,12 @@ defmodule Pleroma.Web.ControllerHelper do
 
   def truthy_param?(value), do: not falsy_param?(value)
 
+  def json_response(conn, status, _) when status in [204, :no_content] do
+    conn
+    |> put_resp_header("content-type", "application/json")
+    |> send_resp(status, "")
+  end
+
   def json_response(conn, status, json) do
     conn
     |> put_status(status)
diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex
index b23918dd1..b50ff1bcc 100644
--- a/test/support/conn_case.ex
+++ b/test/support/conn_case.ex
@@ -56,6 +56,13 @@ defmodule Pleroma.Web.ConnCase do
         [conn: conn]
       end
 
+      defp empty_json_response(conn) do
+        body = response(conn, 204)
+        _ = response_content_type(conn, :json)
+
+        body
+      end
+
       defp json_response_and_validate_schema(
              %{
                private: %{
@@ -79,7 +86,7 @@ defmodule Pleroma.Web.ConnCase do
         end
 
         schema = lookup[op_id].responses[status].content[content_type].schema
-        json = json_response(conn, status)
+        json = if status == 204, do: empty_json_response(conn), else: json_response(conn, status)
 
         case OpenApiSpex.cast_value(json, schema, spec) do
           {:ok, _data} ->
diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs
index b5d5bd8c7..e63268831 100644
--- a/test/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/web/admin_api/controllers/admin_api_controller_test.exs
@@ -439,7 +439,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
       user1: user1,
       user2: user2
     } do
-      assert json_response(conn, :no_content)
+      assert empty_json_response(conn)
       assert User.get_cached_by_id(user1.id).tags == ["x", "foo", "bar"]
       assert User.get_cached_by_id(user2.id).tags == ["y", "foo", "bar"]
 
@@ -457,7 +457,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
     end
 
     test "it does not modify tags of not specified users", %{conn: conn, user3: user3} do
-      assert json_response(conn, :no_content)
+      assert empty_json_response(conn)
       assert User.get_cached_by_id(user3.id).tags == ["unchanged"]
     end
   end
@@ -485,7 +485,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
       user1: user1,
       user2: user2
     } do
-      assert json_response(conn, :no_content)
+      assert empty_json_response(conn)
       assert User.get_cached_by_id(user1.id).tags == []
       assert User.get_cached_by_id(user2.id).tags == ["y"]
 
@@ -503,7 +503,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
     end
 
     test "it does not modify tags of not specified users", %{conn: conn, user3: user3} do
-      assert json_response(conn, :no_content)
+      assert empty_json_response(conn)
       assert User.get_cached_by_id(user3.id).tags == ["unchanged"]
     end
   end
@@ -1756,7 +1756,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
       conn =
         patch(conn, "/api/pleroma/admin/users/force_password_reset", %{nicknames: [user.nickname]})
 
-      assert json_response(conn, 204) == ""
+      assert empty_json_response(conn) == ""
 
       ObanHelpers.perform_all()
 

From 881fdb3a97740425555f4f8b9ddb75123ace73b7 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 7 Aug 2020 21:25:16 +0300
Subject: [PATCH 46/57] Add security policy for Pleroma backend

Closes #1848
---
 SECURITY.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 SECURITY.md

diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 000000000..c212a2505
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,16 @@
+# Pleroma backend security policy
+
+## Supported versions
+
+Currently, Pleroma offers bugfixes and security patches only for the latest minor release.
+
+| Version | Support 
+|---------| --------
+| 2.0     | Bugfixes and security patches
+
+## Reporting a vulnerability
+
+Please use confidential issues (tick the "This issue is confidential and should only be visible to team members with at least Reporter access." box when submitting) at our [bugtracker](https://git.pleroma.social/pleroma/pleroma/-/issues/new) for reporting vulnerabilities.
+## Announcements
+
+New releases are announced at [pleroma.social](https://pleroma.social/announcements/). All security releases are tagged with ["Security"](https://pleroma.social/announcements/tags/security/). You can be notified of them by subscribing to an Atom feed at <https://pleroma.social/announcements/tags/security/feed.xml>. 

From 474147a67a89f8bd92186dbda93d78d8e2045d52 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 7 Aug 2020 14:54:14 -0500
Subject: [PATCH 47/57] Make a new function instead of overloading
 register_changeset/3

---
 lib/pleroma/user.ex                        | 2 +-
 lib/pleroma/web/auth/ldap_authenticator.ex | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 69b0e1781..d1436a688 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -639,7 +639,7 @@ defmodule Pleroma.User do
   def force_password_reset(user), do: update_password_reset_pending(user, true)
 
   # Used to auto-register LDAP accounts which won't have a password hash stored locally
-  def register_changeset(struct, params = %{password: password})
+  def register_changeset_ldap(struct, params = %{password: password})
       when is_nil(password) do
     params = Map.put_new(params, :accepts_chat_messages, true)
 
diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex
index b1645a359..402ab428b 100644
--- a/lib/pleroma/web/auth/ldap_authenticator.ex
+++ b/lib/pleroma/web/auth/ldap_authenticator.ex
@@ -116,7 +116,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
             _ -> params
           end
 
-        changeset = User.register_changeset(%User{}, params)
+        changeset = User.register_changeset_ldap(%User{}, params)
 
         case User.register(changeset) do
           {:ok, user} -> user

From e5557bf8ba6a56996ba8847a522042a748dc046b Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 8 Aug 2020 16:29:40 +0400
Subject: [PATCH 48/57] Add mix task to add expiration to all local statuses

---
 docs/administration/CLI_tasks/database.md | 12 ++++++-
 lib/mix/tasks/pleroma/database.ex         | 24 +++++++++++---
 lib/pleroma/activity_expiration.ex        | 12 ++++---
 test/tasks/database_test.exs              | 39 +++++++++++++++++++++++
 4 files changed, 77 insertions(+), 10 deletions(-)

diff --git a/docs/administration/CLI_tasks/database.md b/docs/administration/CLI_tasks/database.md
index 647f6f274..64dd66c0c 100644
--- a/docs/administration/CLI_tasks/database.md
+++ b/docs/administration/CLI_tasks/database.md
@@ -97,4 +97,14 @@ but should only be run if necessary. **It is safe to cancel this.**
 
 ```sh tab="From Source"
 mix pleroma.database vacuum full
-```
\ No newline at end of file
+```
+
+## Add expiration to all local statuses
+
+```sh tab="OTP"
+./bin/pleroma_ctl database ensure_expiration
+```
+
+```sh tab="From Source"
+mix pleroma.database ensure_expiration
+```
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index 82e2abdcb..d57e59b11 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -10,6 +10,7 @@ defmodule Mix.Tasks.Pleroma.Database do
   alias Pleroma.User
   require Logger
   require Pleroma.Constants
+  import Ecto.Query
   import Mix.Pleroma
   use Mix.Task
 
@@ -53,8 +54,6 @@ defmodule Mix.Tasks.Pleroma.Database do
   end
 
   def run(["prune_objects" | args]) do
-    import Ecto.Query
-
     {options, [], []} =
       OptionParser.parse(
         args,
@@ -94,8 +93,6 @@ defmodule Mix.Tasks.Pleroma.Database do
   end
 
   def run(["fix_likes_collections"]) do
-    import Ecto.Query
-
     start_pleroma()
 
     from(object in Object,
@@ -130,4 +127,23 @@ defmodule Mix.Tasks.Pleroma.Database do
 
     Maintenance.vacuum(args)
   end
+
+  def run(["ensure_expiration"]) do
+    start_pleroma()
+    days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365)
+
+    Pleroma.Activity
+    |> join(:left, [a], u in assoc(a, :expiration))
+    |> where(local: true)
+    |> where([a, u], is_nil(u))
+    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Stream.each(fn activities ->
+      Enum.each(activities, fn activity ->
+        expires_at = Timex.shift(activity.inserted_at, days: days)
+
+        Pleroma.ActivityExpiration.create(activity, expires_at, false)
+      end)
+    end)
+    |> Stream.run()
+  end
 end
diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex
index db9c88d84..7cc9668b3 100644
--- a/lib/pleroma/activity_expiration.ex
+++ b/lib/pleroma/activity_expiration.ex
@@ -20,11 +20,11 @@ defmodule Pleroma.ActivityExpiration do
     field(:scheduled_at, :naive_datetime)
   end
 
-  def changeset(%ActivityExpiration{} = expiration, attrs) do
+  def changeset(%ActivityExpiration{} = expiration, attrs, validate_scheduled_at) do
     expiration
     |> cast(attrs, [:scheduled_at])
     |> validate_required([:scheduled_at])
-    |> validate_scheduled_at()
+    |> validate_scheduled_at(validate_scheduled_at)
   end
 
   def get_by_activity_id(activity_id) do
@@ -33,9 +33,9 @@ defmodule Pleroma.ActivityExpiration do
     |> Repo.one()
   end
 
-  def create(%Activity{} = activity, scheduled_at) do
+  def create(%Activity{} = activity, scheduled_at, validate_scheduled_at \\ true) do
     %ActivityExpiration{activity_id: activity.id}
-    |> changeset(%{scheduled_at: scheduled_at})
+    |> changeset(%{scheduled_at: scheduled_at}, validate_scheduled_at)
     |> Repo.insert()
   end
 
@@ -49,7 +49,9 @@ defmodule Pleroma.ActivityExpiration do
     |> Repo.all()
   end
 
-  def validate_scheduled_at(changeset) do
+  def validate_scheduled_at(changeset, false), do: changeset
+
+  def validate_scheduled_at(changeset, true) do
     validate_change(changeset, :scheduled_at, fn _, scheduled_at ->
       if not expires_late_enough?(scheduled_at) do
         [scheduled_at: "an ephemeral activity must live for at least one hour"]
diff --git a/test/tasks/database_test.exs b/test/tasks/database_test.exs
index 883828d77..3a28aa133 100644
--- a/test/tasks/database_test.exs
+++ b/test/tasks/database_test.exs
@@ -127,4 +127,43 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do
       assert Enum.empty?(Object.get_by_id(object2.id).data["likes"])
     end
   end
+
+  describe "ensure_expiration" do
+    test "it adds to expiration old statuses" do
+      %{id: activity_id1} = insert(:note_activity)
+
+      %{id: activity_id2} =
+        insert(:note_activity, %{inserted_at: NaiveDateTime.from_iso8601!("2015-01-23 23:50:07")})
+
+      %{id: activity_id3} = activity3 = insert(:note_activity)
+
+      expires_at =
+        NaiveDateTime.utc_now()
+        |> NaiveDateTime.add(60 * 61, :second)
+        |> NaiveDateTime.truncate(:second)
+
+      Pleroma.ActivityExpiration.create(activity3, expires_at)
+
+      Mix.Tasks.Pleroma.Database.run(["ensure_expiration"])
+
+      expirations =
+        Pleroma.ActivityExpiration
+        |> order_by(:activity_id)
+        |> Repo.all()
+
+      assert [
+               %Pleroma.ActivityExpiration{
+                 activity_id: ^activity_id1
+               },
+               %Pleroma.ActivityExpiration{
+                 activity_id: ^activity_id2,
+                 scheduled_at: ~N[2016-01-23 23:50:07]
+               },
+               %Pleroma.ActivityExpiration{
+                 activity_id: ^activity_id3,
+                 scheduled_at: ^expires_at
+               }
+             ] = expirations
+    end
+  end
 end

From 66122a11b59a3859f3eb67066148e9c75139d3ee Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 10 Aug 2020 10:33:05 +0200
Subject: [PATCH 49/57] AccountController: Build the correct update activity.

Will fix federation issues.
---
 .../web/mastodon_api/controllers/account_controller.ex        | 2 +-
 .../account_controller/update_credentials_test.exs            | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index f45678184..95d8452df 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -226,7 +226,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
     with changeset <- User.update_changeset(user, user_params),
          {:ok, unpersisted_user} <- Ecto.Changeset.apply_action(changeset, :update),
          updated_object <-
-           Pleroma.Web.ActivityPub.UserView.render("user.json", user: user)
+           Pleroma.Web.ActivityPub.UserView.render("user.json", user: unpersisted_user)
            |> Map.delete("@context"),
          {:ok, update_data, []} <- Builder.update(user, updated_object),
          {:ok, _update, _} <-
diff --git a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
index b888e4c71..2e6704726 100644
--- a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
+++ b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
@@ -214,6 +214,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do
 
       assert user_data = json_response_and_validate_schema(conn, 200)
       assert user_data["display_name"] == "markorepairs"
+
+      update_activity = Repo.one(Pleroma.Activity)
+      assert update_activity.data["type"] == "Update"
+      assert update_activity.data["object"]["name"] == "markorepairs"
     end
 
     test "updates the user's avatar", %{user: user, conn: conn} do

From a818587467feb1f5d5ad1f9499e61dcd7184e864 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 7 Aug 2020 22:05:17 +0300
Subject: [PATCH 50/57] 20200802170532_fix_legacy_tags: Select only fields the
 migration needs

Selecting the full struct will break as soon as a new field is added.
---
 priv/repo/migrations/20200802170532_fix_legacy_tags.exs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/priv/repo/migrations/20200802170532_fix_legacy_tags.exs b/priv/repo/migrations/20200802170532_fix_legacy_tags.exs
index f7274b44e..a84b5d0f6 100644
--- a/priv/repo/migrations/20200802170532_fix_legacy_tags.exs
+++ b/priv/repo/migrations/20200802170532_fix_legacy_tags.exs
@@ -18,7 +18,10 @@ defmodule Pleroma.Repo.Migrations.FixLegacyTags do
   def change do
     legacy_tags = Map.keys(@old_new_map)
 
-    from(u in User, where: fragment("? && ?", u.tags, ^legacy_tags))
+    from(u in User,
+      where: fragment("? && ?", u.tags, ^legacy_tags),
+      select: struct(u, [:tags, :id])
+    )
     |> Repo.all()
     |> Enum.each(fn user ->
       fix_tags_changeset(user)

From 15fa3b6bd8dcc65b74715c500cd23923251d7fd3 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 7 Aug 2020 22:10:09 +0300
Subject: [PATCH 51/57] 20200802170532_fix_legacy_tags: chunk the user query

---
 priv/repo/migrations/20200802170532_fix_legacy_tags.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/priv/repo/migrations/20200802170532_fix_legacy_tags.exs b/priv/repo/migrations/20200802170532_fix_legacy_tags.exs
index a84b5d0f6..ca82fac42 100644
--- a/priv/repo/migrations/20200802170532_fix_legacy_tags.exs
+++ b/priv/repo/migrations/20200802170532_fix_legacy_tags.exs
@@ -22,7 +22,7 @@ defmodule Pleroma.Repo.Migrations.FixLegacyTags do
       where: fragment("? && ?", u.tags, ^legacy_tags),
       select: struct(u, [:tags, :id])
     )
-    |> Repo.all()
+    |> Repo.chunk_stream(100)
     |> Enum.each(fn user ->
       fix_tags_changeset(user)
       |> Repo.update()

From 5c4548d5e74e40e18d8d1ed98ad256568a063370 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 10 Aug 2020 13:05:13 +0000
Subject: [PATCH 52/57] Revert "Merge branch 'issue/1023' into 'develop'"

This reverts merge request !2763
---
 .gitignore                                    |  2 -
 .../CLI_tasks/release_environments.md         |  9 ---
 docs/installation/otp_en.md                   |  7 +-
 installation/init.d/pleroma                   |  1 -
 installation/pleroma.service                  |  2 -
 lib/mix/tasks/pleroma/release_env.ex          | 76 -------------------
 test/tasks/release_env_test.exs               | 30 --------
 7 files changed, 2 insertions(+), 125 deletions(-)
 delete mode 100644 docs/administration/CLI_tasks/release_environments.md
 delete mode 100644 lib/mix/tasks/pleroma/release_env.ex
 delete mode 100644 test/tasks/release_env_test.exs

diff --git a/.gitignore b/.gitignore
index 6ae21e914..599b52b9e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,8 +27,6 @@ erl_crash.dump
 # variables.
 /config/*.secret.exs
 /config/generated_config.exs
-/config/*.env
-
 
 # Database setup file, some may forget to delete it
 /config/setup_db.psql
diff --git a/docs/administration/CLI_tasks/release_environments.md b/docs/administration/CLI_tasks/release_environments.md
deleted file mode 100644
index 36ab43864..000000000
--- a/docs/administration/CLI_tasks/release_environments.md
+++ /dev/null
@@ -1,9 +0,0 @@
-# Generate release environment file
-
-```sh tab="OTP"
- ./bin/pleroma_ctl release_env gen
-```
-
-```sh tab="From Source"
-mix pleroma.release_env gen
-```
diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md
index 338dfa7d0..e4f822d1c 100644
--- a/docs/installation/otp_en.md
+++ b/docs/installation/otp_en.md
@@ -121,9 +121,6 @@ chown -R pleroma /etc/pleroma
 # Run the config generator
 su pleroma -s $SHELL -lc "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql"
 
-# Run the environment file generator.
-su pleroma -s $SHELL -lc "./bin/pleroma_ctl release_env gen"
-
 # Create the postgres database
 su postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql"
 
@@ -134,7 +131,7 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
 # su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
 
 # Start the instance to verify that everything is working as expected
-su pleroma -s $SHELL -lc "export $(cat /opt/pleroma/config/pleroma.env); ./bin/pleroma daemon"
+su pleroma -s $SHELL -lc "./bin/pleroma daemon"
 
 # Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly
 sleep 20 && curl http://localhost:4000/api/v1/instance
@@ -203,7 +200,6 @@ rc-update add pleroma
 # Copy the service into a proper directory
 cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service
 
-
 # Start pleroma and enable it on boot
 systemctl start pleroma
 systemctl enable pleroma
@@ -279,3 +275,4 @@ This will create an account withe the username of 'joeuser' with the email addre
 ## Questions
 
 Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
+
diff --git a/installation/init.d/pleroma b/installation/init.d/pleroma
index e908cda1b..384536f7e 100755
--- a/installation/init.d/pleroma
+++ b/installation/init.d/pleroma
@@ -8,7 +8,6 @@ pidfile="/var/run/pleroma.pid"
 directory=/opt/pleroma
 healthcheck_delay=60
 healthcheck_timer=30
-export $(cat /opt/pleroma/config/pleroma.env)
 
 : ${pleroma_port:-4000}
 
diff --git a/installation/pleroma.service b/installation/pleroma.service
index ee00a3b7a..5dcbc1387 100644
--- a/installation/pleroma.service
+++ b/installation/pleroma.service
@@ -17,8 +17,6 @@ Environment="MIX_ENV=prod"
 Environment="HOME=/var/lib/pleroma"
 ; Path to the folder containing the Pleroma installation.
 WorkingDirectory=/opt/pleroma
-; Path to the environment file. the file contains RELEASE_COOKIE and etc 
-EnvironmentFile=/opt/pleroma/config/pleroma.env
 ; Path to the Mix binary.
 ExecStart=/usr/bin/mix phx.server
 
diff --git a/lib/mix/tasks/pleroma/release_env.ex b/lib/mix/tasks/pleroma/release_env.ex
deleted file mode 100644
index 9da74ffcf..000000000
--- a/lib/mix/tasks/pleroma/release_env.ex
+++ /dev/null
@@ -1,76 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Mix.Tasks.Pleroma.ReleaseEnv do
-  use Mix.Task
-  import Mix.Pleroma
-
-  @shortdoc "Generate Pleroma environment file."
-  @moduledoc File.read!("docs/administration/CLI_tasks/release_environments.md")
-
-  def run(["gen" | rest]) do
-    {options, [], []} =
-      OptionParser.parse(
-        rest,
-        strict: [
-          force: :boolean,
-          path: :string
-        ],
-        aliases: [
-          p: :path,
-          f: :force
-        ]
-      )
-
-    file_path =
-      get_option(
-        options,
-        :path,
-        "Environment file path",
-        "./config/pleroma.env"
-      )
-
-    env_path = Path.expand(file_path)
-
-    proceed? =
-      if File.exists?(env_path) do
-        get_option(
-          options,
-          :force,
-          "Environment file already exists. Do you want to overwrite the #{env_path} file? (y/n)",
-          "n"
-        ) === "y"
-      else
-        true
-      end
-
-    if proceed? do
-      case do_generate(env_path) do
-        {:error, reason} ->
-          shell_error(
-            File.Error.message(%{action: "write to file", reason: reason, path: env_path})
-          )
-
-        _ ->
-          shell_info("\nThe file generated: #{env_path}.\n")
-
-          shell_info("""
-          WARNING: before start pleroma app please make sure to make the file read-only and non-modifiable.
-            Example:
-              chmod 0444 #{file_path}
-              chattr +i #{file_path}
-          """)
-      end
-    else
-      shell_info("\nThe file is exist. #{env_path}.\n")
-    end
-  end
-
-  def do_generate(path) do
-    content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}"
-
-    File.mkdir_p!(Path.dirname(path))
-    File.write(path, content)
-  end
-end
diff --git a/test/tasks/release_env_test.exs b/test/tasks/release_env_test.exs
deleted file mode 100644
index 519f1eba9..000000000
--- a/test/tasks/release_env_test.exs
+++ /dev/null
@@ -1,30 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Mix.Tasks.Pleroma.ReleaseEnvTest do
-  use ExUnit.Case
-  import ExUnit.CaptureIO, only: [capture_io: 1]
-
-  @path "config/pleroma.test.env"
-
-  def do_clean do
-    if File.exists?(@path) do
-      File.rm_rf(@path)
-    end
-  end
-
-  setup do
-    do_clean()
-    on_exit(fn -> do_clean() end)
-    :ok
-  end
-
-  test "generate pleroma.env" do
-    assert capture_io(fn ->
-             Mix.Tasks.Pleroma.ReleaseEnv.run(["gen", "--path", @path, "--force"])
-           end) =~ "The file generated"
-
-    assert File.read!(@path) =~ "RELEASE_COOKIE="
-  end
-end

From 4d76c0ec8b24d7bc91a9e432a0cf3be17c6c10b5 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 10 Aug 2020 15:12:45 +0200
Subject: [PATCH 53/57] Docs: Add cmake dependency

---
 docs/installation/debian_based_en.md | 3 ++-
 docs/installation/debian_based_jp.md | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md
index 8ae5044b5..60c2f47e5 100644
--- a/docs/installation/debian_based_en.md
+++ b/docs/installation/debian_based_en.md
@@ -12,6 +12,7 @@ This guide will assume you are on Debian Stretch. This guide should also work wi
 * `erlang-nox`
 * `git`
 * `build-essential`
+* `cmake`
 
 #### Optional packages used in this guide
 
@@ -30,7 +31,7 @@ sudo apt full-upgrade
 * Install some of the above mentioned programs:
 
 ```shell
-sudo apt install git build-essential postgresql postgresql-contrib
+sudo apt install git build-essential postgresql postgresql-contrib cmake
 ```
 
 ### Install Elixir and Erlang
diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md
index 42e91cda7..c2dd840d3 100644
--- a/docs/installation/debian_based_jp.md
+++ b/docs/installation/debian_based_jp.md
@@ -16,6 +16,7 @@
 - `erlang-nox`
 - `git`
 - `build-essential`
+- `cmake`
 
 #### このガイドで利用している追加パッケージ
 
@@ -32,7 +33,7 @@ sudo apt full-upgrade
 
 * 上記に挙げたパッケージをインストールしておきます。
 ```
-sudo apt install git build-essential postgresql postgresql-contrib
+sudo apt install git build-essential postgresql postgresql-contrib cmake
 ```
 
 

From a2f2ba3fbbc9788b16e7d62044756b99fa0c45e1 Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Mon, 10 Aug 2020 16:24:45 +0300
Subject: [PATCH 54/57] docs: add cmake to other installation guides

---
 docs/installation/alpine_linux_en.md | 3 ++-
 docs/installation/arch_linux_en.md   | 3 ++-
 docs/installation/gentoo_en.md       | 3 ++-
 docs/installation/netbsd_en.md       | 1 +
 docs/installation/openbsd_en.md      | 3 ++-
 docs/installation/openbsd_fi.md      | 2 +-
 6 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/docs/installation/alpine_linux_en.md b/docs/installation/alpine_linux_en.md
index c726d559f..a5683f18c 100644
--- a/docs/installation/alpine_linux_en.md
+++ b/docs/installation/alpine_linux_en.md
@@ -14,6 +14,7 @@ It assumes that you have administrative rights, either as root or a user with [s
 * `erlang-xmerl`
 * `git`
 * Development Tools
+* `cmake`
 
 #### Optional packages used in this guide
 
@@ -39,7 +40,7 @@ sudo apk upgrade
 * Install some tools, which are needed later:
 
 ```shell
-sudo apk add git build-base
+sudo apk add git build-base cmake
 ```
 
 ### Install Elixir and Erlang
diff --git a/docs/installation/arch_linux_en.md b/docs/installation/arch_linux_en.md
index bf9cfb488..7fb69dd60 100644
--- a/docs/installation/arch_linux_en.md
+++ b/docs/installation/arch_linux_en.md
@@ -9,6 +9,7 @@ This guide will assume that you have administrative rights, either as root or a
 * `elixir`
 * `git`
 * `base-devel`
+* `cmake`
 
 #### Optional packages used in this guide
 
@@ -26,7 +27,7 @@ sudo pacman -Syu
 * Install some of the above mentioned programs:
 
 ```shell
-sudo pacman -S git base-devel elixir
+sudo pacman -S git base-devel elixir cmake
 ```
 
 ### Install PostgreSQL
diff --git a/docs/installation/gentoo_en.md b/docs/installation/gentoo_en.md
index 32152aea7..5a676380c 100644
--- a/docs/installation/gentoo_en.md
+++ b/docs/installation/gentoo_en.md
@@ -28,6 +28,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * `dev-db/postgresql`
 * `dev-lang/elixir`
 * `dev-vcs/git`
+* `dev-util/cmake`
 
 #### Optional ebuilds used in this guide
 
@@ -46,7 +47,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * Emerge all required the required and suggested software in one go:
 
 ```shell
- # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx
+ # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake
 ```
 
 If you would not like to install the optional packages, remove them from this line. 
diff --git a/docs/installation/netbsd_en.md b/docs/installation/netbsd_en.md
index 3626acc69..6ad0de2f6 100644
--- a/docs/installation/netbsd_en.md
+++ b/docs/installation/netbsd_en.md
@@ -19,6 +19,7 @@ databases/postgresql11-client
 databases/postgresql11-server
 devel/git-base
 devel/git-docs
+devel/cmake
 lang/elixir
 security/acmesh
 security/sudo
diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md
index 5dbe24f75..eee452845 100644
--- a/docs/installation/openbsd_en.md
+++ b/docs/installation/openbsd_en.md
@@ -14,11 +14,12 @@ The following packages need to be installed:
   * git
   * postgresql-server
   * postgresql-contrib
+  * cmake
 
 To install them, run the following command (with doas or as root):
 
 ```
-pkg_add elixir gmake ImageMagick git postgresql-server postgresql-contrib
+pkg_add elixir gmake ImageMagick git postgresql-server postgresql-contrib cmake
 ```
 
 Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt.
diff --git a/docs/installation/openbsd_fi.md b/docs/installation/openbsd_fi.md
index 272273cff..b5b5056a9 100644
--- a/docs/installation/openbsd_fi.md
+++ b/docs/installation/openbsd_fi.md
@@ -16,7 +16,7 @@ Matrix-kanava #freenode_#pleroma:matrix.org ovat hyviä paikkoja löytää apua
 
 Asenna tarvittava ohjelmisto:
 
-`# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3`
+`# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3 cmake`
 
 Luo postgresql-tietokanta:
 

From 11fc90744c89097969a94d2accaa8f97cb1bbd7d Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 10 Aug 2020 15:31:36 +0200
Subject: [PATCH 55/57] Transmogrifier: Remove duplicate code.

---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 7381d4476..2f04cc6ff 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -644,16 +644,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     end
   end
 
-  def handle_incoming(
-        %{"type" => "Create", "object" => %{"type" => "ChatMessage"}} = data,
-        _options
-      ) do
-    with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
-         {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
-      {:ok, activity}
-    end
-  end
-
   def handle_incoming(%{"type" => type} = data, _options)
       when type in ~w{Like EmojiReact Announce} do
     with :ok <- ObjectValidator.fetch_actor_and_object(data),

From 249f21dcbbbc20f401a9ea9bfc6179c858abce6f Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 10 Aug 2020 17:57:36 +0400
Subject: [PATCH 56/57] Admin API: Filter out unapproved users when the
 `active` filter is on

---
 lib/pleroma/user/query.ex                     |  1 +
 .../controllers/admin_api_controller_test.exs | 21 +++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex
index 45553cb6c..d618432ff 100644
--- a/lib/pleroma/user/query.ex
+++ b/lib/pleroma/user/query.ex
@@ -130,6 +130,7 @@ defmodule Pleroma.User.Query do
   defp compose_query({:active, _}, query) do
     User.restrict_deactivated(query)
     |> where([u], not is_nil(u.nickname))
+    |> where([u], u.approval_pending == false)
   end
 
   defp compose_query({:legacy_active, _}, query) do
diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs
index b5d5bd8c7..7f0f02605 100644
--- a/test/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/web/admin_api/controllers/admin_api_controller_test.exs
@@ -1164,6 +1164,27 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
              }
     end
 
+    test "`active` filters out users pending approval", %{token: token} do
+      insert(:user, approval_pending: true)
+      %{id: user_id} = insert(:user, approval_pending: false)
+      %{id: admin_id} = token.user
+
+      conn =
+        build_conn()
+        |> assign(:user, token.user)
+        |> assign(:token, token)
+        |> get("/api/pleroma/admin/users?filters=active")
+
+      assert %{
+               "count" => 2,
+               "page_size" => 50,
+               "users" => [
+                 %{"id" => ^admin_id},
+                 %{"id" => ^user_id}
+               ]
+             } = json_response(conn, 200)
+    end
+
     test "it works with multiple filters" do
       admin = insert(:user, nickname: "john", is_admin: true)
       token = insert(:oauth_admin_token, user: admin)

From 9a9121805ceccfa62c77e9abc81af5f7c7fd4049 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 11 Aug 2020 09:08:27 +0000
Subject: [PATCH 57/57] Apply 1 suggestion(s) to 1 file(s)

---
 test/support/conn_case.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex
index b50ff1bcc..7ef681258 100644
--- a/test/support/conn_case.ex
+++ b/test/support/conn_case.ex
@@ -58,7 +58,7 @@ defmodule Pleroma.Web.ConnCase do
 
       defp empty_json_response(conn) do
         body = response(conn, 204)
-        _ = response_content_type(conn, :json)
+        response_content_type(conn, :json)
 
         body
       end