Object: remove context_id field

30 to 70% of the objects in the object table are simple JSON objects
containing a single field, 'id', being the context's ID. The reason for
the creation of an object per context seems to be an old relic from the
StatusNet era, and has only been used nowadays as an helper for threads
in Pleroma-FE via the `pleroma.conversation_id` field in status views.
An object per context was created, and its numerical ID (table column)
was used and stored as 'context_id' in the object and activity along
with the full 'context' URI/string.

This commit removes this field and stops creation of objects for each
context, which will also allow incoming activities to use activity IDs
as contexts, something which was not possible before, or would have been
very broken under most circumstances.

The `pleroma.conversation_id` field has been reimplemented in a way to
maintain backwards-compatibility by calculating a CRC32 of the full
context URI/string in the object, instead of relying on the row ID for
the created context object.
This commit is contained in:
Hélène 2022-08-06 03:24:31 +02:00
parent 7de21ec991
commit f3e061c964
No known key found for this signature in database
GPG Key ID: A215F2E9F1589D62
14 changed files with 9 additions and 102 deletions

View File

@ -208,10 +208,6 @@ defmodule Pleroma.Object do
end end
end end
def context_mapping(context) do
Object.change(%Object{}, %{data: %{"id" => context}})
end
def make_tombstone(%Object{data: %{"id" => id, "type" => type}}, deleted \\ DateTime.utc_now()) do def make_tombstone(%Object{data: %{"id" => id, "type" => type}}, deleted \\ DateTime.utc_now()) do
%ObjectTombstone{ %ObjectTombstone{
id: id, id: id,

View File

@ -94,7 +94,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
defp validate_data(data_cng) do defp validate_data(data_cng) do
data_cng data_cng
|> validate_inclusion(:type, ["Article", "Note", "Page"]) |> validate_inclusion(:type, ["Article", "Note", "Page"])
|> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) |> validate_required([:id, :actor, :attributedTo, :type, :context])
|> CommonValidations.validate_any_presence([:cc, :to]) |> CommonValidations.validate_any_presence([:cc, :to])
|> CommonValidations.validate_fields_match([:actor, :attributedTo]) |> CommonValidations.validate_fields_match([:actor, :attributedTo])
|> CommonValidations.validate_actor_presence() |> CommonValidations.validate_actor_presence()

View File

@ -22,14 +22,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
end end
def fix_object_defaults(data) do def fix_object_defaults(data) do
%{data: %{"id" => context}, id: context_id} = context = Utils.maybe_create_context(data["context"] || data["conversation"])
Utils.create_context(data["context"] || data["conversation"])
%User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"]) %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"])
data data
|> Map.put("context", context) |> Map.put("context", context)
|> Map.put("context_id", context_id)
|> cast_and_filter_recipients("to", follower_collection) |> cast_and_filter_recipients("to", follower_collection)
|> cast_and_filter_recipients("cc", follower_collection) |> cast_and_filter_recipients("cc", follower_collection)
|> cast_and_filter_recipients("bto", follower_collection) |> cast_and_filter_recipients("bto", follower_collection)

View File

@ -62,7 +62,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
defp validate_data(data_cng) do defp validate_data(data_cng) do
data_cng data_cng
|> validate_inclusion(:type, ["Event"]) |> validate_inclusion(:type, ["Event"])
|> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) |> validate_required([:id, :actor, :attributedTo, :type, :context])
|> CommonValidations.validate_any_presence([:cc, :to]) |> CommonValidations.validate_any_presence([:cc, :to])
|> CommonValidations.validate_fields_match([:actor, :attributedTo]) |> CommonValidations.validate_fields_match([:actor, :attributedTo])
|> CommonValidations.validate_actor_presence() |> CommonValidations.validate_actor_presence()

View File

@ -80,7 +80,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
defp validate_data(data_cng) do defp validate_data(data_cng) do
data_cng data_cng
|> validate_inclusion(:type, ["Question"]) |> validate_inclusion(:type, ["Question"])
|> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) |> validate_required([:id, :actor, :attributedTo, :type, :context])
|> CommonValidations.validate_any_presence([:cc, :to]) |> CommonValidations.validate_any_presence([:cc, :to])
|> CommonValidations.validate_fields_match([:actor, :attributedTo]) |> CommonValidations.validate_fields_match([:actor, :attributedTo])
|> CommonValidations.validate_actor_presence() |> CommonValidations.validate_actor_presence()

View File

@ -154,22 +154,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
Notification.get_notified_from_activity(%Activity{data: object}, false) Notification.get_notified_from_activity(%Activity{data: object}, false)
end end
def create_context(context) do def maybe_create_context(context), do: context || generate_id("contexts")
context = context || generate_id("contexts")
# Ecto has problems accessing the constraint inside the jsonb,
# so we explicitly check for the existed object before insert
object = Object.get_cached_by_ap_id(context)
with true <- is_nil(object),
changeset <- Object.context_mapping(context),
{:ok, inserted_object} <- Repo.insert(changeset) do
inserted_object
else
_ ->
object
end
end
@doc """ @doc """
Enqueues an activity for federation if it's local Enqueues an activity for federation if it's local
@ -201,18 +186,16 @@ defmodule Pleroma.Web.ActivityPub.Utils do
|> Map.put_new("id", "pleroma:fakeid") |> Map.put_new("id", "pleroma:fakeid")
|> Map.put_new_lazy("published", &make_date/0) |> Map.put_new_lazy("published", &make_date/0)
|> Map.put_new("context", "pleroma:fakecontext") |> Map.put_new("context", "pleroma:fakecontext")
|> Map.put_new("context_id", -1)
|> lazy_put_object_defaults(true) |> lazy_put_object_defaults(true)
end end
def lazy_put_activity_defaults(map, _fake?) do def lazy_put_activity_defaults(map, _fake?) do
%{data: %{"id" => context}, id: context_id} = create_context(map["context"]) context = maybe_create_context(map["context"])
map map
|> Map.put_new_lazy("id", &generate_activity_id/0) |> Map.put_new_lazy("id", &generate_activity_id/0)
|> Map.put_new_lazy("published", &make_date/0) |> Map.put_new_lazy("published", &make_date/0)
|> Map.put_new("context", context) |> Map.put_new("context", context)
|> Map.put_new("context_id", context_id)
|> lazy_put_object_defaults(false) |> lazy_put_object_defaults(false)
end end
@ -226,7 +209,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do
|> Map.put_new("id", "pleroma:fake_object_id") |> Map.put_new("id", "pleroma:fake_object_id")
|> Map.put_new_lazy("published", &make_date/0) |> Map.put_new_lazy("published", &make_date/0)
|> Map.put_new("context", activity["context"]) |> Map.put_new("context", activity["context"])
|> Map.put_new("context_id", activity["context_id"])
|> Map.put_new("fake", true) |> Map.put_new("fake", true)
%{activity | "object" => object} %{activity | "object" => object}
@ -239,7 +221,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do
|> Map.put_new_lazy("id", &generate_object_id/0) |> Map.put_new_lazy("id", &generate_object_id/0)
|> Map.put_new_lazy("published", &make_date/0) |> Map.put_new_lazy("published", &make_date/0)
|> Map.put_new("context", activity["context"]) |> Map.put_new("context", activity["context"])
|> Map.put_new("context_id", activity["context_id"])
%{activity | "object" => object} %{activity | "object" => object}
end end

View File

@ -449,35 +449,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do
def get_report_statuses(_, _), do: {:ok, nil} def get_report_statuses(_, _), do: {:ok, nil}
# DEPRECATED mostly, context objects are now created at insertion time.
def context_to_conversation_id(context) do
with %Object{id: id} <- Object.get_cached_by_ap_id(context) do
id
else
_e ->
changeset = Object.context_mapping(context)
case Repo.insert(changeset) do
{:ok, %{id: id}} ->
id
# This should be solved by an upsert, but it seems ecto
# has problems accessing the constraint inside the jsonb.
{:error, _} ->
Object.get_cached_by_ap_id(context).id
end
end
end
def conversation_id_to_context(id) do
with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do
context
else
_e ->
{:error, dgettext("errors", "No such conversation")}
end
end
def validate_character_limit("" = _full_payload, [] = _attachments) do def validate_character_limit("" = _full_payload, [] = _attachments) do
{:error, dgettext("errors", "Cannot post an empty status without attachments")} {:error, dgettext("errors", "Cannot post an empty status without attachments")}
end end

View File

@ -61,7 +61,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
do: context_id do: context_id
defp get_context_id(%{data: %{"context" => context}}) when is_binary(context), defp get_context_id(%{data: %{"context" => context}}) when is_binary(context),
do: Utils.context_to_conversation_id(context) do: :erlang.crc32(context)
defp get_context_id(_), do: nil defp get_context_id(_), do: nil

View File

@ -554,7 +554,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
assert activity.data["ok"] == data["ok"] assert activity.data["ok"] == data["ok"]
assert activity.data["id"] == given_id assert activity.data["id"] == given_id
assert activity.data["context"] == "blabla" assert activity.data["context"] == "blabla"
assert activity.data["context_id"]
end end
test "adds a context when none is there" do test "adds a context when none is there" do
@ -576,8 +575,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
assert is_binary(activity.data["context"]) assert is_binary(activity.data["context"])
assert is_binary(object.data["context"]) assert is_binary(object.data["context"])
assert activity.data["context_id"]
assert object.data["context_id"]
end end
test "adds an id to a given object if it lacks one and is a note and inserts it to the object database" do test "adds an id to a given object if it lacks one and is a note and inserts it to the object database" do
@ -1612,7 +1609,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
}) })
assert Repo.aggregate(Activity, :count, :id) == 1 assert Repo.aggregate(Activity, :count, :id) == 1
assert Repo.aggregate(Object, :count, :id) == 2 assert Repo.aggregate(Object, :count, :id) == 1
assert Repo.aggregate(Notification, :count, :id) == 0 assert Repo.aggregate(Notification, :count, :id) == 0
end end
end end

View File

@ -33,8 +33,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do
assert object.data["context"] == assert object.data["context"] ==
"tag:mastodon.sdf.org,2019-05-10:objectId=15095122:objectType=Conversation" "tag:mastodon.sdf.org,2019-05-10:objectId=15095122:objectType=Conversation"
assert object.data["context_id"]
assert object.data["anyOf"] == [] assert object.data["anyOf"] == []
assert Enum.sort(object.data["oneOf"]) == assert Enum.sort(object.data["oneOf"]) ==
@ -68,7 +66,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do
reply_object = Object.normalize(reply_activity, fetch: false) reply_object = Object.normalize(reply_activity, fetch: false)
assert reply_object.data["context"] == object.data["context"] assert reply_object.data["context"] == object.data["context"]
assert reply_object.data["context_id"] == object.data["context_id"]
end end
test "Mastodon Question activity with HTML tags in plaintext" do test "Mastodon Question activity with HTML tags in plaintext" do

View File

@ -227,7 +227,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
assert is_nil(modified["object"]["like_count"]) assert is_nil(modified["object"]["like_count"])
assert is_nil(modified["object"]["announcements"]) assert is_nil(modified["object"]["announcements"])
assert is_nil(modified["object"]["announcement_count"]) assert is_nil(modified["object"]["announcement_count"])
assert is_nil(modified["object"]["context_id"])
assert is_nil(modified["object"]["generator"]) assert is_nil(modified["object"]["generator"])
end end
@ -242,7 +241,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
assert is_nil(modified["object"]["like_count"]) assert is_nil(modified["object"]["like_count"])
assert is_nil(modified["object"]["announcements"]) assert is_nil(modified["object"]["announcements"])
assert is_nil(modified["object"]["announcement_count"]) assert is_nil(modified["object"]["announcement_count"])
assert is_nil(modified["object"]["context_id"])
assert is_nil(modified["object"]["likes"]) assert is_nil(modified["object"]["likes"])
end end

View File

@ -429,7 +429,6 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do
object = Object.normalize(note_activity, fetch: false) object = Object.normalize(note_activity, fetch: false)
res = Utils.lazy_put_activity_defaults(%{"context" => object.data["id"]}) res = Utils.lazy_put_activity_defaults(%{"context" => object.data["id"]})
assert res["context"] == object.data["id"] assert res["context"] == object.data["id"]
assert res["context_id"] == object.id
assert res["id"] assert res["id"]
assert res["published"] assert res["published"]
end end
@ -437,7 +436,6 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do
test "returns map with fake id and published data" do test "returns map with fake id and published data" do
assert %{ assert %{
"context" => "pleroma:fakecontext", "context" => "pleroma:fakecontext",
"context_id" => -1,
"id" => "pleroma:fakeid", "id" => "pleroma:fakeid",
"published" => _ "published" => _
} = Utils.lazy_put_activity_defaults(%{}, true) } = Utils.lazy_put_activity_defaults(%{}, true)
@ -454,13 +452,11 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do
}) })
assert res["context"] == object.data["id"] assert res["context"] == object.data["id"]
assert res["context_id"] == object.id
assert res["id"] assert res["id"]
assert res["published"] assert res["published"]
assert res["object"]["id"] assert res["object"]["id"]
assert res["object"]["published"] assert res["object"]["published"]
assert res["object"]["context"] == object.data["id"] assert res["object"]["context"] == object.data["id"]
assert res["object"]["context_id"] == object.id
end end
end end

View File

@ -273,22 +273,6 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
end end
end end
describe "context_to_conversation_id" do
test "creates a mapping object" do
conversation_id = Utils.context_to_conversation_id("random context")
object = Object.get_by_ap_id("random context")
assert conversation_id == object.id
end
test "returns an existing mapping for an existing object" do
{:ok, object} = Object.context_mapping("random context") |> Repo.insert()
conversation_id = Utils.context_to_conversation_id("random context")
assert conversation_id == object.id
end
end
describe "formats date to asctime" do describe "formats date to asctime" do
test "when date is in ISO 8601 format" do test "when date is in ISO 8601 format" do
date = DateTime.utc_now() |> DateTime.to_iso8601() date = DateTime.utc_now() |> DateTime.to_iso8601()
@ -517,17 +501,6 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
end end
end end
describe "conversation_id_to_context/1" do
test "returns id" do
object = insert(:note)
assert Utils.conversation_id_to_context(object.id) == object.data["id"]
end
test "returns error if object not found" do
assert Utils.conversation_id_to_context("123") == {:error, "No such conversation"}
end
end
describe "maybe_notify_mentioned_recipients/2" do describe "maybe_notify_mentioned_recipients/2" do
test "returns recipients when activity is not `Create`" do test "returns recipients when activity is not `Create`" do
activity = insert(:like_activity) activity = insert(:like_activity)

View File

@ -226,7 +226,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
object_data = Object.normalize(note, fetch: false).data object_data = Object.normalize(note, fetch: false).data
user = User.get_cached_by_ap_id(note.data["actor"]) user = User.get_cached_by_ap_id(note.data["actor"])
convo_id = Utils.context_to_conversation_id(object_data["context"]) convo_id = :erlang.crc32(object_data["context"])
status = StatusView.render("show.json", %{activity: note}) status = StatusView.render("show.json", %{activity: note})