Anon/pleroma
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'fix/federation-context-issues' into 'develop'

Browse Source 
Fix reply context fixing (Pleroma replies to Misskey threads) and removal of context objects

See merge request pleroma/pleroma!3717
This commit is contained in:
tusooa 2022-09-04 18:43:36 +00:00
commit 20347898e2
33 changed files with 585 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/config.exs
View File

@ -673,6 +673,8 @@ config :pleroma, :features, improved_hashtag_timeline: :auto
config :pleroma, :populate_hashtags_table, fault_rate_allowance: 0.01
config :pleroma, :delete_context_objects, fault_rate_allowance: 0.01
config :pleroma, :env, Mix.env()
config :http_signatures,

21
config/description.exs
View File

@ -495,6 +495,27 @@ config :pleroma, :config_description, [
}
]
},
%{
group: :pleroma,
key: :delete_context_objects,
type: :group,
description: "`delete_context_objects` background migration settings",
children: [
%{
key: :fault_rate_allowance,
type: :float,
description:
"Max accepted rate of objects that failed in the migration. Any value from 0.0 which tolerates no errors to 1.0 which will enable the feature even if context object deletion failed for all records.",
suggestions: [0.01]
},
%{
key: :sleep_interval_ms,
type: :integer,
description:
"Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances)."
}
]
},
%{
group: :pleroma,
key: :instance,

3
lib/pleroma/application.ex
View File

@ -249,7 +249,8 @@ defmodule Pleroma.Application do
defp background_migrators do
[
Pleroma.Migrators.HashtagsTableMigrator
Pleroma.Migrators.HashtagsTableMigrator,
Pleroma.Migrators.ContextObjectsDeletionMigrator
]
end

1
lib/pleroma/data_migration.ex
View File

@ -42,4 +42,5 @@ defmodule Pleroma.DataMigration do
end
def populate_hashtags_table, do: get_by_name("populate_hashtags_table")
def delete_context_objects, do: get_by_name("delete_context_objects")
end

139
lib/pleroma/migrators/context_objects_deletion_migrator.ex Normal file
View File

@ -0,0 +1,139 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Migrators.ContextObjectsDeletionMigrator do
defmodule State do
use Pleroma.Migrators.Support.BaseMigratorState
@impl Pleroma.Migrators.Support.BaseMigratorState
defdelegate data_migration(), to: Pleroma.DataMigration, as: :delete_context_objects
end
use Pleroma.Migrators.Support.BaseMigrator
alias Pleroma.Migrators.Support.BaseMigrator
alias Pleroma.Object
@doc "This migration removes objects created exclusively for contexts, containing only an `id` field."
@impl BaseMigrator
def feature_config_path, do: [:features, :delete_context_objects]
@impl BaseMigrator
def fault_rate_allowance, do: Config.get([:delete_context_objects, :fault_rate_allowance], 0)
@impl BaseMigrator
def perform do
data_migration_id = data_migration_id()
max_processed_id = get_stat(:max_processed_id, 0)
Logger.info("Deleting context objects from `objects` (from oid: #{max_processed_id})...")
query()
|> where([object], object.id > ^max_processed_id)
|> Repo.chunk_stream(100, :batches, timeout: :infinity)
|> Stream.each(fn objects ->
object_ids = Enum.map(objects, & &1.id)
results = Enum.map(object_ids, &delete_context_object(&1))
failed_ids =
results
|> Enum.filter(&(elem(&1, 0) == :error))
|> Enum.map(&elem(&1, 1))
chunk_affected_count =
results
|> Enum.filter(&(elem(&1, 0) == :ok))
|> length()
for failed_id <- failed_ids do
_ =
Repo.query(
"INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <>
"VALUES ($1, $2) ON CONFLICT DO NOTHING;",
[data_migration_id, failed_id]
)
end
_ =
Repo.query(
"DELETE FROM data_migration_failed_ids " <>
"WHERE data_migration_id = $1 AND record_id = ANY($2)",
[data_migration_id, object_ids -- failed_ids]
)
max_object_id = Enum.at(object_ids, -1)
put_stat(:max_processed_id, max_object_id)
increment_stat(:iteration_processed_count, length(object_ids))
increment_stat(:processed_count, length(object_ids))
increment_stat(:failed_count, length(failed_ids))
increment_stat(:affected_count, chunk_affected_count)
put_stat(:records_per_second, records_per_second())
persist_state()
# A quick and dirty approach to controlling the load this background migration imposes
sleep_interval = Config.get([:delete_context_objects, :sleep_interval_ms], 0)
Process.sleep(sleep_interval)
end)
|> Stream.run()
end
@impl BaseMigrator
def query do
# Context objects have no activity type, and only one field, `id`.
# Only those context objects are without types.
from(
object in Object,
where: fragment("(?)->'type' IS NULL", object.data),
select: %{
id: object.id
}
)
end
@spec delete_context_object(integer()) :: {:ok | :error, integer()}
defp delete_context_object(id) do
result =
%Object{id: id}
|> Repo.delete()
|> elem(0)
{result, id}
end
@impl BaseMigrator
def retry_failed do
data_migration_id = data_migration_id()
failed_objects_query()
|> Repo.chunk_stream(100, :one)
|> Stream.each(fn object ->
with {res, _} when res != :error <- delete_context_object(object.id) do
_ =
Repo.query(
"DELETE FROM data_migration_failed_ids " <>
"WHERE data_migration_id = $1 AND record_id = $2",
[data_migration_id, object.id]
)
end
end)
|> Stream.run()
put_stat(:failed_count, failures_count())
persist_state()
force_continue()
end
defp failed_objects_query do
from(o in Object)
|> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"),
on: dmf.record_id == o.id
)
|> where([_o, dmf], dmf.data_migration_id == ^data_migration_id())
|> order_by([o], asc: o.id)
end
end

4
lib/pleroma/object.ex
View File

@ -207,10 +207,6 @@ defmodule Pleroma.Object do
end
end
def context_mapping(context) do
Object.change(%Object{}, %{data: %{"id" => context}})
end
def make_tombstone(%Object{data: %{"id" => id, "type" => type}}, deleted \\ DateTime.utc_now()) do
%ObjectTombstone{
id: id,

2
lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
View File

@ -97,7 +97,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Article", "Note", "Page"])
|> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
|> validate_required([:id, :actor, :attributedTo, :type, :context])
|> CommonValidations.validate_any_presence([:cc, :to])
|> CommonValidations.validate_fields_match([:actor, :attributedTo])
|> CommonValidations.validate_actor_presence()

2
lib/pleroma/web/activity_pub/object_validators/common_fields.ex
View File

@ -52,8 +52,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do
field(:summary, :string)
field(:context, :string)
# short identifier for PleromaFE to group statuses by context
field(:context_id, :integer)
field(:sensitive, :boolean, default: false)
field(:replies_count, :integer, default: 0)

7
lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
View File

@ -22,14 +22,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
end
def fix_object_defaults(data) do
%{data: %{"id" => context}, id: context_id} =
Utils.create_context(data["context"] || data["conversation"])
context =
Utils.maybe_create_context(
data["context"] || data["conversation"] || data["inReplyTo"] || data["id"]
)
%User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"])
data
|> Map.put("context", context)
|> Map.put("context_id", context_id)
|> cast_and_filter_recipients("to", follower_collection)
|> cast_and_filter_recipients("cc", follower_collection)
|> cast_and_filter_recipients("bto", follower_collection)

2
lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
View File

@ -75,7 +75,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
data
|> CommonFixes.fix_actor()
|> Map.put_new("context", object["context"])
|> Map.put("context", object["context"])
|> fix_addressing(object)
end

2
lib/pleroma/web/activity_pub/object_validators/event_validator.ex
View File

@ -62,7 +62,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Event"])
|> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
|> validate_required([:id, :actor, :attributedTo, :type, :context])
|> CommonValidations.validate_any_presence([:cc, :to])
|> CommonValidations.validate_fields_match([:actor, :attributedTo])
|> CommonValidations.validate_actor_presence()

2
lib/pleroma/web/activity_pub/object_validators/question_validator.ex
View File

@ -80,7 +80,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Question"])
|> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
|> validate_required([:id, :actor, :attributedTo, :type, :context])
|> CommonValidations.validate_any_presence([:cc, :to])
|> CommonValidations.validate_fields_match([:actor, :attributedTo])
|> CommonValidations.validate_actor_presence()

23
lib/pleroma/web/activity_pub/utils.ex
View File

@ -154,22 +154,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
Notification.get_notified_from_activity(%Activity{data: object}, false)
end
def create_context(context) do
context = context || generate_id("contexts")
# Ecto has problems accessing the constraint inside the jsonb,
# so we explicitly check for the existed object before insert
object = Object.get_cached_by_ap_id(context)
with true <- is_nil(object),
changeset <- Object.context_mapping(context),
{:ok, inserted_object} <- Repo.insert(changeset) do
inserted_object
else
_ ->
object
end
end
def maybe_create_context(context), do: context || generate_id("contexts")
@doc """
Enqueues an activity for federation if it's local
@ -201,18 +186,16 @@ defmodule Pleroma.Web.ActivityPub.Utils do
|> Map.put_new("id", "pleroma:fakeid")
|> Map.put_new_lazy("published", &make_date/0)
|> Map.put_new("context", "pleroma:fakecontext")
|> Map.put_new("context_id", -1)
|> lazy_put_object_defaults(true)
end
def lazy_put_activity_defaults(map, _fake?) do
%{data: %{"id" => context}, id: context_id} = create_context(map["context"])
context = maybe_create_context(map["context"])
map
|> Map.put_new_lazy("id", &generate_activity_id/0)
|> Map.put_new_lazy("published", &make_date/0)
|> Map.put_new("context", context)
|> Map.put_new("context_id", context_id)
|> lazy_put_object_defaults(false)
end
@ -226,7 +209,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do
|> Map.put_new("id", "pleroma:fake_object_id")
|> Map.put_new_lazy("published", &make_date/0)
|> Map.put_new("context", activity["context"])
|> Map.put_new("context_id", activity["context_id"])
|> Map.put_new("fake", true)
%{activity | "object" => object}
@ -239,7 +221,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do
|> Map.put_new_lazy("id", &generate_object_id/0)
|> Map.put_new_lazy("published", &make_date/0)
|> Map.put_new("context", activity["context"])
|> Map.put_new("context_id", activity["context_id"])
%{activity | "object" => object}
end

9
lib/pleroma/web/api_spec/schemas/status.ex
View File

@ -148,9 +148,15 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
description:
"A map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`"
},
context: %Schema{
type: :string,
description: "The thread identifier the status is associated with"
},
conversation_id: %Schema{
type: :integer,
description: "The ID of the AP context the status is associated with (if any)"
deprecated: true,
description:
"The ID of the AP context the status is associated with (if any); deprecated, please use `context` instead"
},
direct_conversation_id: %Schema{
type: :integer,
@ -325,6 +331,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
"pinned" => false,
"pleroma" => %{
"content" => %{"text/plain" => "foobar"},
"context" => "http://localhost:4001/objects/8b4c0c80-6a37-4d2a-b1b9-05a19e3875aa",
"conversation_id" => 345_972,
"direct_conversation_id" => nil,
"emoji_reactions" => [],

29
lib/pleroma/web/common_api/utils.ex
View File

@ -453,35 +453,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do
def get_report_statuses(_, _), do: {:ok, nil}
# DEPRECATED mostly, context objects are now created at insertion time.
def context_to_conversation_id(context) do
with %Object{id: id} <- Object.get_cached_by_ap_id(context) do
id
else
_e ->
changeset = Object.context_mapping(context)
case Repo.insert(changeset) do
{:ok, %{id: id}} ->
id
# This should be solved by an upsert, but it seems ecto
# has problems accessing the constraint inside the jsonb.
{:error, _} ->
Object.get_cached_by_ap_id(context).id
end
end
end
def conversation_id_to_context(id) do
with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do
context
else
_e ->
{:error, dgettext("errors", "No such conversation")}
end
end
def validate_character_limit("" = _full_payload, [] = _attachments) do
{:error, dgettext("errors", "Cannot post an empty status without attachments")}
end

17
lib/pleroma/web/mastodon_api/views/status_view.ex
View File

@ -57,11 +57,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
end)
end
defp get_context_id(%{data: %{"context_id" => context_id}}) when not is_nil(context_id),
do: context_id
# DEPRECATED This field seems to be a left-over from the StatusNet era.
# If your application uses `pleroma.conversation_id`: this field is deprecated.
# It is currently stubbed instead by doing a CRC32 of the context, and
# clearing the MSB to avoid overflow exceptions with signed integers on the
# different clients using this field (Java/Kotlin code, mostly; see Husky.)
# This should be removed in a future version of Pleroma. Pleroma-FE currently
# depends on this field, as well.
defp get_context_id(%{data: %{"context" => context}}) when is_binary(context) do
use Bitwise
defp get_context_id(%{data: %{"context" => context}}) when is_binary(context),
do: Utils.context_to_conversation_id(context)
:erlang.crc32(context)
|> band(bnot(0x8000_0000))
end
defp get_context_id(_), do: nil
@ -388,6 +396,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
pleroma: %{
local: activity.local,
conversation_id: get_context_id(activity),
context: object.data["context"],
in_reply_to_account_acct: reply_to_user && reply_to_user.nickname,
content: %{"text/plain" => content_plaintext},
spoiler_text: %{"text/plain" => summary},

18
priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs Normal file
View File

@ -0,0 +1,18 @@
defmodule Pleroma.Repo.Migrations.DataMigrationDeleteContextObjects do
use Ecto.Migration
require Logger
def up do
dt = NaiveDateTime.utc_now()
execute(
"INSERT INTO data_migrations(name, inserted_at, updated_at) " <>
"VALUES ('delete_context_objects', '#{dt}', '#{dt}') ON CONFLICT DO NOTHING;"
)
end
def down do
execute("DELETE FROM data_migrations WHERE name = 'delete_context_objects';")
end
end

61
test/fixtures/create-pleroma-reply-to-misskey-thread.json vendored Normal file
View File

@ -0,0 +1,61 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://p.helene.moe/schemas/litepub-0.1.jsonld",
{
"@language": "und"
}
],
"actor": "https://p.helene.moe/users/helene",
"attachment": [],
"attributedTo": "https://p.helene.moe/users/helene",
"cc": [
"https://p.helene.moe/users/helene/followers"
],
"context": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
"conversation": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
"directMessage": false,
"id": "https://p.helene.moe/activities/5f80db86-a9bb-4883-9845-fbdbd1478f3a",
"object": {
"actor": "https://p.helene.moe/users/helene",
"attachment": [],
"attributedTo": "https://p.helene.moe/users/helene",
"cc": [
"https://p.helene.moe/users/helene/followers"
],
"content": "<span class=\"h-card\"><a class=\"u-url mention\" data-user=\"AHntpQ4T3J4OSnpgMC\" href=\"https://mk.absturztau.be/@mametsuko\" rel=\"ugc\">@<span>mametsuko</span></a></span> meow",
"context": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
"conversation": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
"id": "https://p.helene.moe/objects/fd5910ac-d9dc-412e-8d1d-914b203296c4",
"inReplyTo": "https://mk.absturztau.be/notes/93e7nm8wqg",
"published": "2022-08-02T13:46:58.403996Z",
"sensitive": null,
"source": "@mametsuko@mk.absturztau.be meow",
"summary": "",
"tag": [
{
"href": "https://mk.absturztau.be/users/8ozbzjs3o8",
"name": "@mametsuko@mk.absturztau.be",
"type": "Mention"
}
],
"to": [
"https://mk.absturztau.be/users/8ozbzjs3o8",
"https://www.w3.org/ns/activitystreams#Public"
],
"type": "Note"
},
"published": "2022-08-02T13:46:58.403883Z",
"tag": [
{
"href": "https://mk.absturztau.be/users/8ozbzjs3o8",
"name": "@mametsuko@mk.absturztau.be",
"type": "Mention"
}
],
"to": [
"https://mk.absturztau.be/users/8ozbzjs3o8",
"https://www.w3.org/ns/activitystreams#Public"
],
"type": "Create"
}

50
test/fixtures/tesla_mock/helene@p.helene.moe.json vendored Normal file
View File

@ -0,0 +1,50 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://p.helene.moe/schemas/litepub-0.1.jsonld",
{
"@language": "und"
}
],
"alsoKnownAs": [],
"attachment": [
{
"name": "Timezone",
"type": "PropertyValue",
"value": "UTC+2 (Paris/Berlin)"
}
],
"capabilities": {
"acceptsChatMessages": true
},
"discoverable": true,
"endpoints": {
"oauthAuthorizationEndpoint": "https://p.helene.moe/oauth/authorize",
"oauthRegistrationEndpoint": "https://p.helene.moe/api/v1/apps",
"oauthTokenEndpoint": "https://p.helene.moe/oauth/token",
"sharedInbox": "https://p.helene.moe/inbox",
"uploadMedia": "https://p.helene.moe/api/ap/upload_media"
},
"featured": "https://p.helene.moe/users/helene/collections/featured",
"followers": "https://p.helene.moe/users/helene/followers",
"following": "https://p.helene.moe/users/helene/following",
"icon": {
"type": "Image",
"url": "https://p.helene.moe/media/9a39209daa5a66b7ebb0547b08bf8360aa9d8d65a4ffba2603c6ffbe6aecb432.jpg"
},
"id": "https://p.helene.moe/users/helene",
"inbox": "https://p.helene.moe/users/helene/inbox",
"manuallyApprovesFollowers": false,
"name": "Hélène",
"outbox": "https://p.helene.moe/users/helene/outbox",
"preferredUsername": "helene",
"publicKey": {
"id": "https://p.helene.moe/users/helene#main-key",
"owner": "https://p.helene.moe/users/helene",
"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtoSBPU/VS2Kx3f6ap3zv\nZVacJsgUfaoFb3c2ii/FRh9RmRVlarq8sJXcjsQt1e0oxWaWJaIDDwyKZPt6hXae\nrY/AiGGeNu+NA+BtY7l7+9Yu67HUyT62+1qAwYHKBXX3fLOPs/YmQI0Tt0c4wKAG\nKEkiYsRizghgpzUC6jqdKV71DJkUZ8yhckCGb2fLko1ajbWEssdaP51aLsyRMyC2\nuzeWrxtD4O/HG0ea4S6y5X6hnsAHIK4Y3nnyIQ6pn4tOsl3HgqkjXE9MmZSvMCFx\nBq89TfZrVXNa2gSZdZLdbbJstzEScQWNt1p6tA6rM+e4JXYGr+rMdF3G+jV7afI2\nFQIDAQAB\n-----END PUBLIC KEY-----\n\n"
},
"summary": "I can speak: Français, English, Deutsch (nicht sehr gut), 日本語 (not very well)",
"tag": [],
"type": "Person",
"url": "https://p.helene.moe/users/helene"
}

65
test/fixtures/tesla_mock/mametsuko@mk.absturztau.be.json vendored Normal file
View File

@ -0,0 +1,65 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
{
"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
"sensitive": "as:sensitive",
"Hashtag": "as:Hashtag",
"quoteUrl": "as:quoteUrl",
"toot": "http://joinmastodon.org/ns#",
"Emoji": "toot:Emoji",
"featured": "toot:featured",
"discoverable": "toot:discoverable",
"schema": "http://schema.org#",
"PropertyValue": "schema:PropertyValue",
"value": "schema:value",
"misskey": "https://misskey-hub.net/ns#",
"_misskey_content": "misskey:_misskey_content",
"_misskey_quote": "misskey:_misskey_quote",
"_misskey_reaction": "misskey:_misskey_reaction",
"_misskey_votes": "misskey:_misskey_votes",
"_misskey_talk": "misskey:_misskey_talk",
"isCat": "misskey:isCat",
"vcard": "http://www.w3.org/2006/vcard/ns#"
}
],
"type": "Person",
"id": "https://mk.absturztau.be/users/8ozbzjs3o8",
"inbox": "https://mk.absturztau.be/users/8ozbzjs3o8/inbox",
"outbox": "https://mk.absturztau.be/users/8ozbzjs3o8/outbox",
"followers": "https://mk.absturztau.be/users/8ozbzjs3o8/followers",
"following": "https://mk.absturztau.be/users/8ozbzjs3o8/following",
"featured": "https://mk.absturztau.be/users/8ozbzjs3o8/collections/featured",
"sharedInbox": "https://mk.absturztau.be/inbox",
"endpoints": {
"sharedInbox": "https://mk.absturztau.be/inbox"
},
"url": "https://mk.absturztau.be/@mametsuko",
"preferredUsername": "mametsuko",
"name": "mametschko",
"summary": "<p><span>nya, ich bin eine Brotperson</span></p>",
"icon": {
"type": "Image",
"url": "https://mk.absturztau.be/files/webpublic-3b5594f4-fa52-4548-b4e3-c379ae2143ed",
"sensitive": false,
"name": null
},
"image": {
"type": "Image",
"url": "https://mk.absturztau.be/files/webpublic-0d03b03d-b14b-4916-ac3d-8a137118ec84",
"sensitive": false,
"name": null
},
"tag": [],
"manuallyApprovesFollowers": true,
"discoverable": false,
"publicKey": {
"id": "https://mk.absturztau.be/users/8ozbzjs3o8#main-key",
"type": "Key",
"owner": "https://mk.absturztau.be/users/8ozbzjs3o8",
"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAuN/S1spBGmh8FXI1Bt16\nXB7Cc0QutBp7UPgmDNHjOfsq0zrF4g3L1UBxvrpU0XX77XPMCd9yPvGwAYURH2mv\ntIcYuE+R90VLDmBu5MTVthcG2D874eCZ2rD2YsEYmN5AjTX7QBIqCck+qDhVWkkM\nEZ6S5Ht6IJ5Of74eKffXElQI/C6QB+9uEDOmPk0jCzgI5gw7xvJqFj/DIF4kUUAu\nA89JqaFZzZlkrSrj4cr48bLN/YOmpdaHu0BKHaDSHct4+MqlixqovgdB6RboCEDw\ne4Aeav7+Q0Y9oGIvuggg0Q+nCubnVNnaPyzd817tpPVzyZmTts+DKyDuv90SX3nR\nsPaNa5Ty60eqplUk4b7X1gSvuzBJUFBxTVV84WnjwoeoydaS6rSyjCDPGLBjaByc\nFyWMMEb/zlQyhLZfBlvT7k96wRSsMszh2hDALWmgYIhq/jNwINvALJ1GKLNHHKZ4\nyz2LnxVpRm2rWrZzbvtcnSQOt3LaPSZn8Wgwv4buyHF02iuVuIamZVtKexsE1Ixl\nIi9qa3AKEc5gOzYXhRhvHaruzoCehUbb/UHC5c8Tto8L5G1xYzjLP3qj3PT9w/wM\n+k1Ra/4JhuAnVFROOoOmx9rIELLHH7juY2nhM7plGhyt1M5gysgqEloij8QzyQU2\nZK1YlAERG2XFO6br8omhcmECAwEAAQ==\n-----END PUBLIC KEY-----\n"
},
"isCat": true,
"vcard:Address": "Vienna, Austria"
}

1
test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json vendored Normal file
View File

@ -0,0 +1 @@
{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","sensitive":"as:sensitive","Hashtag":"as:Hashtag","quoteUrl":"as:quoteUrl","toot":"http://joinmastodon.org/ns#","Emoji":"toot:Emoji","featured":"toot:featured","discoverable":"toot:discoverable","schema":"http://schema.org#","PropertyValue":"schema:PropertyValue","value":"schema:value","misskey":"https://misskey-hub.net/ns#","_misskey_content":"misskey:_misskey_content","_misskey_quote":"misskey:_misskey_quote","_misskey_reaction":"misskey:_misskey_reaction","_misskey_votes":"misskey:_misskey_votes","_misskey_talk":"misskey:_misskey_talk","isCat":"misskey:isCat","vcard":"http://www.w3.org/2006/vcard/ns#"}],"id":"https://mk.absturztau.be/notes/93e7nm8wqg/activity","actor":"https://mk.absturztau.be/users/8ozbzjs3o8","type":"Create","published":"2022-08-01T11:06:49.568Z","object":{"id":"https://mk.absturztau.be/notes/93e7nm8wqg","type":"Note","attributedTo":"https://mk.absturztau.be/users/8ozbzjs3o8","summary":null,"content":"<p><span>meow</span></p>","_misskey_content":"meow","published":"2022-08-01T11:06:49.568Z","to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://mk.absturztau.be/users/8ozbzjs3o8/followers"],"inReplyTo":null,"attachment":[],"sensitive":false,"tag":[]},"to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://mk.absturztau.be/users/8ozbzjs3o8/followers"]}

44
test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg.json vendored Normal file
View File

@ -0,0 +1,44 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
{
"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
"sensitive": "as:sensitive",
"Hashtag": "as:Hashtag",
"quoteUrl": "as:quoteUrl",
"toot": "http://joinmastodon.org/ns#",
"Emoji": "toot:Emoji",
"featured": "toot:featured",
"discoverable": "toot:discoverable",
"schema": "http://schema.org#",
"PropertyValue": "schema:PropertyValue",
"value": "schema:value",
"misskey": "https://misskey-hub.net/ns#",
"_misskey_content": "misskey:_misskey_content",
"_misskey_quote": "misskey:_misskey_quote",
"_misskey_reaction": "misskey:_misskey_reaction",
"_misskey_votes": "misskey:_misskey_votes",
"_misskey_talk": "misskey:_misskey_talk",
"isCat": "misskey:isCat",
"vcard": "http://www.w3.org/2006/vcard/ns#"
}
],
"id": "https://mk.absturztau.be/notes/93e7nm8wqg",
"type": "Note",
"attributedTo": "https://mk.absturztau.be/users/8ozbzjs3o8",
"summary": null,
"content": "<p><span>meow</span></p>",
"_misskey_content": "meow",
"published": "2022-08-01T11:06:49.568Z",
"to": [
"https://www.w3.org/ns/activitystreams#Public"
],
"cc": [
"https://mk.absturztau.be/users/8ozbzjs3o8/followers"
],
"inReplyTo": null,
"attachment": [],
"sensitive": false,
"tag": []
}

36
test/fixtures/tesla_mock/p.helene.moe-AM7S6vZQmL6pI9TgPY.json vendored Normal file
View File

@ -0,0 +1,36 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://p.helene.moe/schemas/litepub-0.1.jsonld",
{
"@language": "und"
}
],
"actor": "https://p.helene.moe/users/helene",
"attachment": [],
"attributedTo": "https://p.helene.moe/users/helene",
"cc": [
"https://p.helene.moe/users/helene/followers"
],
"content": "<span class=\"h-card\"><a class=\"u-url mention\" data-user=\"AHntpQ4T3J4OSnpgMC\" href=\"https://mk.absturztau.be/@mametsuko\" rel=\"ugc\">@<span>mametsuko</span></a></span> meow",
"context": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
"conversation": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
"id": "https://p.helene.moe/objects/fd5910ac-d9dc-412e-8d1d-914b203296c4",
"inReplyTo": "https://mk.absturztau.be/notes/93e7nm8wqg",
"published": "2022-08-02T13:46:58.403996Z",
"sensitive": null,
"source": "@mametsuko@mk.absturztau.be meow",
"summary": "",
"tag": [
{
"href": "https://mk.absturztau.be/users/8ozbzjs3o8",
"name": "@mametsuko@mk.absturztau.be",
"type": "Mention"
}
],
"to": [
"https://mk.absturztau.be/users/8ozbzjs3o8",
"https://www.w3.org/ns/activitystreams#Public"
],
"type": "Note"
}

5
test/pleroma/web/activity_pub/activity_pub_test.exs
View File

@ -554,7 +554,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
assert activity.data["ok"] == data["ok"]
assert activity.data["id"] == given_id
assert activity.data["context"] == "blabla"
assert activity.data["context_id"]
end
test "adds a context when none is there" do
@ -576,8 +575,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
assert is_binary(activity.data["context"])
assert is_binary(object.data["context"])
assert activity.data["context_id"]
assert object.data["context_id"]
end
test "adds an id to a given object if it lacks one and is a note and inserts it to the object database" do
@ -1612,7 +1609,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
})
assert Repo.aggregate(Activity, :count, :id) == 1
assert Repo.aggregate(Object, :count, :id) == 2
assert Repo.aggregate(Object, :count, :id) == 1
assert Repo.aggregate(Notification, :count, :id) == 0
end
end

9
test/pleroma/web/activity_pub/object_validators/create_generic_validator_test.exs
View File

@ -23,10 +23,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidatorTest do
{:ok, object_data} = ObjectValidator.cast_and_apply(note_activity["object"])
meta = [object_data: ObjectValidator.stringify_keys(object_data)]
%{valid?: true} = CreateGenericValidator.cast_and_validate(note_activity, meta)
assert %{valid?: true} = CreateGenericValidator.cast_and_validate(note_activity, meta)
end
test "a Create/Note with mismatched context is invalid" do
test "a Create/Note with mismatched context uses the Note's context" do
user = insert(:user)
note = %{
@ -54,6 +54,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidatorTest do
{:ok, object_data} = ObjectValidator.cast_and_apply(note_activity["object"])
meta = [object_data: ObjectValidator.stringify_keys(object_data)]
%{valid?: false} = CreateGenericValidator.cast_and_validate(note_activity, meta)
validated = CreateGenericValidator.cast_and_validate(note_activity, meta)
assert validated.valid?
assert {:context, note["context"]} in validated.changes
end
end

38
test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs
View File

@ -707,4 +707,42 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
}
]
end
test "the standalone note uses its own ID when context is missing" do
insert(:user, ap_id: "https://mk.absturztau.be/users/8ozbzjs3o8")
activity =
"test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json"
|> File.read!()
|> Jason.decode!()
{:ok, %Activity{} = modified} = Transmogrifier.handle_incoming(activity)
object = Object.normalize(modified, fetch: false)
assert object.data["context"] == object.data["id"]
assert modified.data["context"] == object.data["id"]
end
test "the reply note uses its parent's ID when context is missing and reply is unreachable" do
insert(:user, ap_id: "https://mk.absturztau.be/users/8ozbzjs3o8")
activity =
"test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json"
|> File.read!()
|> Jason.decode!()
object =
activity["object"]
|> Map.put("inReplyTo", "https://404.site/object/went-to-buy-milk")
activity =
activity
|> Map.put("object", object)
{:ok, %Activity{} = modified} = Transmogrifier.handle_incoming(activity)
object = Object.normalize(modified, fetch: false)
assert object.data["context"] == object.data["inReplyTo"]
assert modified.data["context"] == object.data["inReplyTo"]
end
end

3
test/pleroma/web/activity_pub/transmogrifier/question_handling_test.exs
View File

@ -33,8 +33,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do
assert object.data["context"] ==
"tag:mastodon.sdf.org,2019-05-10:objectId=15095122:objectType=Conversation"
assert object.data["context_id"]
assert object.data["anyOf"] == []
assert Enum.sort(object.data["oneOf"]) ==
@ -68,7 +66,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do
reply_object = Object.normalize(reply_activity, fetch: false)
assert reply_object.data["context"] == object.data["context"]
assert reply_object.data["context_id"] == object.data["context_id"]
end
test "Mastodon Question activity with HTML tags in plaintext" do

17
test/pleroma/web/activity_pub/transmogrifier_test.exs
View File

@ -108,15 +108,20 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
assert activity.data["type"] == "Move"
end
test "a reply with mismatched context is rejected" do
insert(:user, ap_id: "https://macgirvin.com/channel/mike")
test "it fixes both the Create and object contexts in a reply" do
insert(:user, ap_id: "https://mk.absturztau.be/users/8ozbzjs3o8")
insert(:user, ap_id: "https://p.helene.moe/users/helene")
note_activity =
"test/fixtures/roadhouse-create-activity.json"
create_activity =
"test/fixtures/create-pleroma-reply-to-misskey-thread.json"
|> File.read!()
|> Jason.decode!()
assert {:error, _} = Transmogrifier.handle_incoming(note_activity)
assert {:ok, %Activity{} = activity} = Transmogrifier.handle_incoming(create_activity)
object = Object.normalize(activity, fetch: false)
assert activity.data["context"] == object.data["context"]
end
end
@ -227,7 +232,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
assert is_nil(modified["object"]["like_count"])
assert is_nil(modified["object"]["announcements"])
assert is_nil(modified["object"]["announcement_count"])
assert is_nil(modified["object"]["context_id"])
assert is_nil(modified["object"]["generator"])
end
@ -242,7 +246,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
assert is_nil(modified["object"]["like_count"])
assert is_nil(modified["object"]["announcements"])
assert is_nil(modified["object"]["announcement_count"])
assert is_nil(modified["object"]["context_id"])
assert is_nil(modified["object"]["likes"])
end

4
test/pleroma/web/activity_pub/utils_test.exs
View File

@ -429,7 +429,6 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do
object = Object.normalize(note_activity, fetch: false)
res = Utils.lazy_put_activity_defaults(%{"context" => object.data["id"]})
assert res["context"] == object.data["id"]
assert res["context_id"] == object.id
assert res["id"]
assert res["published"]
end
@ -437,7 +436,6 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do
test "returns map with fake id and published data" do
assert %{
"context" => "pleroma:fakecontext",
"context_id" => -1,
"id" => "pleroma:fakeid",
"published" => _
} = Utils.lazy_put_activity_defaults(%{}, true)
@ -454,13 +452,11 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do
})
assert res["context"] == object.data["id"]
assert res["context_id"] == object.id
assert res["id"]
assert res["published"]
assert res["object"]["id"]
assert res["object"]["published"]
assert res["object"]["context"] == object.data["id"]
assert res["object"]["context_id"] == object.id
end
end

28
test/pleroma/web/common_api/utils_test.exs
View File

@ -4,7 +4,6 @@
defmodule Pleroma.Web.CommonAPI.UtilsTest do
alias Pleroma.Builders.UserBuilder
alias Pleroma.Object
alias Pleroma.Web.CommonAPI
alias Pleroma.Web.CommonAPI.ActivityDraft
alias Pleroma.Web.CommonAPI.Utils
@ -273,22 +272,6 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
end
end
describe "context_to_conversation_id" do
test "creates a mapping object" do
conversation_id = Utils.context_to_conversation_id("random context")
object = Object.get_by_ap_id("random context")
assert conversation_id == object.id
end
test "returns an existing mapping for an existing object" do
{:ok, object} = Object.context_mapping("random context") |> Repo.insert()
conversation_id = Utils.context_to_conversation_id("random context")
assert conversation_id == object.id
end
end
describe "formats date to asctime" do
test "when date is in ISO 8601 format" do
date = DateTime.utc_now() |> DateTime.to_iso8601()
@ -517,17 +500,6 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
end
end
describe "conversation_id_to_context/1" do
test "returns id" do
object = insert(:note)
assert Utils.conversation_id_to_context(object.id) == object.data["id"]
end
test "returns error if object not found" do
assert Utils.conversation_id_to_context("123") == {:error, "No such conversation"}
end
end
describe "maybe_notify_mentioned_recipients/2" do
test "returns recipients when activity is not `Create`" do
activity = insert(:like_activity)

2
test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
View File

@ -262,6 +262,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
|> Map.put("url", nil)
|> Map.put("uri", nil)
|> Map.put("created_at", nil)
|> Kernel.put_in(["pleroma", "context"], nil)
|> Kernel.put_in(["pleroma", "conversation_id"], nil)
fake_conn =
@ -285,6 +286,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
|> Map.put("url", nil)
|> Map.put("uri", nil)
|> Map.put("created_at", nil)
|> Kernel.put_in(["pleroma", "context"], nil)
|> Kernel.put_in(["pleroma", "conversation_id"], nil)
assert real_status == fake_status

6
test/pleroma/web/mastodon_api/views/status_view_test.exs
View File

@ -14,10 +14,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
alias Pleroma.User
alias Pleroma.UserRelationship
alias Pleroma.Web.CommonAPI
alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.MastodonAPI.AccountView
alias Pleroma.Web.MastodonAPI.StatusView
require Bitwise
import Pleroma.Factory
import Tesla.Mock
import OpenApiSpex.TestAssertions
@ -226,7 +227,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
object_data = Object.normalize(note, fetch: false).data
user = User.get_cached_by_ap_id(note.data["actor"])
convo_id = Utils.context_to_conversation_id(object_data["context"])
convo_id = :erlang.crc32(object_data["context"]) |> Bitwise.band(Bitwise.bnot(0x8000_0000))
status = StatusView.render("show.json", %{activity: note})
@ -280,6 +281,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
pleroma: %{
local: true,
conversation_id: convo_id,
context: object_data["context"],
in_reply_to_account_acct: nil,
content: %{"text/plain" => HTML.strip_tags(object_data["content"])},
spoiler_text: %{"text/plain" => HTML.strip_tags(object_data["summary"])},

53
test/support/http_request_mock.ex
View File

@ -1084,6 +1084,14 @@ defmodule HttpRequestMock do
}}
end
def get("https://404.site" <> _, _, _, _) do
{:ok,
%Tesla.Env{
status: 404,
body: ""
}}
end
def get(
"https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=acct:lain@zetsubou.xn--q9jyb4c",
_,
@ -1401,6 +1409,51 @@ defmodule HttpRequestMock do
}}
end
def get("https://mk.absturztau.be/users/8ozbzjs3o8", _, _, _) do
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/mametsuko@mk.absturztau.be.json"),
headers: activitypub_object_headers()
}}
end
def get("https://p.helene.moe/users/helene", _, _, _) do
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/helene@p.helene.moe.json"),
headers: activitypub_object_headers()
}}
end
def get("https://mk.absturztau.be/notes/93e7nm8wqg", _, _, _) do
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg.json"),
headers: activitypub_object_headers()
}}
end
def get("https://mk.absturztau.be/notes/93e7nm8wqg/activity", _, _, _) do
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json"),
headers: activitypub_object_headers()
}}
end
def get("https://p.helene.moe/objects/fd5910ac-d9dc-412e-8d1d-914b203296c4", _, _, _) do
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/p.helene.moe-AM7S6vZQmL6pI9TgPY.json"),
headers: activitypub_object_headers()
}}
end
def get(url, query, body, headers) do
{:error,
"Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"}