From d6d5f46baea90e9b8a305a010457ac6d404d8829 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 11 Apr 2019 16:02:38 +0000 Subject: [PATCH 01/64] Update OAuth web template --- lib/pleroma/web/templates/layout/app.html.eex | 45 ++++++++++++--- .../web/templates/o_auth/o_auth/show.html.eex | 57 ++++++++++++++----- 2 files changed, 79 insertions(+), 23 deletions(-) diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index 8333bc921..7d2d609d1 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -63,13 +63,14 @@ .scopes-input { display: flex; + flex-direction: column; margin-top: 1em; text-align: left; color: #89898a; } .scopes-input label:first-child { - flex-basis: 40%; + height: 2em; } .scopes { @@ -80,13 +81,22 @@ } .scope { - flex-basis: 100%; display: flex; + flex-basis: 100%; height: 2em; align-items: center; } + .scope:before { + color: #b9b9ba; + content: "✔"; + margin-left: 1em; + margin-right: 1em; + } + [type="checkbox"] + label { + display: none; + cursor: pointer; margin: 0.5em; } @@ -95,10 +105,12 @@ } [type="checkbox"] + label:before { + cursor: pointer; display: inline-block; color: white; background-color: #121a24; border: 4px solid #121a24; + box-shadow: 0px 0px 1px 0 #d8a070; box-sizing: border-box; width: 1.2em; height: 1.2em; @@ -128,7 +140,8 @@ border-radius: 4px; border: none; padding: 10px; - margin-top: 30px; + margin-top: 20px; + margin-bottom: 20px; text-transform: uppercase; font-size: 16px; box-shadow: 0px 0px 2px 0px black, @@ -147,8 +160,9 @@ box-sizing: border-box; width: 100%; background-color: #931014; + border: 1px solid #a06060; + color: #902020; border-radius: 4px; - border: none; padding: 10px; margin-top: 20px; font-weight: 500; @@ -171,12 +185,27 @@ margin-top: 0 } - .scopes-input { - flex-direction: column; + .scope { + flex-basis: 0%; } - .scope { - flex-basis: 50%; + .scope:before { + content: ""; + margin-left: 0em; + margin-right: 1em; + } + + .scope:first-child:before { + margin-left: 1em; + content: "✔"; + } + + .scope:after { + content: ","; + } + + .scope:last-child:after { + content: ""; } } diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index 87278e636..c7b4ef792 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -6,26 +6,53 @@ <% end %>

OAuth Authorization

- <%= form_for @conn, o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %> -
- <%= label f, :name, "Name or email" %> - <%= text_input f, :name %> -
-
- <%= label f, :password, "Password" %> - <%= password_input f, :password %> -
-<%= render @view_module, "_scopes.html", Map.merge(assigns, %{form: f, scope_param: "authorization[scope][]"}) %> +<%= if @params["registration"] in ["true", true] do %> +

This is the first time you visit! Please enter your Pleroma handle.

+

Choose carefully! You won't be able to change this later. You will be able to change your display name, though.

+

Please only use lowercase letters and no special characters

+
+ <%= label f, :nickname, "Pleroma Handle" %> + <%= text_input f, :nickname, placeholder: "lain" %> +
+ <%= hidden_input f, :name, value: @params["name"] %> + <%= hidden_input f, :password, value: @params["password"] %> +
+ +<% else %> +
+ <%= label f, :name, "Username" %> + <%= text_input f, :name %> +
+
+ <%= label f, :password, "Password" %> + <%= password_input f, :password %> +
+ <%= submit "Log In" %> +
+ <%= label f, :scope, "The following permissions will be granted" %> +
+ <%= for scope <- @available_scopes do %> + <%# Note: using hidden input with `unchecked_value` in order to distinguish user's empty selection from `scope` param being omitted %> + <%= if scope in @scopes do %> +
+ <%= checkbox f, :"scope_#{scope}", value: scope in @scopes && scope, checked_value: scope, unchecked_value: "", name: "authorization[scope][]" %> + <%= label f, :"scope_#{scope}", String.capitalize(scope) %> + <%= if scope in @scopes && scope do %> + <%= String.capitalize(scope) %> + <% end %> +
+ <% else %> + <%= checkbox f, :"scope_#{scope}", value: scope in @scopes && scope, checked_value: scope, unchecked_value: "", name: "authorization[scope][]" %> + <% end %> + <% end %> +
+
+<% end %> <%= hidden_input f, :client_id, value: @client_id %> <%= hidden_input f, :response_type, value: @response_type %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> <%= hidden_input f, :state, value: @state %> -<%= submit "Authorize" %> -<% end %> - -<%= if Pleroma.Config.oauth_consumer_enabled?() do %> - <%= render @view_module, Pleroma.Web.Auth.Authenticator.oauth_consumer_template(), assigns %> <% end %> From 9a98f48ec3f438e543a5a621624401bb22a0d44a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 1 May 2019 12:29:48 -0500 Subject: [PATCH 02/64] Remove incorrect statement about valid characters --- lib/pleroma/web/templates/o_auth/o_auth/show.html.eex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index c7b4ef792..45f2b5cc0 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -11,7 +11,6 @@ <%= if @params["registration"] in ["true", true] do %>

This is the first time you visit! Please enter your Pleroma handle.

Choose carefully! You won't be able to change this later. You will be able to change your display name, though.

-

Please only use lowercase letters and no special characters

<%= label f, :nickname, "Pleroma Handle" %> <%= text_input f, :nickname, placeholder: "lain" %> From 62e42b03abd2cede85e85f62c35f62a8c42e8ea1 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 15 May 2019 20:10:16 +0300 Subject: [PATCH 03/64] Handle incoming Question objects --- .../web/activity_pub/transmogrifier.ex | 2 +- lib/pleroma/web/activity_pub/utils.ex | 2 +- test/fixtures/httpoison_mock/rinpatch.json | 64 ++++++++++++ test/fixtures/mastodon-question-activity.json | 99 +++++++++++++++++++ test/support/http_request_mock.ex | 8 ++ test/web/activity_pub/transmogrifier_test.exs | 17 ++++ 6 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 test/fixtures/httpoison_mock/rinpatch.json create mode 100644 test/fixtures/mastodon-question-activity.json diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 508f3532f..c2596cfec 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -399,7 +399,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # - tags # - emoji def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data) - when objtype in ["Article", "Note", "Video", "Page"] do + when objtype in ["Article", "Note", "Video", "Page", "Question"] do actor = Containment.get_actor(data) data = diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 581b9d1ab..de91fa03f 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do require Logger - @supported_object_types ["Article", "Note", "Video", "Page"] + @supported_object_types ["Article", "Note", "Video", "Page", "Question"] # Some implementations send the actor URI as the actor field, others send the entire actor object, # so figure out what the actor's URI is based on what we have. diff --git a/test/fixtures/httpoison_mock/rinpatch.json b/test/fixtures/httpoison_mock/rinpatch.json new file mode 100644 index 000000000..59311ecb6 --- /dev/null +++ b/test/fixtures/httpoison_mock/rinpatch.json @@ -0,0 +1,64 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "toot": "http://joinmastodon.org/ns#", + "featured": { + "@id": "toot:featured", + "@type": "@id" + }, + "alsoKnownAs": { + "@id": "as:alsoKnownAs", + "@type": "@id" + }, + "movedTo": { + "@id": "as:movedTo", + "@type": "@id" + }, + "schema": "http://schema.org#", + "PropertyValue": "schema:PropertyValue", + "value": "schema:value", + "Hashtag": "as:Hashtag", + "Emoji": "toot:Emoji", + "IdentityProof": "toot:IdentityProof", + "focalPoint": { + "@container": "@list", + "@id": "toot:focalPoint" + } + } + ], + "id": "https://mastodon.sdf.org/users/rinpatch", + "type": "Person", + "following": "https://mastodon.sdf.org/users/rinpatch/following", + "followers": "https://mastodon.sdf.org/users/rinpatch/followers", + "inbox": "https://mastodon.sdf.org/users/rinpatch/inbox", + "outbox": "https://mastodon.sdf.org/users/rinpatch/outbox", + "featured": "https://mastodon.sdf.org/users/rinpatch/collections/featured", + "preferredUsername": "rinpatch", + "name": "rinpatch", + "summary": "

umu

", + "url": "https://mastodon.sdf.org/@rinpatch", + "manuallyApprovesFollowers": false, + "publicKey": { + "id": "https://mastodon.sdf.org/users/rinpatch#main-key", + "owner": "https://mastodon.sdf.org/users/rinpatch", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1vbhYKDopb5xzfJB2TZY\n0ZvgxqdAhbSKKkQC5Q2b0ofhvueDy2AuZTnVk1/BbHNlqKlwhJUSpA6LiTZVvtcc\nMn6cmSaJJEg30gRF5GARP8FMcuq8e2jmceiW99NnUX17MQXsddSf2JFUwD0rUE8H\nBsgD7UzE9+zlA/PJOTBO7fvBEz9PTQ3r4sRMTJVFvKz2MU/U+aRNTuexRKMMPnUw\nfp6VWh1F44VWJEQOs4tOEjGiQiMQh5OfBk1w2haT3vrDbQvq23tNpUP1cRomLUtx\nEBcGKi5DMMBzE1RTVT1YUykR/zLWlA+JSmw7P6cWtsHYZovs8dgn8Po3X//6N+ng\nTQIDAQAB\n-----END PUBLIC KEY-----\n" + }, + "tag": [], + "attachment": [], + "endpoints": { + "sharedInbox": "https://mastodon.sdf.org/inbox" + }, + "icon": { + "type": "Image", + "mediaType": "image/jpeg", + "url": "https://mastodon.sdf.org/system/accounts/avatars/000/067/580/original/bf05521bf711b7a0.jpg?1533238802" + }, + "image": { + "type": "Image", + "mediaType": "image/gif", + "url": "https://mastodon.sdf.org/system/accounts/headers/000/067/580/original/a99b987e798f7063.gif?1533278217" + } +} diff --git a/test/fixtures/mastodon-question-activity.json b/test/fixtures/mastodon-question-activity.json new file mode 100644 index 000000000..ac329c7d5 --- /dev/null +++ b/test/fixtures/mastodon-question-activity.json @@ -0,0 +1,99 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + { + "ostatus": "http://ostatus.org#", + "atomUri": "ostatus:atomUri", + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "sensitive": "as:sensitive", + "Hashtag": "as:Hashtag", + "toot": "http://joinmastodon.org/ns#", + "Emoji": "toot:Emoji", + "focalPoint": { + "@container": "@list", + "@id": "toot:focalPoint" + } + } + ], + "id": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304/activity", + "type": "Create", + "actor": "https://mastodon.sdf.org/users/rinpatch", + "published": "2019-05-10T09:03:36Z", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://mastodon.sdf.org/users/rinpatch/followers" + ], + "object": { + "id": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304", + "type": "Question", + "summary": null, + "inReplyTo": null, + "published": "2019-05-10T09:03:36Z", + "url": "https://mastodon.sdf.org/@rinpatch/102070944809637304", + "attributedTo": "https://mastodon.sdf.org/users/rinpatch", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://mastodon.sdf.org/users/rinpatch/followers" + ], + "sensitive": false, + "atomUri": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304", + "inReplyToAtomUri": null, + "conversation": "tag:mastodon.sdf.org,2019-05-10:objectId=15095122:objectType=Conversation", + "content": "

Why is Tenshi eating a corndog so cute?

", + "contentMap": { + "en": "

Why is Tenshi eating a corndog so cute?

" + }, + "endTime": "2019-05-11T09:03:36Z", + "closed": "2019-05-11T09:03:36Z", + "attachment": [], + "tag": [], + "replies": { + "id": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304/replies", + "type": "Collection", + "first": { + "type": "CollectionPage", + "partOf": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304/replies", + "items": [] + } + }, + "oneOf": [ + { + "type": "Note", + "name": "Dunno", + "replies": { + "type": "Collection", + "totalItems": 0 + } + }, + { + "type": "Note", + "name": "Everyone knows that!", + "replies": { + "type": "Collection", + "totalItems": 1 + } + }, + { + "type": "Note", + "name": "25 char limit is dumb", + "replies": { + "type": "Collection", + "totalItems": 0 + } + }, + { + "type": "Note", + "name": "I can't even fit a funny", + "replies": { + "type": "Collection", + "totalItems": 1 + } + } + ] + } +} diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 5b355bfe6..3064c032b 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -52,6 +52,14 @@ defmodule HttpRequestMock do }} end + def get("https://mastodon.sdf.org/users/rinpatch", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/httpoison_mock/rinpatch.json") + }} + end + def get( "https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/emelie", _, diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index c24b50f8c..727abbd17 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -113,6 +113,23 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert Enum.at(object.data["tag"], 2) == "moo" end + test "it works for incoming questions" do + data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!() + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + + object = Object.normalize(activity) + + assert Enum.all?(object.data["oneOf"], fn choice -> + choice["name"] in [ + "Dunno", + "Everyone knows that!", + "25 char limit is dumb", + "I can't even fit a funny" + ] + end) + end + test "it works for incoming notices with contentMap" do data = File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!() From 642a67dd4492f31b5b9fe457e34c1589c9d70c3f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 17 May 2019 11:44:47 +0300 Subject: [PATCH 04/64] Render polls in statuses --- .../web/mastodon_api/views/status_view.ex | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index bd2372944..6337c50e8 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -232,6 +232,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do spoiler_text: summary_html, visibility: get_visibility(object), media_attachments: attachments, + poll: render("poll.json", %{object: object, for: opts[:for]}), mentions: mentions, tags: build_tags(tags), application: %{ @@ -321,6 +322,57 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do } end + # TODO: Add tests for this view + def render("poll.json", %{object: object} = opts) do + {multiple, options} = + case object.data do + %{"anyOf" => options} when is_list(options) -> {true, options} + %{"oneOf" => options} when is_list(options) -> {false, options} + _ -> {nil, nil} + end + + if options do + end_time = + (object.data["closed"] || object.data["endTime"]) + |> NaiveDateTime.from_iso8601!() + + votes_count = object.data["votes_count"] || 0 + + expired = + end_time + |> NaiveDateTime.compare(NaiveDateTime.utc_now()) + |> case do + :lt -> true + _ -> false + end + + options = + Enum.map(options, fn %{"name" => name} = option -> + name = + HTML.filter_tags( + name, + User.html_filter_policy(opts[:for]) + ) + + %{title: name, votes_count: option["replies"]["votes_count"] || 0} + end) + + %{ + # Mastodon uses separate ids for polls, but an object can't have more than one poll embedded so object id is fine + id: object.id, + expires_at: Utils.to_masto_date(end_time), + expired: expired, + multiple: multiple, + votes_count: votes_count, + options: options, + voted: false, + emojis: build_emojis(object.data["emoji"]) + } + else + nil + end + end + def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do object = Object.normalize(activity) From fd920c897339b9cedea042dd6698d14380cedae7 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 18 May 2019 13:29:28 +0300 Subject: [PATCH 05/64] Mastodon API: Add support for posting polls --- lib/pleroma/web/common_api/common_api.ex | 4 +- lib/pleroma/web/common_api/utils.ex | 64 ++++++++++++++++--- .../mastodon_api_controller_test.exs | 22 +++++++ test/web/mastodon_api/status_view_test.exs | 1 + 4 files changed, 82 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index b53869c75..335ae70b0 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -149,6 +149,7 @@ defmodule Pleroma.Web.CommonAPI do data, visibility ), + {poll, mentions, tags} <- make_poll_data(data, mentions, tags), {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility), context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", @@ -164,7 +165,8 @@ defmodule Pleroma.Web.CommonAPI do in_reply_to, tags, cw, - cc + cc, + poll ), object <- Map.put( diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1dfe50b40..13cdffbbd 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -102,6 +102,48 @@ defmodule Pleroma.Web.CommonAPI.Utils do end end + def make_poll_data( + %{"poll" => %{"options" => options, "expires_in" => expires_in}} = data, + mentions, + tags + ) + when is_list(options) and is_integer(expires_in) do + content_type = get_content_type(data["content_type"]) + # XXX: There is probably a more performant/cleaner way to do this + {poll, {mentions, tags}} = + Enum.map_reduce(options, {mentions, tags}, fn option, {mentions, tags} -> + # TODO: Custom emoji + {option, mentions_merge, tags_merge} = format_input(option, content_type) + mentions = mentions ++ mentions_merge + tags = tags ++ tags_merge + + {%{ + "name" => option, + "type" => "Note", + "replies" => %{"type" => "Collection", "totalItems" => 0} + }, {mentions, tags}} + end) + + end_time = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(expires_in) + |> NaiveDateTime.to_iso8601() + + poll = + if Pleroma.Web.ControllerHelper.truthy_param?(data["poll"]["multiple"]) do + %{"type" => "Question", "anyOf" => poll, "closed" => end_time} + else + %{"type" => "Question", "oneOf" => poll, "closed" => end_time} + end + + {poll, mentions, tags} + end + + def make_poll_data(data, mentions, tags) do + IO.inspect(data, label: "data") + {%{}, mentions, tags} + end + def make_content_html( status, attachments, @@ -223,8 +265,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do in_reply_to, tags, cw \\ nil, - cc \\ [] + cc \\ [], + merge \\ %{} ) do + IO.inspect(merge, label: "merge") + object = %{ "type" => "Note", "to" => to, @@ -237,14 +282,17 @@ defmodule Pleroma.Web.CommonAPI.Utils do "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq() } - if in_reply_to do - in_reply_to_object = Object.normalize(in_reply_to) + object = + if in_reply_to do + in_reply_to_object = Object.normalize(in_reply_to) - object - |> Map.put("inReplyTo", in_reply_to_object.data["id"]) - else - object - end + object + |> Map.put("inReplyTo", in_reply_to_object.data["id"]) + else + object + end + + Map.merge(object, merge) end def format_naive_asctime(date) do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 505e45010..ce581c092 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -132,6 +132,28 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do refute id == third_id end + test "posting a poll", %{conn: conn} do + user = insert(:user) + time = NaiveDateTime.utc_now() + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "Who is the best girl?", + "poll" => %{"options" => ["Rei", "Asuka", "Misato"], "expires_in" => 420} + }) + + response = json_response(conn, 200) + + assert Enum.all?(response["poll"]["options"], fn %{"title" => title} -> + title in ["Rei", "Asuka", "Misato"] + end) + + assert NaiveDateTime.diff(NaiveDateTime.from_iso8601!(response["poll"]["expires_at"]), time) in 420..430 + refute response["poll"]["expred"] + end + test "posting a sensitive status", %{conn: conn} do user = insert(:user) diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index d7c800e83..9f2ebda4e 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -103,6 +103,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do muted: false, pinned: false, sensitive: false, + poll: nil, spoiler_text: HtmlSanitizeEx.basic_html(note.data["object"]["summary"]), visibility: "public", media_attachments: [], From 1d90f9b96999f8bad3fa3e3ec58bf50c8666be4f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 19 May 2019 17:06:44 +0300 Subject: [PATCH 06/64] Remove tags/mentions/rich text from poll options because Mastodon and add custom emoji --- lib/pleroma/web/common_api/common_api.ex | 4 +-- lib/pleroma/web/common_api/utils.ex | 25 ++++++------------- .../web/mastodon_api/views/status_view.ex | 14 +++++------ 3 files changed, 15 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index bc8f80389..374967a1b 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -154,7 +154,7 @@ defmodule Pleroma.Web.CommonAPI do data, visibility ), - {poll, mentions, tags} <- make_poll_data(data, mentions, tags), + {poll, poll_emoji} <- make_poll_data(data), {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility), context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", @@ -179,7 +179,7 @@ defmodule Pleroma.Web.CommonAPI do Map.put( object, "emoji", - Formatter.get_emoji_map(full_payload) + Map.merge(Formatter.get_emoji_map(full_payload), poll_emoji) ) do res = ActivityPub.create( diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 2ea997789..cd8483c11 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -102,26 +102,15 @@ defmodule Pleroma.Web.CommonAPI.Utils do end end - def make_poll_data( - %{"poll" => %{"options" => options, "expires_in" => expires_in}} = data, - mentions, - tags - ) + def make_poll_data(%{"poll" => %{"options" => options, "expires_in" => expires_in}} = data) when is_list(options) and is_integer(expires_in) do - content_type = get_content_type(data["content_type"]) - # XXX: There is probably a more performant/cleaner way to do this - {poll, {mentions, tags}} = - Enum.map_reduce(options, {mentions, tags}, fn option, {mentions, tags} -> - # TODO: Custom emoji - {option, mentions_merge, tags_merge} = format_input(option, content_type) - mentions = mentions ++ mentions_merge - tags = tags ++ tags_merge - + {poll, emoji} = + Enum.map_reduce(options, %{}, fn option, emoji -> {%{ "name" => option, "type" => "Note", "replies" => %{"type" => "Collection", "totalItems" => 0} - }, {mentions, tags}} + }, Map.merge(emoji, Formatter.get_emoji_map(option))} end) end_time = @@ -136,11 +125,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do %{"type" => "Question", "oneOf" => poll, "closed" => end_time} end - {poll, mentions, tags} + {poll, emoji} end - def make_poll_data(_data, mentions, tags) do - {%{}, mentions, tags} + def make_poll_data(_data) do + {%{}, %{}} end def make_content_html( diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 2a5691e1f..0df8bb5c2 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -325,7 +325,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end # TODO: Add tests for this view - def render("poll.json", %{object: object} = opts) do + def render("poll.json", %{object: object} = _opts) do {multiple, options} = case object.data do %{"anyOf" => options} when is_list(options) -> {true, options} @@ -350,13 +350,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do options = Enum.map(options, fn %{"name" => name} = option -> - name = - HTML.filter_tags( - name, - User.html_filter_policy(opts[:for]) - ) - - %{title: name, votes_count: option["replies"]["votes_count"] || 0} + %{ + title: HTML.strip_tags(name), + votes_count: option["replies"]["votes_count"] || 0 + } end) %{ @@ -367,6 +364,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do multiple: multiple, votes_count: votes_count, options: options, + # TODO: Actually check for a vote voted: false, emojis: build_emojis(object.data["emoji"]) } From 6430cb1bf78e7949cc023a30df7a8d1547c36524 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 19 May 2019 17:44:18 +0300 Subject: [PATCH 07/64] Restrict poll replies from fetch queries by default --- lib/pleroma/web/activity_pub/activity_pub.ex | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5c3156978..035fb75d5 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -820,6 +820,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query + defp restrict_poll_replies(query, %{"include_poll_replies" => "true"}), do: query + + defp restrict_poll_replies(query, _) do + if has_named_binding?(query, :object) do + from([activity, object: o] in query, where: fragment("?->'name' is null", o.data)) + else + query + end + end + defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query defp maybe_preload_objects(query, _) do @@ -873,6 +883,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_pinned(opts) |> restrict_muted_reblogs(opts) |> Activity.restrict_deactivated_users() + |> restrict_poll_replies(opts) end def fetch_activities(recipients, opts \\ %{}) do From 76a7429befb2e9a819b653ff8328cc42a565c29d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 09:13:10 +0300 Subject: [PATCH 08/64] Add poll limits to /api/v1/instance and initial state --- config/config.exs | 6 ++++++ docs/config.md | 5 +++++ lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 4 +++- test/web/mastodon_api/mastodon_api_controller_test.exs | 3 ++- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 9a10b0ff7..47d8dfb42 100644 --- a/config/config.exs +++ b/config/config.exs @@ -211,6 +211,12 @@ config :pleroma, :instance, avatar_upload_limit: 2_000_000, background_upload_limit: 4_000_000, banner_upload_limit: 4_000_000, + poll_limits: %{ + max_options: 20, + max_option_chars: 200, + min_expiration: 0, + max_expiration: 365 * 24 * 60 * 60 + }, registrations_open: true, federating: true, federation_reachability_timeout_days: 7, diff --git a/docs/config.md b/docs/config.md index 450d73fda..f9903332c 100644 --- a/docs/config.md +++ b/docs/config.md @@ -71,6 +71,11 @@ config :pleroma, Pleroma.Emails.Mailer, * `avatar_upload_limit`: File size limit of user’s profile avatars * `background_upload_limit`: File size limit of user’s profile backgrounds * `banner_upload_limit`: File size limit of user’s profile banners +* `poll_limits`: A map with poll limits for **local** polls + * `max_options`: Maximum number of options + * `max_option_chars`: Maximum number of characters per option + * `min_expiration`: Minimum expiration time (in seconds) + * `max_expiration`: Maximum expiration time (in seconds) * `registrations_open`: Enable registrations for anyone, invitations can be enabled when false. * `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`). * `account_activation_required`: Require users to confirm their emails before signing in. diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 1051861ff..81cc5a972 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -197,7 +197,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do languages: ["en"], registrations: Pleroma.Config.get([:instance, :registrations_open]), # Extra (not present in Mastodon): - max_toot_chars: Keyword.get(instance, :limit) + max_toot_chars: Keyword.get(instance, :limit), + poll_limits: Keyword.get(instance, :poll_limits) } json(conn, response) @@ -1331,6 +1332,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do max_toot_chars: limit, mascot: "/images/pleroma-fox-tan-smol.png" }, + poll_limits: Config.get([:instance, :poll_limits]), rights: %{ delete_others_notice: present?(user.info.is_moderator), admin: present?(user.info.is_admin) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 68fe9c1b4..48268d4f7 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2494,7 +2494,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do "stats" => _, "thumbnail" => _, "languages" => _, - "registrations" => _ + "registrations" => _, + "poll_limits" => _ } = result assert email == from_config_email From 3f96b3e4b8114ec1cf924d452907b17c2aea2003 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 10:54:20 +0300 Subject: [PATCH 09/64] Enforce poll limits and add error handling for MastodonAPI's post endpoint --- lib/pleroma/web/common_api/utils.ex | 71 ++++++++---- .../mastodon_api/mastodon_api_controller.ex | 43 ++++--- .../mastodon_api_controller_test.exs | 107 +++++++++++++++--- 3 files changed, 173 insertions(+), 48 deletions(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index cd8483c11..97172fd94 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -104,28 +104,61 @@ defmodule Pleroma.Web.CommonAPI.Utils do def make_poll_data(%{"poll" => %{"options" => options, "expires_in" => expires_in}} = data) when is_list(options) and is_integer(expires_in) do - {poll, emoji} = - Enum.map_reduce(options, %{}, fn option, emoji -> - {%{ - "name" => option, - "type" => "Note", - "replies" => %{"type" => "Collection", "totalItems" => 0} - }, Map.merge(emoji, Formatter.get_emoji_map(option))} - end) + %{max_expiration: max_expiration, min_expiration: min_expiration} = + limits = Pleroma.Config.get([:instance, :poll_limits]) - end_time = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(expires_in) - |> NaiveDateTime.to_iso8601() - - poll = - if Pleroma.Web.ControllerHelper.truthy_param?(data["poll"]["multiple"]) do - %{"type" => "Question", "anyOf" => poll, "closed" => end_time} - else - %{"type" => "Question", "oneOf" => poll, "closed" => end_time} + # XXX: There is probably a cleaner way of doing this + try do + if Enum.count(options) > limits.max_options do + raise ArgumentError, message: "Poll can't contain more than #{limits.max_options} options" end - {poll, emoji} + {poll, emoji} = + Enum.map_reduce(options, %{}, fn option, emoji -> + if String.length(option) > limits.max_option_chars do + raise ArgumentError, + message: + "Poll options cannot be longer than #{limits.max_option_chars} characters each" + end + + {%{ + "name" => option, + "type" => "Note", + "replies" => %{"type" => "Collection", "totalItems" => 0} + }, Map.merge(emoji, Formatter.get_emoji_map(option))} + end) + + case expires_in do + expires_in when expires_in > max_expiration -> + raise ArgumentError, message: "Expiration date is too far in the future" + + expires_in when expires_in < min_expiration -> + raise ArgumentError, message: "Expiration date is too soon" + + _ -> + :noop + end + + end_time = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(expires_in) + |> NaiveDateTime.to_iso8601() + + poll = + if Pleroma.Web.ControllerHelper.truthy_param?(data["poll"]["multiple"]) do + %{"type" => "Question", "anyOf" => poll, "closed" => end_time} + else + %{"type" => "Question", "oneOf" => poll, "closed" => end_time} + end + + {poll, emoji} + rescue + e in ArgumentError -> e.message + end + end + + def make_poll_data(%{"poll" => _}) do + "Invalid poll" end def make_poll_data(_data) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 81cc5a972..e2cab86f1 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -473,12 +473,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do params |> Map.put("in_reply_to_status_id", params["in_reply_to_id"]) - idempotency_key = - case get_req_header(conn, "idempotency-key") do - [key] -> key - _ -> Ecto.UUID.generate() - end - scheduled_at = params["scheduled_at"] if scheduled_at && ScheduledActivity.far_enough?(scheduled_at) do @@ -491,17 +485,40 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do else params = Map.drop(params, ["scheduled_at"]) - {:ok, activity} = - Cachex.fetch!(:idempotency_cache, idempotency_key, fn _ -> - CommonAPI.post(user, params) - end) + case get_cached_status_or_post(conn, params) do + {:ignore, message} -> + conn + |> put_status(401) + |> json(%{error: message}) - conn - |> put_view(StatusView) - |> try_render("status.json", %{activity: activity, for: user, as: :activity}) + {:error, message} -> + conn + |> put_status(401) + |> json(%{error: message}) + + {_, activity} -> + conn + |> put_view(StatusView) + |> try_render("status.json", %{activity: activity, for: user, as: :activity}) + end end end + defp get_cached_status_or_post(%{assigns: %{user: user}} = conn, params) do + idempotency_key = + case get_req_header(conn, "idempotency-key") do + [key] -> key + _ -> Ecto.UUID.generate() + end + + Cachex.fetch(:idempotency_cache, idempotency_key, fn _ -> + case CommonAPI.post(user, params) do + {:ok, activity} -> activity + {:error, message} -> {:ignore, message} + end + end) + end + def delete_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do json(conn, %{}) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 48268d4f7..e1df79ffb 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -146,26 +146,101 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do refute id == third_id end - test "posting a poll", %{conn: conn} do - user = insert(:user) - time = NaiveDateTime.utc_now() + describe "posting polls" do + test "posting a poll", %{conn: conn} do + user = insert(:user) + time = NaiveDateTime.utc_now() - conn = - conn - |> assign(:user, user) - |> post("/api/v1/statuses", %{ - "status" => "Who is the best girl?", - "poll" => %{"options" => ["Rei", "Asuka", "Misato"], "expires_in" => 420} - }) + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "Who is the #bestgrill?", + "poll" => %{"options" => ["Rei", "Asuka", "Misato"], "expires_in" => 420} + }) - response = json_response(conn, 200) + response = json_response(conn, 200) - assert Enum.all?(response["poll"]["options"], fn %{"title" => title} -> - title in ["Rei", "Asuka", "Misato"] - end) + assert Enum.all?(response["poll"]["options"], fn %{"title" => title} -> + title in ["Rei", "Asuka", "Misato"] + end) - assert NaiveDateTime.diff(NaiveDateTime.from_iso8601!(response["poll"]["expires_at"]), time) in 420..430 - refute response["poll"]["expred"] + assert NaiveDateTime.diff(NaiveDateTime.from_iso8601!(response["poll"]["expires_at"]), time) in 420..430 + refute response["poll"]["expred"] + end + + test "option limit is enforced", %{conn: conn} do + user = insert(:user) + limit = Pleroma.Config.get([:instance, :poll_limits, :max_options]) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "desu~", + "poll" => %{"options" => Enum.map(0..limit, fn _ -> "desu" end), "expires_in" => 1} + }) + + %{"error" => error} = json_response(conn, 401) + assert error == "Poll can't contain more than #{limit} options" + end + + test "option character limit is enforced", %{conn: conn} do + user = insert(:user) + limit = Pleroma.Config.get([:instance, :poll_limits, :max_option_chars]) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "...", + "poll" => %{ + "options" => [Enum.reduce(0..limit, "", fn _, acc -> acc <> "." end)], + "expires_in" => 1 + } + }) + + %{"error" => error} = json_response(conn, 401) + assert error == "Poll options cannot be longer than #{limit} characters each" + end + + test "minimal date limit is enforced", %{conn: conn} do + user = insert(:user) + limit = Pleroma.Config.get([:instance, :poll_limits, :min_expiration]) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "imagine arbitrary limits", + "poll" => %{ + "options" => ["this post was made by pleroma gang"], + "expires_in" => limit - 1 + } + }) + + %{"error" => error} = json_response(conn, 401) + assert error == "Expiration date is too soon" + end + + test "maximum date limit is enforced", %{conn: conn} do + user = insert(:user) + limit = Pleroma.Config.get([:instance, :poll_limits, :max_expiration]) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "imagine arbitrary limits", + "poll" => %{ + "options" => ["this post was made by pleroma gang"], + "expires_in" => limit + 1 + } + }) + + %{"error" => error} = json_response(conn, 401) + assert error == "Expiration date is too far in the future" + end end test "posting a sensitive status", %{conn: conn} do From aafe30d94e68ccf251c56139d07bda154dde3af9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 14:12:10 +0300 Subject: [PATCH 10/64] Handle poll votes --- lib/pleroma/object.ex | 30 ++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 10 ++++++ test/fixtures/mastodon-vote.json | 16 ++++++++++ test/web/activity_pub/transmogrifier_test.exs | 31 +++++++++++++++++++ 4 files changed, 87 insertions(+) create mode 100644 test/fixtures/mastodon-vote.json diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 740d687a3..a0f7659eb 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -188,4 +188,34 @@ defmodule Pleroma.Object do _ -> {:error, "Not found"} end end + + def increase_vote_count(ap_id, name) do + with %Object{} = object <- Object.normalize(ap_id), + "Question" <- object.data["type"] do + multiple = Map.has_key?(object.data, "anyOf") + + options = + (object.data["anyOf"] || object.data["oneOf"] || []) + |> Enum.map(fn + %{"name" => ^name} = option -> + Kernel.update_in(option["replies"]["totalItems"], &(&1 + 1)) + + option -> + option + end) + + data = + if multiple do + Map.put(object.data, "anyOf", options) + else + Map.put(object.data, "oneOf", options) + end + + object + |> Object.change(%{data: data}) + |> update_and_set_cache() + else + _ -> :noop + end + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 035fb75d5..ce78d895b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -108,6 +108,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def decrease_replies_count_if_reply(_object), do: :noop + def increase_poll_votes_if_vote(%{ + "object" => %{"inReplyTo" => reply_ap_id, "name" => name}, + "type" => "Create" + }) do + Object.increase_vote_count(reply_ap_id, name) + end + + def increase_poll_votes_if_vote(_create_data), do: :noop + def insert(map, local \\ true, fake \\ false) when is_map(map) do with nil <- Activity.normalize(map), map <- lazy_put_activity_defaults(map, fake), @@ -235,6 +244,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {:ok, activity} <- insert(create_data, local, fake), {:fake, false, activity} <- {:fake, fake, activity}, _ <- increase_replies_count_if_reply(create_data), + _ <- increase_poll_votes_if_vote(create_data), # Changing note count prior to enqueuing federation task in order to avoid # race conditions on updating user.info {:ok, _actor} <- increase_note_count_if_public(actor, activity), diff --git a/test/fixtures/mastodon-vote.json b/test/fixtures/mastodon-vote.json new file mode 100644 index 000000000..c2c5f40c0 --- /dev/null +++ b/test/fixtures/mastodon-vote.json @@ -0,0 +1,16 @@ +{ + "@context": "https://www.w3.org/ns/activitystreams", + "actor": "https://mastodon.sdf.org/users/rinpatch", + "id": "https://mastodon.sdf.org/users/rinpatch#votes/387/activity", + "nickname": "rin", + "object": { + "attributedTo": "https://mastodon.sdf.org/users/rinpatch", + "id": "https://mastodon.sdf.org/users/rinpatch#votes/387", + "inReplyTo": "https://testing.uguu.ltd/objects/9d300947-2dcb-445d-8978-9a3b4b84fa14", + "name": "suya..", + "to": "https://testing.uguu.ltd/users/rin", + "type": "Note" + }, + "to": "https://testing.uguu.ltd/users/rin", + "type": "Create" +} diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 727abbd17..32d94e3e9 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -130,6 +130,37 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end) end + test "in increments vote counters on question activities" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "suya...", + "poll" => %{"options" => ["suya", "suya.", "suya.."], "expires_in" => 10} + }) + + object = Object.normalize(activity) + + data = + File.read!("test/fixtures/mastodon-vote.json") + |> Poison.decode!() + |> Kernel.put_in(["to"], user.ap_id) + |> Kernel.put_in(["object", "inReplyTo"], object.data["id"]) + |> Kernel.put_in(["object", "to"], user.ap_id) + + {:ok, %Activity{local: false}} = Transmogrifier.handle_incoming(data) + + object = Object.get_by_ap_id(object.data["id"]) + + assert Enum.any?( + object.data["oneOf"], + fn + %{"name" => "suya..", "replies" => %{"totalItems" => 1}} -> true + _ -> false + end + ) + end + test "it works for incoming notices with contentMap" do data = File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!() From a53d06273080525fdda332291838b0c95ed69690 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 14:19:03 +0300 Subject: [PATCH 11/64] Fix posting non-polls from mastofe --- lib/pleroma/web/common_api/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 97172fd94..66153a105 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -157,7 +157,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do end end - def make_poll_data(%{"poll" => _}) do + def make_poll_data(%{"poll" => poll}) when is_map(poll) do "Invalid poll" end From f28747858bf3265e8b82eb587919f5a89386bed7 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 14:27:09 +0300 Subject: [PATCH 12/64] Actual vote count in poll view --- .../web/mastodon_api/views/status_view.ex | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 0df8bb5c2..c501c213c 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -338,8 +338,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do (object.data["closed"] || object.data["endTime"]) |> NaiveDateTime.from_iso8601!() - votes_count = object.data["votes_count"] || 0 - expired = end_time |> NaiveDateTime.compare(NaiveDateTime.utc_now()) @@ -348,12 +346,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do _ -> false end - options = - Enum.map(options, fn %{"name" => name} = option -> - %{ - title: HTML.strip_tags(name), - votes_count: option["replies"]["votes_count"] || 0 - } + {options, votes_count} = + Enum.map_reduce(options, 0, fn %{"name" => name} = option, count -> + current_count = option["replies"]["totalItems"] || 0 + + {%{ + title: HTML.strip_tags(name), + votes_count: current_count + }, current_count + count} end) %{ From d7c4d029c82c833278b3640d585235dc704f30d1 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 14:35:20 +0300 Subject: [PATCH 13/64] Restrict poll replies when fetching activiites for context --- lib/pleroma/web/activity_pub/activity_pub.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index ce78d895b..c04e6c2b8 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -491,6 +491,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do from(activity in Activity) |> restrict_blocked(opts) + |> restrict_poll_replies(opts) |> restrict_recipients(recipients, opts["user"]) |> where( [activity], From ee682441415d7abe4acb2643e1d76fe8d78e80c1 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 16:58:15 +0300 Subject: [PATCH 14/64] Do not stream out poll replies --- lib/pleroma/web/activity_pub/activity_pub.ex | 54 ++++++++++---------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index c04e6c2b8..fdebf1f6b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -192,40 +192,42 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do public = "https://www.w3.org/ns/activitystreams#Public" if activity.data["type"] in ["Create", "Announce", "Delete"] do - Pleroma.Web.Streamer.stream("user", activity) - Pleroma.Web.Streamer.stream("list", activity) + object = Object.normalize(activity) + # Do not stream out poll replies + unless object.data["name"] do + Pleroma.Web.Streamer.stream("user", activity) + Pleroma.Web.Streamer.stream("list", activity) - if Enum.member?(activity.data["to"], public) do - Pleroma.Web.Streamer.stream("public", activity) + if Enum.member?(activity.data["to"], public) do + Pleroma.Web.Streamer.stream("public", activity) - if activity.local do - Pleroma.Web.Streamer.stream("public:local", activity) - end + if activity.local do + Pleroma.Web.Streamer.stream("public:local", activity) + end - if activity.data["type"] in ["Create"] do - object = Object.normalize(activity) + if activity.data["type"] in ["Create"] do + object.data + |> Map.get("tag", []) + |> Enum.filter(fn tag -> is_bitstring(tag) end) + |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) - object.data - |> Map.get("tag", []) - |> Enum.filter(fn tag -> is_bitstring(tag) end) - |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) + if object.data["attachment"] != [] do + Pleroma.Web.Streamer.stream("public:media", activity) - if object.data["attachment"] != [] do - Pleroma.Web.Streamer.stream("public:media", activity) - - if activity.local do - Pleroma.Web.Streamer.stream("public:local:media", activity) + if activity.local do + Pleroma.Web.Streamer.stream("public:local:media", activity) + end end end + else + # TODO: Write test, replace with visibility test + if !Enum.member?(activity.data["cc"] || [], public) && + !Enum.member?( + activity.data["to"], + User.get_cached_by_ap_id(activity.data["actor"]).follower_address + ), + do: Pleroma.Web.Streamer.stream("direct", activity) end - else - # TODO: Write test, replace with visibility test - if !Enum.member?(activity.data["cc"] || [], public) && - !Enum.member?( - activity.data["to"], - User.get_cached_by_ap_id(activity.data["actor"]).follower_address - ), - do: Pleroma.Web.Streamer.stream("direct", activity) end end end From 0407ffe75f7e91db240d491492eadf1385b1726b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 17:12:38 +0300 Subject: [PATCH 15/64] Change validation error status codes to be more appropriate --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 4 ++-- test/web/mastodon_api/mastodon_api_controller_test.exs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index e2cab86f1..aef2abf0b 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -488,12 +488,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do case get_cached_status_or_post(conn, params) do {:ignore, message} -> conn - |> put_status(401) + |> put_status(422) |> json(%{error: message}) {:error, message} -> conn - |> put_status(401) + |> put_status(422) |> json(%{error: message}) {_, activity} -> diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index e1df79ffb..4f332f83c 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -181,7 +181,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do "poll" => %{"options" => Enum.map(0..limit, fn _ -> "desu" end), "expires_in" => 1} }) - %{"error" => error} = json_response(conn, 401) + %{"error" => error} = json_response(conn, 422) assert error == "Poll can't contain more than #{limit} options" end @@ -200,7 +200,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do } }) - %{"error" => error} = json_response(conn, 401) + %{"error" => error} = json_response(conn, 422) assert error == "Poll options cannot be longer than #{limit} characters each" end @@ -219,7 +219,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do } }) - %{"error" => error} = json_response(conn, 401) + %{"error" => error} = json_response(conn, 422) assert error == "Expiration date is too soon" end @@ -238,7 +238,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do } }) - %{"error" => error} = json_response(conn, 401) + %{"error" => error} = json_response(conn, 422) assert error == "Expiration date is too far in the future" end end From 5f67c26baf2bdc98480fa4cda5895f33351b13ab Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 17:30:51 +0300 Subject: [PATCH 16/64] Accept strings in expires_in because sasuga javascript --- lib/pleroma/web/common_api/utils.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 66153a105..1a239de97 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -103,12 +103,15 @@ defmodule Pleroma.Web.CommonAPI.Utils do end def make_poll_data(%{"poll" => %{"options" => options, "expires_in" => expires_in}} = data) - when is_list(options) and is_integer(expires_in) do + when is_list(options) do %{max_expiration: max_expiration, min_expiration: min_expiration} = limits = Pleroma.Config.get([:instance, :poll_limits]) # XXX: There is probably a cleaner way of doing this try do + # In some cases mastofe sends out strings instead of integers + expires_in = if is_binary(expires_in), do: String.to_integer(expires_in), else: expires_in + if Enum.count(options) > limits.max_options do raise ArgumentError, message: "Poll can't contain more than #{limits.max_options} options" end From ff61d345020b9ee79b243fb8d23a37a06cc41b8e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 17:33:54 +0300 Subject: [PATCH 17/64] Accept question objects for conversations --- lib/pleroma/conversation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 238c1acf2..bc97b39ca 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -49,7 +49,7 @@ defmodule Pleroma.Conversation do with true <- Pleroma.Web.ActivityPub.Visibility.is_direct?(activity), "Create" <- activity.data["type"], object <- Pleroma.Object.normalize(activity), - "Note" <- object.data["type"], + true <- object.data["type"] in ["Note", "Question"], ap_id when is_binary(ap_id) and byte_size(ap_id) > 0 <- object.data["context"] do {:ok, conversation} = create_for_ap_id(ap_id) From 63b0b7190cb652cd27d236e3f9daaaf5e50701a6 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 20:40:35 +0300 Subject: [PATCH 18/64] MastoAPI: Add GET /api/v1/polls/:id --- lib/pleroma/object.ex | 3 ++ .../mastodon_api/mastodon_api_controller.ex | 20 +++++++++ lib/pleroma/web/router.ex | 2 + .../mastodon_api_controller_test.exs | 44 +++++++++++++++++++ 4 files changed, 69 insertions(+) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index a0f7659eb..3ffa290eb 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -35,6 +35,9 @@ defmodule Pleroma.Object do |> unique_constraint(:ap_id, name: :objects_unique_apid_index) end + def get_by_id(nil), do: nil + def get_by_id(id), do: Repo.get(Object, id) + def get_by_ap_id(nil), do: nil def get_by_ap_id(ap_id) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index aef2abf0b..ecb7df459 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -410,6 +410,26 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end end + def get_poll(%{assigns: %{user: user}} = conn, %{"id" => id}) do + with %Object{} = object <- Object.get_by_id(id), + %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), + true <- Visibility.visible_for_user?(activity, user) do + conn + |> put_view(StatusView) + |> try_render("poll.json", %{object: object, for: user}) + else + nil -> + conn + |> put_status(404) + |> json(%{error: "Record not found"}) + + false -> + conn + |> put_status(404) + |> json(%{error: "Record not found"}) + end + end + def scheduled_statuses(%{assigns: %{user: user}} = conn, params) do with scheduled_activities <- MastodonAPI.get_scheduled_activities(user, params) do conn diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6a4e4a1d4..e0611e3fc 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -423,6 +423,8 @@ defmodule Pleroma.Web.Router do get("/statuses/:id", MastodonAPIController, :get_status) get("/statuses/:id/context", MastodonAPIController, :get_context) + get("/polls/:id", MastodonAPIController, :get_poll) + get("/accounts/:id/statuses", MastodonAPIController, :user_statuses) get("/accounts/:id/followers", MastodonAPIController, :followers) get("/accounts/:id/following", MastodonAPIController, :following) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 4f332f83c..0d56b6ff2 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3453,4 +3453,48 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert json_response(conn, 403) == %{"error" => "Rate limit exceeded."} end end + + describe "GET /api/v1/polls/:id" do + test "returns poll entity for object id", %{conn: conn} do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "Pleroma does", + "poll" => %{"options" => ["what Mastodon't", "n't what Mastodoes"], "expires_in" => 20} + }) + + object = Object.normalize(activity) + + conn = + conn + |> assign(:user, user) + |> get("/api/v1/polls/#{object.id}") + + response = json_response(conn, 200) + id = object.id + assert %{"id" => ^id, "expired" => false, "multiple" => false} = response + end + + test "does not expose polls for private statuses", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "Pleroma does", + "poll" => %{"options" => ["what Mastodon't", "n't what Mastodoes"], "expires_in" => 20}, + "visibility" => "private" + }) + + object = Object.normalize(activity) + + conn = + conn + |> assign(:user, other_user) + |> get("/api/v1/polls/#{object.id}") + + assert json_response(conn, 404) + end + end end From 10ca1f91de12c3ff3a1adff2cf36e9ec47c659dd Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 11:56:53 +0300 Subject: [PATCH 19/64] Add GIN index on object data->'name' --- .../migrations/20190522084924_add_object_index_on_name.exs | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 priv/repo/migrations/20190522084924_add_object_index_on_name.exs diff --git a/priv/repo/migrations/20190522084924_add_object_index_on_name.exs b/priv/repo/migrations/20190522084924_add_object_index_on_name.exs new file mode 100644 index 000000000..886e8499e --- /dev/null +++ b/priv/repo/migrations/20190522084924_add_object_index_on_name.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddObjectIndexOnName do + use Ecto.Migration + + def change do + create(index(:objects, ["(data->'name')"], name: :objects_name_index, using: :gin)) + end +end From 19c90d47c4f649d0962098050f6cbc65eeb889d0 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 21:17:57 +0300 Subject: [PATCH 20/64] Normalize poll votes to Answer objects --- lib/pleroma/web/activity_pub/activity_pub.ex | 14 +-------- .../web/activity_pub/transmogrifier.ex | 22 ++++++++++++- lib/pleroma/web/activity_pub/utils.ex | 2 +- test/web/activity_pub/transmogrifier_test.exs | 31 +++++++++++++++++-- 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index fdebf1f6b..5b3fb7e84 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -194,7 +194,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do if activity.data["type"] in ["Create", "Announce", "Delete"] do object = Object.normalize(activity) # Do not stream out poll replies - unless object.data["name"] do + unless object.data["type"] == "Answer" do Pleroma.Web.Streamer.stream("user", activity) Pleroma.Web.Streamer.stream("list", activity) @@ -493,7 +493,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do from(activity in Activity) |> restrict_blocked(opts) - |> restrict_poll_replies(opts) |> restrict_recipients(recipients, opts["user"]) |> where( [activity], @@ -833,16 +832,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query - defp restrict_poll_replies(query, %{"include_poll_replies" => "true"}), do: query - - defp restrict_poll_replies(query, _) do - if has_named_binding?(query, :object) do - from([activity, object: o] in query, where: fragment("?->'name' is null", o.data)) - else - query - end - end - defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query defp maybe_preload_objects(query, _) do @@ -896,7 +885,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_pinned(opts) |> restrict_muted_reblogs(opts) |> Activity.restrict_deactivated_users() - |> restrict_poll_replies(opts) end def fetch_activities(recipients, opts \\ %{}) do diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 8b2427258..70b467b3f 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -35,6 +35,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> fix_likes |> fix_addressing |> fix_summary + |> fix_type end def fix_summary(%{"summary" => nil} = object) do @@ -328,6 +329,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_content_map(object), do: object + def fix_type(%{"inReplyTo" => reply_id} = object) when is_binary(reply_id) do + reply = Object.normalize(reply_id) + + if reply.data["type"] == "Question" and object["name"] do + Map.put(object, "type", "Answer") + else + object + end + end + + def fix_type(object), do: object + defp mastodon_follow_hack(%{"id" => id, "actor" => follower_id}, followed) do with true <- id =~ "follows", %User{local: true} = follower <- User.get_cached_by_ap_id(follower_id), @@ -398,7 +411,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # - tags # - emoji def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data) - when objtype in ["Article", "Note", "Video", "Page", "Question"] do + when objtype in ["Article", "Note", "Video", "Page", "Question", "Answer"] do actor = Containment.get_actor(data) data = @@ -731,6 +744,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> set_reply_to_uri |> strip_internal_fields |> strip_internal_tags + |> set_type end # @doc @@ -895,6 +909,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Map.put(object, "sensitive", "nsfw" in tags) end + def set_type(%{"type" => "Answer"} = object) do + Map.put(object, "type", "Note") + end + + def set_type(object), do: object + def add_attributed_to(object) do attributed_to = object["attributedTo"] || object["actor"] diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 63454e3f7..9646bbee9 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do require Logger - @supported_object_types ["Article", "Note", "Video", "Page", "Question"] + @supported_object_types ["Article", "Note", "Video", "Page", "Question", "Answer"] @supported_report_states ~w(open closed resolved) @valid_visibilities ~w(public unlisted private direct) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 32d94e3e9..8422fc3d5 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -130,7 +130,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end) end - test "in increments vote counters on question activities" do + test "it rewrites Note votes to Answers and increments vote counters on question activities" do user = insert(:user) {:ok, activity} = @@ -148,8 +148,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do |> Kernel.put_in(["object", "inReplyTo"], object.data["id"]) |> Kernel.put_in(["object", "to"], user.ap_id) - {:ok, %Activity{local: false}} = Transmogrifier.handle_incoming(data) - + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + answer_object = Object.normalize(activity) + assert answer_object.data["type"] == "Answer" object = Object.get_by_ap_id(object.data["id"]) assert Enum.any?( @@ -1257,4 +1258,28 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do {:ok, _} = Transmogrifier.prepare_outgoing(activity.data) end end + + test "Rewrites Answers to Notes" do + user = insert(:user) + + {:ok, poll_activity} = + CommonAPI.post(user, %{ + "status" => "suya...", + "poll" => %{"options" => ["suya", "suya.", "suya.."], "expires_in" => 10} + }) + + poll_object = Object.normalize(poll_activity) + # TODO: Replace with CommonAPI vote creation when implemented + data = + File.read!("test/fixtures/mastodon-vote.json") + |> Poison.decode!() + |> Kernel.put_in(["to"], user.ap_id) + |> Kernel.put_in(["object", "inReplyTo"], poll_object.data["id"]) + |> Kernel.put_in(["object", "to"], user.ap_id) + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) + + assert data["object"]["type"] == "Note" + end end From ac7702f8009bf244a9e77cba39114dbb6584739f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 21:49:19 +0300 Subject: [PATCH 21/64] Exclude Answers from fetching by default --- lib/pleroma/web/activity_pub/activity_pub.ex | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5b3fb7e84..db5a4a7ee 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -492,6 +492,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do if opts["user"], do: [opts["user"].ap_id | opts["user"].following] ++ public, else: public from(activity in Activity) + |> Activity.with_preloaded_object() + |> exclude_poll_votes(opts) |> restrict_blocked(opts) |> restrict_recipients(recipients, opts["user"]) |> where( @@ -832,6 +834,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query + defp exclude_poll_votes(query, %{"include_poll_votes" => "true"}), do: query + + defp exclude_poll_votes(query, _) do + if has_named_binding?(query, :object) do + from([activity, object: o] in query, + where: fragment("not(?->>'type' = ?)", o.data, "Answer") + ) + else + query + end + end + defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query defp maybe_preload_objects(query, _) do @@ -865,6 +879,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> maybe_preload_objects(opts) |> maybe_preload_bookmarks(opts) |> maybe_order(opts) + |> exclude_poll_votes(opts) |> restrict_recipients(recipients, opts["user"]) |> restrict_tag(opts) |> restrict_tag_reject(opts) From e6b175ed6cafcd6a05120e003cfe40a04b38849f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 21:57:46 +0300 Subject: [PATCH 22/64] Fix credo issues --- lib/pleroma/web/mastodon_api/views/status_view.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index c501c213c..7a393a817 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -357,7 +357,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end) %{ - # Mastodon uses separate ids for polls, but an object can't have more than one poll embedded so object id is fine + # Mastodon uses separate ids for polls, but an object can't have + # more than one poll embedded so object id is fine id: object.id, expires_at: Utils.to_masto_date(end_time), expired: expired, From 8b2d39c1ec8f47df8a2159c23eabdc61b983ddf0 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 23 May 2019 14:03:16 +0300 Subject: [PATCH 23/64] Change the order of preloading when fetching activities for context --- lib/pleroma/web/activity_pub/activity_pub.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index db5a4a7ee..b06200cb5 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -492,7 +492,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do if opts["user"], do: [opts["user"].ap_id | opts["user"].following] ++ public, else: public from(activity in Activity) - |> Activity.with_preloaded_object() + |> maybe_preload_objects(opts) |> exclude_poll_votes(opts) |> restrict_blocked(opts) |> restrict_recipients(recipients, opts["user"]) @@ -513,7 +513,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def fetch_activities_for_context(context, opts \\ %{}) do context |> fetch_activities_for_context_query(opts) - |> Activity.with_preloaded_object() |> Repo.all() end @@ -521,7 +520,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Pleroma.FlakeId.t() | nil def fetch_latest_activity_id_for_context(context, opts \\ %{}) do context - |> fetch_activities_for_context_query(opts) + |> fetch_activities_for_context_query(Map.merge(%{"skip_preload" => true}, opts)) |> limit(1) |> select([a], a.id) |> Repo.one() From 4030837d91b7ff8525513589d5d810e9a1a6b959 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:19:47 +0000 Subject: [PATCH 24/64] notification: add non_follows/non_followers notification control settings --- lib/pleroma/notification.ex | 32 +++++++++++++++++++++++++++++++- lib/pleroma/user/info.ex | 11 +++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 844264307..4095e0474 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -166,7 +166,17 @@ defmodule Pleroma.Notification do def get_notified_from_activity(_, _local_only), do: [] def skip?(activity, user) do - [:self, :blocked, :local, :muted, :followers, :follows, :recently_followed] + [ + :self, + :blocked, + :local, + :muted, + :followers, + :follows, + :non_followers, + :non_follows, + :recently_followed + ] |> Enum.any?(&skip?(&1, activity, user)) end @@ -201,12 +211,32 @@ defmodule Pleroma.Notification do User.following?(follower, user) end + def skip?( + :non_followers, + activity, + %{info: %{notification_settings: %{"non_followers" => false}}} = user + ) do + actor = activity.data["actor"] + follower = User.get_cached_by_ap_id(actor) + !User.following?(follower, user) + end + def skip?(:follows, activity, %{info: %{notification_settings: %{"follows" => false}}} = user) do actor = activity.data["actor"] followed = User.get_cached_by_ap_id(actor) User.following?(user, followed) end + def skip?( + :non_follows, + activity, + %{info: %{notification_settings: %{"non_follows" => false}}} = user + ) do + actor = activity.data["actor"] + followed = User.get_cached_by_ap_id(actor) + !User.following?(user, followed) + end + def skip?(:recently_followed, %{data: %{"type" => "Follow"}} = activity, user) do actor = activity.data["actor"] diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 6397e2737..fb4cf3cc3 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -47,7 +47,14 @@ defmodule Pleroma.User.Info do field(:emoji, {:array, :map}, default: []) field(:notification_settings, :map, - default: %{"remote" => true, "local" => true, "followers" => true, "follows" => true} + default: %{ + "remote" => true, + "local" => true, + "followers" => true, + "follows" => true, + "non_follows" => true, + "non_followers" => true + } ) # Found in the wild @@ -71,7 +78,7 @@ defmodule Pleroma.User.Info do notification_settings = info.notification_settings |> Map.merge(settings) - |> Map.take(["remote", "local", "followers", "follows"]) + |> Map.take(["remote", "local", "followers", "follows", "non_follows", "non_followers"]) params = %{notification_settings: notification_settings} From 1542cccbbcc2935add83d1428b4cd9e4b146f1ec Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:22:13 +0000 Subject: [PATCH 25/64] tests: chase notification setting changes --- test/web/mastodon_api/account_view_test.exs | 4 +++- test/web/twitter_api/util_controller_test.exs | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index aaf2261bb..6f8480ee2 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -81,7 +81,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do "remote" => true, "local" => true, "followers" => true, - "follows" => true + "follows" => true, + "non_follows" => true, + "non_followers" => true } privacy = user.info.default_scope diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 2cd82b3e7..ca0b8cc26 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -110,8 +110,14 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do user = Repo.get(User, user.id) - assert %{"remote" => false, "local" => true, "followers" => false, "follows" => true} == - user.info.notification_settings + assert %{ + "remote" => false, + "local" => true, + "followers" => false, + "follows" => true, + "non_follows" => true, + "non_followers" => true + } == user.info.notification_settings end end From 0f7eeb0943472da80405639f17b9bc99e4fcd417 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:25:40 +0000 Subject: [PATCH 26/64] tests: add tests for non-follows/non-followers settings --- test/notification_test.exs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/notification_test.exs b/test/notification_test.exs index 581db58a8..b54414dcd 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -113,6 +113,13 @@ defmodule Pleroma.NotificationTest do assert nil == Notification.create_notification(activity, followed) end + test "it disables notifications from non-followers" do + follower = insert(:user) + followed = insert(:user, info: %{notification_settings: %{"non_followers" => false}}) + {:ok, activity} = CommonAPI.post(follower, %{"status" => "hey @#{followed.nickname}"}) + assert nil == Notification.create_notification(activity, followed) + end + test "it disables notifications from people the user follows" do follower = insert(:user, info: %{notification_settings: %{"follows" => false}}) followed = insert(:user) @@ -122,6 +129,13 @@ defmodule Pleroma.NotificationTest do assert nil == Notification.create_notification(activity, follower) end + test "it disables notifications from people the user does not follow" do + follower = insert(:user, info: %{notification_settings: %{"non_follows" => false}}) + followed = insert(:user) + {:ok, activity} = CommonAPI.post(followed, %{"status" => "hey @#{follower.nickname}"}) + assert nil == Notification.create_notification(activity, follower) + end + test "it doesn't create a notification for user if he is the activity author" do activity = insert(:note_activity) author = User.get_cached_by_ap_id(activity.data["actor"]) From 59a703fcbe6436c92d0e276caaf55f599c3165f4 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:31:13 +0000 Subject: [PATCH 27/64] twitter api: user view: expose user notification settings under pleroma object --- lib/pleroma/web/twitter_api/views/user_view.ex | 7 +++++++ test/web/twitter_api/views/user_view_test.exs | 2 ++ 2 files changed, 9 insertions(+) diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex index f0a4ddbd3..550f35f5f 100644 --- a/lib/pleroma/web/twitter_api/views/user_view.ex +++ b/lib/pleroma/web/twitter_api/views/user_view.ex @@ -121,6 +121,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do "tags" => user.tags } |> maybe_with_activation_status(user, for_user) + |> with_notification_settings(user, for_user) } |> maybe_with_user_settings(user, for_user) |> maybe_with_role(user, for_user) @@ -132,6 +133,12 @@ defmodule Pleroma.Web.TwitterAPI.UserView do end end + defp with_notification_settings(data, %User{id: user_id} = user, %User{id: user_id}) do + Map.put(data, "notification_settings", user.info.notification_settings) + end + + defp with_notification_settings(data, _, _), do: data + defp maybe_with_activation_status(data, user, %User{info: %{is_admin: true}}) do Map.put(data, "deactivated", user.info.deactivated) end diff --git a/test/web/twitter_api/views/user_view_test.exs b/test/web/twitter_api/views/user_view_test.exs index 74526673c..a48fc9b78 100644 --- a/test/web/twitter_api/views/user_view_test.exs +++ b/test/web/twitter_api/views/user_view_test.exs @@ -112,9 +112,11 @@ defmodule Pleroma.Web.TwitterAPI.UserViewTest do as_user = UserView.render("show.json", %{user: user, for: user}) assert as_user["default_scope"] == user.info.default_scope assert as_user["no_rich_text"] == user.info.no_rich_text + assert as_user["pleroma"]["notification_settings"] == user.info.notification_settings as_stranger = UserView.render("show.json", %{user: user}) refute as_stranger["default_scope"] refute as_stranger["no_rich_text"] + refute as_stranger["pleroma"]["notification_settings"] end test "A user for a given other follower", %{user: user} do From e7e2e7a1a633e2f7e493d040e290c931320d8cc8 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:54:02 +0000 Subject: [PATCH 28/64] user info: allow formdata for notification settings like every other API --- lib/pleroma/user/info.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index fb4cf3cc3..b0bfdf4f4 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -75,6 +75,11 @@ defmodule Pleroma.User.Info do end def update_notification_settings(info, settings) do + settings = + settings + |> Enum.map(fn {k, v} -> {k, v in [true, "true", "True", "1"]} end) + |> Map.new() + notification_settings = info.notification_settings |> Map.merge(settings) From 5fbbc57c1b497f8d194f3bbdefd889f0a943525a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 07:25:13 +0000 Subject: [PATCH 29/64] add migration to add notification settings to user accounts --- ...d_non_followers_fields_to_notification_settings.exs | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs diff --git a/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs b/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs new file mode 100644 index 000000000..a88b0ea61 --- /dev/null +++ b/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.AddNonFollowsAndNonFollowersFieldsToNotificationSettings do + use Ecto.Migration + + def change do + execute(""" + update users set info = jsonb_set(info, '{notification_settings}', '{"local": true, "remote": true, "follows": true, "followers": true, "non_follows": true, "non_followers": true}') + where local=true + """) + end +end From 750ede5764d30063587182696f7ebc50c05f8278 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 26 May 2019 00:05:47 +0000 Subject: [PATCH 30/64] notification: remove local/remote match rules (too complicated) --- lib/pleroma/notification.ex | 7 ------- lib/pleroma/user/info.ex | 4 +--- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 4095e0474..35beffb87 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -169,7 +169,6 @@ defmodule Pleroma.Notification do [ :self, :blocked, - :local, :muted, :followers, :follows, @@ -189,12 +188,6 @@ defmodule Pleroma.Notification do User.blocks?(user, %{ap_id: actor}) end - def skip?(:local, %{local: true}, %{info: %{notification_settings: %{"local" => false}}}), - do: true - - def skip?(:local, %{local: false}, %{info: %{notification_settings: %{"remote" => false}}}), - do: true - def skip?(:muted, activity, user) do actor = activity.data["actor"] diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index b0bfdf4f4..74573ba83 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -48,8 +48,6 @@ defmodule Pleroma.User.Info do field(:notification_settings, :map, default: %{ - "remote" => true, - "local" => true, "followers" => true, "follows" => true, "non_follows" => true, @@ -83,7 +81,7 @@ defmodule Pleroma.User.Info do notification_settings = info.notification_settings |> Map.merge(settings) - |> Map.take(["remote", "local", "followers", "follows", "non_follows", "non_followers"]) + |> Map.take(["followers", "follows", "non_follows", "non_followers"]) params = %{notification_settings: notification_settings} From 45e4642a58f5299d2cd3f142aea110a474eb477f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 26 May 2019 00:20:54 +0000 Subject: [PATCH 31/64] tests: chase remote/local removal --- test/notification_test.exs | 27 ------------------- test/web/mastodon_api/account_view_test.exs | 2 -- test/web/twitter_api/util_controller_test.exs | 3 --- 3 files changed, 32 deletions(-) diff --git a/test/notification_test.exs b/test/notification_test.exs index b54414dcd..be292abd9 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -78,33 +78,6 @@ defmodule Pleroma.NotificationTest do assert nil == Notification.create_notification(activity, muter) end - test "it disables notifications from people on remote instances" do - user = insert(:user, info: %{notification_settings: %{"remote" => false}}) - other_user = insert(:user) - - create_activity = %{ - "@context" => "https://www.w3.org/ns/activitystreams", - "type" => "Create", - "to" => ["https://www.w3.org/ns/activitystreams#Public"], - "actor" => other_user.ap_id, - "object" => %{ - "type" => "Note", - "content" => "Hi @#{user.nickname}", - "attributedTo" => other_user.ap_id - } - } - - {:ok, %{local: false} = activity} = Transmogrifier.handle_incoming(create_activity) - assert nil == Notification.create_notification(activity, user) - end - - test "it disables notifications from people on the local instance" do - user = insert(:user, info: %{notification_settings: %{"local" => false}}) - other_user = insert(:user) - {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hey @#{user.nickname}"}) - assert nil == Notification.create_notification(activity, user) - end - test "it disables notifications from followers" do follower = insert(:user) followed = insert(:user, info: %{notification_settings: %{"followers" => false}}) diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index 6f8480ee2..23f250990 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -78,8 +78,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do user = insert(:user) notification_settings = %{ - "remote" => true, - "local" => true, "followers" => true, "follows" => true, "non_follows" => true, diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index ca0b8cc26..cab9e5d90 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -102,7 +102,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do conn |> assign(:user, user) |> put("/api/pleroma/notification_settings", %{ - "remote" => false, "followers" => false, "bar" => 1 }) @@ -111,8 +110,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do user = Repo.get(User, user.id) assert %{ - "remote" => false, - "local" => true, "followers" => false, "follows" => true, "non_follows" => true, From 97fb50d9faaea509e3e7809689f4aa045634fa81 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 31 May 2019 11:27:14 +0200 Subject: [PATCH 32/64] Mastodon Conversation API: Don't return own account in 'accounts'. --- lib/pleroma/web/mastodon_api/views/conversation_view.ex | 7 ++++++- test/web/mastodon_api/mastodon_api_controller_test.exs | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index 8e8f7cf31..af1dcf66d 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -22,9 +22,14 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do last_status = StatusView.render("status.json", %{activity: activity, for: user}) + # Conversations return all users except the current user. + users = + participation.conversation.users + |> Enum.reject(&(&1.id == user.id)) + accounts = AccountView.render("accounts.json", %{ - users: participation.conversation.users, + users: users, as: :user }) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 1d9f5a816..24cc911c3 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -317,12 +317,13 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do test "Conversations", %{conn: conn} do user_one = insert(:user) user_two = insert(:user) + user_three = insert(:user) {:ok, user_two} = User.follow(user_two, user_one) {:ok, direct} = CommonAPI.post(user_one, %{ - "status" => "Hi @#{user_two.nickname}!", + "status" => "Hi @#{user_two.nickname}, @#{user_three.nickname}!", "visibility" => "direct" }) @@ -348,7 +349,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do } ] = response + account_ids = Enum.map(res_accounts, & &1["id"]) assert length(res_accounts) == 2 + assert user_two.id in account_ids + assert user_three.id in account_ids assert is_binary(res_id) assert unread == true assert res_last_status["id"] == direct.id From eb2963bc43f2cb195c2f19e6081c3faa6375fe4e Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 31 May 2019 14:27:15 +0200 Subject: [PATCH 33/64] User: Add settings store to Info, AccountView This is to provide a generic frontend settings storage mechanism for all kinds of frontends. --- lib/pleroma/user/info.ex | 1 + .../web/mastodon_api/views/account_view.ex | 10 ++++++++++ test/web/mastodon_api/account_view_test.exs | 15 +++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 6397e2737..e6623160d 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -45,6 +45,7 @@ defmodule Pleroma.User.Info do field(:flavour, :string, default: nil) field(:mascot, :map, default: nil) field(:emoji, {:array, :map}, default: []) + field(:pleroma_settings_store, :map, default: %{}) field(:notification_settings, :map, default: %{"remote" => true, "local" => true, "followers" => true, "follows" => true} diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index b82d3319b..04d485340 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -130,6 +130,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do |> maybe_put_role(user, opts[:for]) |> maybe_put_settings(user, opts[:for], user_info) |> maybe_put_notification_settings(user, opts[:for]) + |> maybe_put_settings_store(user, opts[:for], opts) end defp username_from_nickname(string) when is_binary(string) do @@ -152,6 +153,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do defp maybe_put_settings(data, _, _, _), do: data + defp maybe_put_settings_store(data, %User{info: info, id: id}, %User{id: id}, %{ + with_pleroma_settings: true + }) do + data + |> Kernel.put_in([:pleroma, :settings], info.pleroma_settings_store) + end + + defp maybe_put_settings_store(data, _, _, _), do: data + defp maybe_put_role(data, %User{info: %{show_role: true}} = user, _) do data |> Kernel.put_in([:pleroma, :is_admin], user.info.is_admin) diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index aaf2261bb..ca73d6581 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -239,4 +239,19 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do assert expected == AccountView.render("account.json", %{user: user, for: other_user}) end + + test "returns the settings store if the requesting user is the represented user and it's requested specifically" do + user = insert(:user, %{info: %User.Info{pleroma_settings_store: %{fe: "test"}}}) + + result = + AccountView.render("account.json", %{user: user, for: user, with_pleroma_settings: true}) + + assert result.pleroma.settings == %{:fe => "test"} + + result = AccountView.render("account.json", %{user: user, with_pleroma_settings: true}) + assert result.pleroma[:settings] == nil + + result = AccountView.render("account.json", %{user: user, for: user}) + assert result.pleroma[:settings] == nil + end end From aaad85c4d908dd14c1d836ebe2a5302cfdfaff2e Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 31 May 2019 14:49:46 +0200 Subject: [PATCH 34/64] AccountView: settings -> settings_store --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- test/web/mastodon_api/account_view_test.exs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 04d485340..dc32a1525 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -157,7 +157,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do with_pleroma_settings: true }) do data - |> Kernel.put_in([:pleroma, :settings], info.pleroma_settings_store) + |> Kernel.put_in([:pleroma, :settings_store], info.pleroma_settings_store) end defp maybe_put_settings_store(data, _, _, _), do: data diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index ca73d6581..5e6f1d00b 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -246,12 +246,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do result = AccountView.render("account.json", %{user: user, for: user, with_pleroma_settings: true}) - assert result.pleroma.settings == %{:fe => "test"} + assert result.pleroma.settings_store == %{:fe => "test"} result = AccountView.render("account.json", %{user: user, with_pleroma_settings: true}) - assert result.pleroma[:settings] == nil + assert result.pleroma[:settings_store] == nil result = AccountView.render("account.json", %{user: user, for: user}) - assert result.pleroma[:settings] == nil + assert result.pleroma[:settings_store] == nil end end From 7861974ab2cda55a97992f76e48ed082b234a7cf Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 31 May 2019 14:50:18 +0200 Subject: [PATCH 35/64] MastodonAPI: Add extension to set and get pleroma_settings_store. --- lib/pleroma/user/info.ex | 3 +- .../mastodon_api/mastodon_api_controller.ex | 12 +++- .../mastodon_api_controller_test.exs | 60 +++++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index e6623160d..37cdf2fa7 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -210,7 +210,8 @@ defmodule Pleroma.User.Info do :hide_followers, :hide_favorites, :background, - :show_role + :show_role, + :pleroma_settings_store ]) end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 2110027c3..ce3e149cc 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -124,6 +124,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end) end) |> add_if_present(params, "default_scope", :default_scope) + |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store, fn value -> + {:ok, Map.merge(user.info.pleroma_settings_store, value)} + end) |> add_if_present(params, "header", :banner, fn value -> with %Plug.Upload{} <- value, {:ok, object} <- ActivityPub.upload(value, type: :banner) do @@ -143,7 +146,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do CommonAPI.update(user) end - json(conn, AccountView.render("account.json", %{user: user, for: user})) + json( + conn, + AccountView.render("account.json", %{user: user, for: user, with_pleroma_settings: true}) + ) else _e -> conn @@ -153,7 +159,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def verify_credentials(%{assigns: %{user: user}} = conn, _) do - account = AccountView.render("account.json", %{user: user, for: user}) + account = + AccountView.render("account.json", %{user: user, for: user, with_pleroma_settings: true}) + json(conn, account) end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 93ef630f2..59a7967bd 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2322,6 +2322,66 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do end describe "updating credentials" do + test "sets user settings in a generic way", %{conn: conn} do + user = insert(:user) + + res_conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{ + "pleroma_settings_store" => %{ + pleroma_fe: %{ + theme: "bla" + } + } + }) + + assert user = json_response(res_conn, 200) + assert user["pleroma"]["settings_store"] == %{"pleroma_fe" => %{"theme" => "bla"}} + + user = Repo.get(User, user["id"]) + + res_conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{ + "pleroma_settings_store" => %{ + masto_fe: %{ + theme: "bla" + } + } + }) + + assert user = json_response(res_conn, 200) + + assert user["pleroma"]["settings_store"] == + %{ + "pleroma_fe" => %{"theme" => "bla"}, + "masto_fe" => %{"theme" => "bla"} + } + + user = Repo.get(User, user["id"]) + + res_conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{ + "pleroma_settings_store" => %{ + masto_fe: %{ + theme: "blub" + } + } + }) + + assert user = json_response(res_conn, 200) + + assert user["pleroma"]["settings_store"] == + %{ + "pleroma_fe" => %{"theme" => "bla"}, + "masto_fe" => %{"theme" => "blub"} + } + end + test "updates the user's bio", %{conn: conn} do user = insert(:user) user2 = insert(:user) From 10fe02acefca47e6013c4b0f70e4077a6d59d488 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 31 May 2019 14:58:28 +0200 Subject: [PATCH 36/64] Documentation: Document Settings store mechanism. --- docs/api/differences_in_mastoapi_responses.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 36b47608e..21c1b76e5 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -43,6 +43,7 @@ Has these additional fields under the `pleroma` object: - `confirmation_pending`: boolean, true if a new user account is waiting on email confirmation to be activated - `hide_followers`: boolean, true when the user has follower hiding enabled - `hide_follows`: boolean, true when the user has follow hiding enabled +- `settings_store`: A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials` ### Source @@ -80,6 +81,14 @@ Additional parameters can be added to the JSON body/Form data: - `hide_favorites` - if true, user's favorites timeline will be hidden - `show_role` - if true, user's role (e.g admin, moderator) will be exposed to anyone in the API - `default_scope` - the scope returned under `privacy` key in Source subentity +- `pleroma_settings_store` - Opaque user settings to be saved on the backend. + +### Pleroma Settings Store +Pleroma has mechanism that allows frontends to save blobs of json for each user on the backend. This can be used to save frontend-specific settings for a user that the backend does not need to know about. + +The parameter should have a form of `{frontend_name: {...}}`, with `frontend_name` identifying your type of client, e.g. `pleroma_fe`. It will overwrite everything under this property, but will not overwrite other frontend's settings. + +This information is returned in the `verify_credentials` endpoint. ## Authentication From 7591a8928d7ca440dbc2d45428988208ef617bdc Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 31 May 2019 19:15:44 +0200 Subject: [PATCH 37/64] Setting Store: Document in changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4abb6eb8..f8ed529ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Added +- Add a generic settings store for frontends / clients to use. - Optional SSH access mode. (Needs `erlang-ssh` package on some distributions). - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication From 300d94c62829d0ec961f3ed6c0242dea102ab0ad Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 1 Jun 2019 16:07:01 +0300 Subject: [PATCH 38/64] Add poll votes Also in this commit by accident: - Fix query ordering causing exclude_poll_votes to not work - Do not create notifications for Answer objects --- lib/pleroma/notification.ex | 11 ++- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/activity_pub/utils.ex | 17 +++++ lib/pleroma/web/common_api/common_api.ex | 46 +++++++++++ lib/pleroma/web/common_api/utils.ex | 11 +++ .../mastodon_api/mastodon_api_controller.ex | 27 +++++++ lib/pleroma/web/router.ex | 2 + .../mastodon_api_controller_test.exs | 76 +++++++++++++++++++ 8 files changed, 188 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 844264307..80e2800ae 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -127,10 +127,15 @@ defmodule Pleroma.Notification do def create_notifications(%Activity{data: %{"to" => _, "type" => type}} = activity) when type in ["Create", "Like", "Announce", "Follow"] do - users = get_notified_from_activity(activity) + object = Object.normalize(activity) - notifications = Enum.map(users, fn user -> create_notification(activity, user) end) - {:ok, notifications} + unless object && object.data["type"] == "Answer" do + users = get_notified_from_activity(activity) + notifications = Enum.map(users, fn user -> create_notification(activity, user) end) + {:ok, notifications} + else + {:ok, []} + end end def create_notifications(_), do: {:ok, []} diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index b06200cb5..5a11dadcf 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -878,7 +878,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> maybe_preload_objects(opts) |> maybe_preload_bookmarks(opts) |> maybe_order(opts) - |> exclude_poll_votes(opts) |> restrict_recipients(recipients, opts["user"]) |> restrict_tag(opts) |> restrict_tag_reject(opts) @@ -899,6 +898,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_pinned(opts) |> restrict_muted_reblogs(opts) |> Activity.restrict_deactivated_users() + |> exclude_poll_votes(opts) end def fetch_activities(recipients, opts \\ %{}) do diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 9646bbee9..b292d7d8d 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -789,4 +789,21 @@ defmodule Pleroma.Web.ActivityPub.Utils do [to, cc, recipients] end end + + def get_existing_votes(actor, %{data: %{"id" => id}}) do + query = + from( + [activity, object: object] in Activity.with_preloaded_object(Activity), + where: fragment("(?)->>'actor' = ?", activity.data, ^actor), + where: + fragment( + "(?)->'inReplyTo' = ?", + object.data, + ^to_string(id) + ), + where: fragment("(?)->>'type' = 'Answer'", object.data) + ) + + Repo.all(query) + end end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 374967a1b..f54f8a7b9 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -119,6 +119,52 @@ defmodule Pleroma.Web.CommonAPI do end end + def vote(user, object, choices) do + with "Question" <- object.data["type"], + {:author, false} <- {:author, object.data["actor"] == user.ap_id}, + {:existing_votes, []} <- {:existing_votes, Utils.get_existing_votes(user.ap_id, object)}, + {options, max_count} <- get_options_and_max_count(object), + option_count <- Enum.count(options), + {:choice_check, {choices, true}} <- + {:choice_check, normalize_and_validate_choice_indices(choices, option_count)}, + {:count_check, true} <- {:count_check, Enum.count(choices) <= max_count} do + answer_activities = + Enum.map(choices, fn index -> + answer_data = make_answer_data(user, object, Enum.at(options, index)["name"]) + + ActivityPub.create(%{ + to: answer_data["to"], + actor: user, + context: object.data["context"], + object: answer_data, + additional: %{"cc" => answer_data["cc"]} + }) + end) + + {:ok, answer_activities, object} + else + {:author, _} -> {:error, "Already voted"} + {:existing_votes, _} -> {:error, "Already voted"} + {:choice_check, {_, false}} -> {:error, "Invalid indices"} + {:count_check, false} -> {:error, "Too many choices"} + end + end + + defp get_options_and_max_count(object) do + if Map.has_key?(object.data, "anyOf") do + {object.data["anyOf"], Enum.count(object.data["anyOf"])} + else + {object.data["oneOf"], 1} + end + end + + defp normalize_and_validate_choice_indices(choices, count) do + Enum.map_reduce(choices, true, fn index, valid -> + index = if is_binary(index), do: String.to_integer(index), else: index + {index, if(valid, do: index < count, else: valid)} + end) + end + def get_visibility(%{"visibility" => visibility}, in_reply_to) when visibility in ~w{public unlisted private direct}, do: {visibility, get_replied_to_visibility(in_reply_to)} diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1a239de97..f35ed36ab 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -491,4 +491,15 @@ defmodule Pleroma.Web.CommonAPI.Utils do {:error, "No such conversation"} end end + + def make_answer_data(%User{ap_id: ap_id}, object, name) do + %{ + "type" => "Answer", + "actor" => ap_id, + "cc" => [object.data["actor"]], + "to" => [], + "name" => name, + "inReplyTo" => object.data["id"] + } + end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index ecb7df459..13bb609e5 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -430,6 +430,33 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end end + def poll_vote(%{assigns: %{user: user}} = conn, %{"id" => id, "choices" => choices}) do + with %Object{} = object <- Object.get_by_id(id), + true <- object.data["type"] == "Question", + %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), + true <- Visibility.visible_for_user?(activity, user), + {:ok, _activities, object} <- CommonAPI.vote(user, object, choices) do + conn + |> put_view(StatusView) + |> try_render("poll.json", %{object: object, for: user}) + else + nil -> + conn + |> put_status(404) + |> json(%{error: "Record not found"}) + + false -> + conn + |> put_status(404) + |> json(%{error: "Record not found"}) + + {:error, message} -> + conn + |> put_status(422) + |> json(%{error: message}) + end + end + def scheduled_statuses(%{assigns: %{user: user}} = conn, params) do with scheduled_activities <- MastodonAPI.get_scheduled_activities(user, params) do conn diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e0611e3fc..d17f58f52 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -335,6 +335,8 @@ defmodule Pleroma.Web.Router do put("/scheduled_statuses/:id", MastodonAPIController, :update_scheduled_status) delete("/scheduled_statuses/:id", MastodonAPIController, :delete_scheduled_status) + post("/polls/:id/votes", MastodonAPIController, :poll_vote) + post("/media", MastodonAPIController, :upload) put("/media/:id", MastodonAPIController, :update_media) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 0d56b6ff2..b160a4db0 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3497,4 +3497,80 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert json_response(conn, 404) end end + + describe "POST /api/v1/polls/:id/votes" do + test "votes are added to the poll", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "A very delicious sandwich", + "poll" => %{ + "options" => ["Lettuce", "Grilled Bacon", "Tomato"], + "expires_in" => 20, + "multiple" => true + } + }) + + object = Object.normalize(activity) + + conn = + conn + |> assign(:user, other_user) + |> post("/api/v1/polls/#{object.id}/votes", %{"choices" => [0, 1, 2]}) + + assert json_response(conn, 200) + object = Object.get_by_id(object.id) + + assert Enum.all?(object.data["anyOf"], fn %{"replies" => %{"totalItems" => totalItems}} -> + totalItems == 1 + end) + end + + test "author can't vote", %{conn: conn} do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "Am I cute?", + "poll" => %{"options" => ["Yes", "No"], "expires_in" => 20} + }) + + object = Object.normalize(activity) + + assert conn + |> assign(:user, user) + |> post("/api/v1/polls/#{object.id}/votes", %{"choices" => [1]}) + |> json_response(422) == %{"error" => "Already voted"} + + object = Object.get_by_id(object.id) + + refute Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 1 + end + + test "does not allow multiple choices on a single-choice question", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "The glass is", + "poll" => %{"options" => ["half empty", "half full"], "expires_in" => 20} + }) + + object = Object.normalize(activity) + + assert conn + |> assign(:user, other_user) + |> post("/api/v1/polls/#{object.id}/votes", %{"choices" => [0, 1]}) + |> json_response(422) == %{"error" => "Too many choices"} + + object = Object.get_by_id(object.id) + + refute Enum.any?(object.data["oneOf"], fn %{"replies" => %{"totalItems" => totalItems}} -> + totalItems == 1 + end) + end + end end From 444406167b050524efb016cfee78636f7f6828ca Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 1 Jun 2019 21:41:23 +0300 Subject: [PATCH 39/64] Mastodon API: actually check for poll votes --- lib/pleroma/web/mastodon_api/views/status_view.ex | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 5bf4a6ba2..7eea0122b 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -331,7 +331,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end # TODO: Add tests for this view - def render("poll.json", %{object: object} = _opts) do + def render("poll.json", %{object: object} = opts) do {multiple, options} = case object.data do %{"anyOf" => options} when is_list(options) -> {true, options} @@ -352,6 +352,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do _ -> false end + voted = + if opts[:for] do + existing_votes = + Pleroma.Web.ActivityPub.Utils.get_existing_votes(opts[:for].ap_id, object) + + existing_votes != [] or opts[:for].ap_id == object.data["actor"] + else + false + end + {options, votes_count} = Enum.map_reduce(options, 0, fn %{"name" => name} = option, count -> current_count = option["replies"]["totalItems"] || 0 @@ -371,8 +381,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do multiple: multiple, votes_count: votes_count, options: options, - # TODO: Actually check for a vote - voted: false, + voted: voted, emojis: build_emojis(object.data["emoji"]) } else From 6bc9e5c020d97edcaaa386bd2ec8e8ec57cb19af Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 1 Jun 2019 21:41:49 +0300 Subject: [PATCH 40/64] Mastodon API: Refresh the object before rendering it after voting --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index bab6d693d..8da31161f 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -436,6 +436,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), true <- Visibility.visible_for_user?(activity, user), {:ok, _activities, object} <- CommonAPI.vote(user, object, choices) do + object = Object.get_cached_by_ap_id(object.data["id"]) + conn |> put_view(StatusView) |> try_render("poll.json", %{object: object, for: user}) From cfa588e3574f3083798de6f4e0eca63f00aaf578 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 1 Jun 2019 21:42:29 +0300 Subject: [PATCH 41/64] Fix Credo issues --- test/web/mastodon_api/mastodon_api_controller_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 24fd319d3..9c1db37db 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3630,8 +3630,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert json_response(conn, 200) object = Object.get_by_id(object.id) - assert Enum.all?(object.data["anyOf"], fn %{"replies" => %{"totalItems" => totalItems}} -> - totalItems == 1 + assert Enum.all?(object.data["anyOf"], fn %{"replies" => %{"totalItems" => total_items}} -> + total_items == 1 end) end @@ -3675,8 +3675,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do object = Object.get_by_id(object.id) - refute Enum.any?(object.data["oneOf"], fn %{"replies" => %{"totalItems" => totalItems}} -> - totalItems == 1 + refute Enum.any?(object.data["oneOf"], fn %{"replies" => %{"totalItems" => total_items}} -> + total_items == 1 end) end end From 67bcc3ccc433bd5c7b59f5bf1a45db93da93d093 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 1 Jun 2019 21:49:18 +0300 Subject: [PATCH 42/64] Remove index on object name used for excluding poll votes (replaced by just rewriting Note to Answer) --- .../migrations/20190522084924_add_object_index_on_name.exs | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 priv/repo/migrations/20190522084924_add_object_index_on_name.exs diff --git a/priv/repo/migrations/20190522084924_add_object_index_on_name.exs b/priv/repo/migrations/20190522084924_add_object_index_on_name.exs deleted file mode 100644 index 886e8499e..000000000 --- a/priv/repo/migrations/20190522084924_add_object_index_on_name.exs +++ /dev/null @@ -1,7 +0,0 @@ -defmodule Pleroma.Repo.Migrations.AddObjectIndexOnName do - use Ecto.Migration - - def change do - create(index(:objects, ["(data->'name')"], name: :objects_name_index, using: :gin)) - end -end From 52e09807d4b118b17c2f504f6224218a047440d8 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 2 Jun 2019 09:09:58 +0000 Subject: [PATCH 43/64] reverse proxy: clean up some @hackney leftovers --- lib/pleroma/reverse_proxy.ex | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index 983e156f5..285d57309 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -61,8 +61,6 @@ defmodule Pleroma.ReverseProxy do * `http`: options for [hackney](https://github.com/benoitc/hackney). """ - @hackney Pleroma.Config.get(:hackney, :hackney) - @default_hackney_options [] @inline_content_types [ @@ -148,7 +146,7 @@ defmodule Pleroma.ReverseProxy do Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}") method = method |> String.downcase() |> String.to_existing_atom() - case @hackney.request(method, url, headers, "", hackney_opts) do + case :hackney.request(method, url, headers, "", hackney_opts) do {:ok, code, headers, client} when code in @valid_resp_codes -> {:ok, code, downcase_headers(headers), client} @@ -198,7 +196,7 @@ defmodule Pleroma.ReverseProxy do duration, Keyword.get(opts, :max_read_duration, @max_read_duration) ), - {:ok, data} <- @hackney.stream_body(client), + {:ok, data} <- :hackney.stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_size)), From edf772d41ea7b44d7286e442061e94a347167be2 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 2 Jun 2019 09:44:42 +0000 Subject: [PATCH 44/64] mrf: allow a policy chain to be specified when filtering --- lib/pleroma/web/activity_pub/mrf.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 3bf7955f3..10ceef715 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.ActivityPub.MRF do @callback filter(Map.t()) :: {:ok | :reject, Map.t()} - def filter(object) do - get_policies() + def filter(policies, %{} = object) do + policies |> Enum.reduce({:ok, object}, fn policy, {:ok, object} -> policy.filter(object) @@ -16,6 +16,8 @@ defmodule Pleroma.Web.ActivityPub.MRF do end) end + def filter(%{} = object), do: get_policies() |> filter(object) + def get_policies do Pleroma.Config.get([:instance, :rewrite_policy], []) |> get_policies() end From 4087ccdab8fc906fb7029e8f98651555e40fea4f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 2 Jun 2019 09:50:16 +0000 Subject: [PATCH 45/64] mrf: add subchain policy --- config/config.exs | 3 ++ .../web/activity_pub/mrf/subchain_policy.ex | 36 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/subchain_policy.ex diff --git a/config/config.exs b/config/config.exs index 68168b279..450da8930 100644 --- a/config/config.exs +++ b/config/config.exs @@ -320,6 +320,9 @@ config :pleroma, :mrf_keyword, federated_timeline_removal: [], replace: [] +config :pleroma, :mrf_subchain, + match_actor: %{} + config :pleroma, :rich_media, enabled: true config :pleroma, :media_proxy, diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex new file mode 100644 index 000000000..7fb4a607e --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do + alias Pleroma.Config + alias Pleroma.Web.ActivityPub.MRF + + require Logger + + @behaviour MRF + + defp lookup_subchain(actor) do + with matches <- Config.get([:mrf_subchain, :match_actor]), + {match, subchain} <- Enum.find(matches, fn {k, _v} -> String.match?(actor, k) end) do + {:ok, match, subchain} + else + _e -> {:error, :notfound} + end + end + + @impl true + def filter(%{"actor" => actor} = message) do + with {:ok, match, subchain} <- lookup_subchain(actor) do + Logger.debug("[SubchainPolicy] Matched #{actor} against #{inspect(match)} with subchain #{inspect(subchain)}") + + subchain + |> MRF.filter(message) + else + _e -> {:ok, message} + end + end + + @impl true + def filter(message), do: {:ok, message} +end From 38a275b31f33262658e5bfc8310f44ed00cae9db Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 2 Jun 2019 10:08:51 +0000 Subject: [PATCH 46/64] test: add tests for subchain policy --- .../activity_pub/mrf/subchain_policy_test.exs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 test/web/activity_pub/mrf/subchain_policy_test.exs diff --git a/test/web/activity_pub/mrf/subchain_policy_test.exs b/test/web/activity_pub/mrf/subchain_policy_test.exs new file mode 100644 index 000000000..f7cbcad48 --- /dev/null +++ b/test/web/activity_pub/mrf/subchain_policy_test.exs @@ -0,0 +1,32 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicyTest do + use Pleroma.DataCase + + alias Pleroma.Web.ActivityPub.MRF.DropPolicy + alias Pleroma.Web.ActivityPub.MRF.SubchainPolicy + + @message %{ + "actor" => "https://banned.com", + "type" => "Create", + "object" => %{"content" => "hi"} + } + + test "it matches and processes subchains when the actor matches a configured target" do + Pleroma.Config.put([:mrf_subchain, :match_actor], %{ + ~r/^https:\/\/banned.com/s => [DropPolicy] + }) + + {:reject, _} = SubchainPolicy.filter(@message) + end + + test "it doesn't match and process subchains when the actor doesn't match a configured target" do + Pleroma.Config.put([:mrf_subchain, :match_actor], %{ + ~r/^https:\/\/borked.com/s => [DropPolicy] + }) + + {:ok, _message} = SubchainPolicy.filter(@message) + end +end From c724d8df9831409df7990dfea3fd07ffb627a156 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 2 Jun 2019 10:14:56 +0000 Subject: [PATCH 47/64] docs: document mrf_subchain --- docs/config.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/config.md b/docs/config.md index 67b062fe9..5d9de647c 100644 --- a/docs/config.md +++ b/docs/config.md @@ -81,6 +81,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default) * `Pleroma.Web.ActivityPub.MRF.DropPolicy`: Drops all activities. It generally doesn’t makes sense to use in production * `Pleroma.Web.ActivityPub.MRF.SimplePolicy`: Restrict the visibility of activities from certains instances (See ``:mrf_simple`` section) + * `Pleroma.Web.ActivityPub.MRF.SubchainPolicy`: Selectively runs other MRF policies when messages match (see ``:mrf_subchain`` section) * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. * `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. @@ -224,6 +225,21 @@ relates to mascots on the mastodon frontend * `avatar_removal`: List of instances to strip avatars from * `banner_removal`: List of instances to strip banners from +## :mrf_subchain +This policy processes messages through an alternate pipeline when a given message matches certain criteria. +All criteria are configured as a map of regular expressions to lists of policy modules. + +* `match_actor`: Matches a series of regular expressions against the actor field. + +Example: + +``` +config :pleroma, :mrf_subchain, + match_actor: %{ + ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy] + } +``` + ## :mrf_rejectnonpublic * `allow_followersonly`: whether to allow followers-only posts * `allow_direct`: whether to allow direct messages From 561a21986d312f52bd1d06a477f1c88fd1adc727 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 2 Jun 2019 10:29:15 +0000 Subject: [PATCH 48/64] formatting --- config/config.exs | 3 +-- lib/pleroma/web/activity_pub/mrf/subchain_policy.ex | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/config/config.exs b/config/config.exs index 450da8930..a5bb05a80 100644 --- a/config/config.exs +++ b/config/config.exs @@ -320,8 +320,7 @@ config :pleroma, :mrf_keyword, federated_timeline_removal: [], replace: [] -config :pleroma, :mrf_subchain, - match_actor: %{} +config :pleroma, :mrf_subchain, match_actor: %{} config :pleroma, :rich_media, enabled: true diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex index 7fb4a607e..765704389 100644 --- a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex @@ -22,7 +22,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do @impl true def filter(%{"actor" => actor} = message) do with {:ok, match, subchain} <- lookup_subchain(actor) do - Logger.debug("[SubchainPolicy] Matched #{actor} against #{inspect(match)} with subchain #{inspect(subchain)}") + Logger.debug( + "[SubchainPolicy] Matched #{actor} against #{inspect(match)} with subchain #{ + inspect(subchain) + }" + ) subchain |> MRF.filter(message) From 83663caa81f1ccca37fe3898feb4ec2d829ad893 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 2 Jun 2019 17:45:32 +0300 Subject: [PATCH 49/64] Ueberauth: extended format of OAUTH_CONSUMER_STRATEGIES to allow explicit dependency specification. --- config/config.exs | 6 +++++- docs/config.md | 2 +- mix.exs | 25 ++++++++++++++++++------- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/config/config.exs b/config/config.exs index 68168b279..5f1a1d0f8 100644 --- a/config/config.exs +++ b/config/config.exs @@ -453,7 +453,11 @@ config :pleroma, :ldap, config :esshd, enabled: false -oauth_consumer_strategies = String.split(System.get_env("OAUTH_CONSUMER_STRATEGIES") || "") +oauth_consumer_strategies = + System.get_env("OAUTH_CONSUMER_STRATEGIES") + |> to_string() + |> String.split() + |> Enum.map(&hd(String.split(&1, ":"))) ueberauth_providers = for strategy <- oauth_consumer_strategies do diff --git a/docs/config.md b/docs/config.md index 67b062fe9..08088f269 100644 --- a/docs/config.md +++ b/docs/config.md @@ -492,7 +492,7 @@ Authentication / authorization settings. * `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`. * `oauth_consumer_template`: OAuth consumer mode authentication form template. By default it's `consumer.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex`. -* `oauth_consumer_strategies`: the list of enabled OAuth consumer strategies; by default it's set by OAUTH_CONSUMER_STRATEGIES environment variable. +* `oauth_consumer_strategies`: the list of enabled OAuth consumer strategies; by default it's set by OAUTH_CONSUMER_STRATEGIES environment variable. Each entry in this space-delimited string should be of format `` or `:` (e.g. `twitter` or `keycloak:ueberauth_keycloak_strategy` in case dependency is named differently than `ueberauth_`). ## OAuth consumer mode diff --git a/mix.exs b/mix.exs index b2017ef9b..df1a7ced4 100644 --- a/mix.exs +++ b/mix.exs @@ -51,16 +51,27 @@ defmodule Pleroma.Mixfile do defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] + # Specifies OAuth dependencies. + defp oauth_deps do + oauth_strategy_packages = + System.get_env("OAUTH_CONSUMER_STRATEGIES") + |> to_string() + |> String.split() + |> Enum.map(fn strategy_entry -> + with [_strategy, dependency] <- String.split(strategy_entry, ":") do + dependency + else + [strategy] -> "ueberauth_#{strategy}" + end + end) + + for s <- oauth_strategy_packages, do: {String.to_atom(s), ">= 0.0.0"} + end + # Specifies your project dependencies. # # Type `mix help deps` for examples and options. defp deps do - oauth_strategies = String.split(System.get_env("OAUTH_CONSUMER_STRATEGIES") || "") - - oauth_deps = - for s <- oauth_strategies, - do: {String.to_atom("ueberauth_#{s}"), ">= 0.0.0"} - [ {:phoenix, "~> 1.4.1"}, {:plug_cowboy, "~> 2.0"}, @@ -121,7 +132,7 @@ defmodule Pleroma.Mixfile do {:ex_rated, "~> 1.2"}, {:plug_static_index_html, "~> 1.0.0"}, {:excoveralls, "~> 0.11.1", only: :test} - ] ++ oauth_deps + ] ++ oauth_deps() end # Aliases are shortcuts or tasks specific to the current project. From e3c460353dacb796a867a0a3afa3632746d57aa2 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 2 Jun 2019 23:24:48 +0300 Subject: [PATCH 50/64] Refresh the object in CommonAPI.vote instead of MastoAPI controller --- lib/pleroma/web/common_api/common_api.ex | 1 + lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index f54f8a7b9..a12ee011b 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -141,6 +141,7 @@ defmodule Pleroma.Web.CommonAPI do }) end) + object = Object.get_cached_by_ap_id(object.data["id"]) {:ok, answer_activities, object} else {:author, _} -> {:error, "Already voted"} diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 8da31161f..bab6d693d 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -436,8 +436,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), true <- Visibility.visible_for_user?(activity, user), {:ok, _activities, object} <- CommonAPI.vote(user, object, choices) do - object = Object.get_cached_by_ap_id(object.data["id"]) - conn |> put_view(StatusView) |> try_render("poll.json", %{object: object, for: user}) From c47da0e65da32848b1f7e8cecce33a46d6f0115f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 2 Jun 2019 23:25:33 +0300 Subject: [PATCH 51/64] Add tests for poll view --- test/web/mastodon_api/status_view_test.exs | 102 +++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 9f2ebda4e..ec75150ab 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -342,4 +342,106 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do StatusView.render("card.json", %{page_url: page_url, rich_media: card}) end end + + describe "poll view" do + test "renders a poll" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "Is Tenshi eating a corndog cute?", + "poll" => %{ + "options" => ["absolutely!", "sure", "yes", "why are you even asking?"], + "expires_in" => 20 + } + }) + + object = Object.normalize(activity) + + expected = %{ + emojis: [], + expired: false, + id: object.id, + multiple: false, + options: [ + %{title: "absolutely!", votes_count: 0}, + %{title: "sure", votes_count: 0}, + %{title: "yes", votes_count: 0}, + %{title: "why are you even asking?", votes_count: 0} + ], + voted: false, + votes_count: 0 + } + + result = StatusView.render("poll.json", %{object: object}) + expires_at = result.expires_at + result = Map.delete(result, :expires_at) + + assert result == expected + + expires_at = NaiveDateTime.from_iso8601!(expires_at) + assert NaiveDateTime.diff(expires_at, NaiveDateTime.utc_now()) in 15..20 + end + + test "detects if it is multiple choice" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "Which Mastodon developer is your favourite?", + "poll" => %{ + "options" => ["Gargron", "Eugen"], + "expires_in" => 20, + "multiple" => true + } + }) + + object = Object.normalize(activity) + + assert %{multiple: true} = StatusView.render("poll.json", %{object: object}) + end + + test "detects emoji" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "What's with the smug face?", + "poll" => %{ + "options" => [":blank: sip", ":blank::blank: sip", ":blank::blank::blank: sip"], + "expires_in" => 20 + } + }) + + object = Object.normalize(activity) + + assert %{emojis: [%{shortcode: "blank"}]} = + StatusView.render("poll.json", %{object: object}) + end + + test "detects vote status" do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "Which input devices do you use?", + "poll" => %{ + "options" => ["mouse", "trackball", "trackpoint"], + "multiple" => true, + "expires_in" => 20 + } + }) + + object = Object.normalize(activity) + + {:ok, _, object} = CommonAPI.vote(other_user, object, [1, 2]) + + result = StatusView.render("poll.json", %{object: object, for: other_user}) + + assert result[:voted] == true + assert Enum.at(result[:options], 1)[:votes_count] == 1 + assert Enum.at(result[:options], 2)[:votes_count] == 1 + end + end end From 2fe3a20638789a8fb6e1a8e63cd5eb2247a9308a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 2 Jun 2019 23:30:36 +0300 Subject: [PATCH 52/64] Make error message about author's inability to vote more sensible --- lib/pleroma/web/common_api/common_api.ex | 2 +- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index a12ee011b..5212d5ce5 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -144,7 +144,7 @@ defmodule Pleroma.Web.CommonAPI do object = Object.get_cached_by_ap_id(object.data["id"]) {:ok, answer_activities, object} else - {:author, _} -> {:error, "Already voted"} + {:author, _} -> {:error, "Poll's author can't vote"} {:existing_votes, _} -> {:error, "Already voted"} {:choice_check, {_, false}} -> {:error, "Invalid indices"} {:count_check, false} -> {:error, "Too many choices"} diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 9c1db37db..c158c4df2 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3649,7 +3649,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert conn |> assign(:user, user) |> post("/api/v1/polls/#{object.id}/votes", %{"choices" => [1]}) - |> json_response(422) == %{"error" => "Already voted"} + |> json_response(422) == %{"error" => "Poll's author can't vote"} object = Object.get_by_id(object.id) From 1fd8e19d767e03475555e879c8e5e0fcc8d5c6bf Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 2 Jun 2019 23:46:17 +0300 Subject: [PATCH 53/64] Remove a TODO comment as the tests for poll view were written --- lib/pleroma/web/mastodon_api/views/status_view.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 7eea0122b..6836d331a 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -330,7 +330,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do } end - # TODO: Add tests for this view def render("poll.json", %{object: object} = opts) do {multiple, options} = case object.data do From baefb97dc4f4435027f6e23e77433e4bfaf0dcd1 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 3 Jun 2019 10:55:16 +0300 Subject: [PATCH 54/64] Add a changelog entry for polls --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff1fff876..2144bbe28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix Tasks: `mix pleroma.database remove_embedded_objects` - Mix Tasks: `mix pleroma.database update_users_following_followers_counts` - Mix Tasks: `mix pleroma.user toggle_confirmed` +- Federation: Support for `Question` and `Answer` objects - Federation: Support for reports +- Configuration: `poll_limits` option - Configuration: `safe_dm_mentions` option - Configuration: `link_name` option - Configuration: `fetch_initial_posts` option @@ -37,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: `/api/v1/pleroma/accounts/:id/favourites` (API extension) - Mastodon API: [Reports](https://docs.joinmastodon.org/api/rest/reports/) - Mastodon API: `POST /api/v1/accounts` (account creation API) +- Mastodon API: [Polls](https://docs.joinmastodon.org/api/rest/polls/) - ActivityPub C2S: OAuth endpoints - Metadata: RelMe provider - OAuth: added support for refresh tokens From 5bd41fef8b5aeff53ed6b096e04507d51c93a83a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 3 Jun 2019 10:58:37 +0300 Subject: [PATCH 55/64] Change query order in fetch_activities_for_context_query to make poll vote exclusion work --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index d494d5cd1..45feae25a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -489,7 +489,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do from(activity in Activity) |> maybe_preload_objects(opts) - |> exclude_poll_votes(opts) |> restrict_blocked(opts) |> restrict_recipients(recipients, opts["user"]) |> where( @@ -502,6 +501,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do ^context ) ) + |> exclude_poll_votes(opts) |> order_by([activity], desc: activity.id) end From c3bcc4f6a2d23574de2feafe8b649745a9b96179 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 3 Jun 2019 11:24:03 +0300 Subject: [PATCH 56/64] CI: Replace mix test and mix coveralls with just mix coveralls `mix coveralls` runs the tests by itself, exits with failure if one of them fails and accepts the same args, so there is no reason to run two of them at the same time. --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 53f05ae92..939175218 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,8 +52,7 @@ unit-testing: - mix deps.get - mix ecto.create - mix ecto.migrate - - mix test --trace --preload-modules - - mix coveralls + - mix coveralls --trace --preload-modules unit-testing-rum: stage: test From 9ded9443a3551360991dfa28ced07502037c4559 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 3 Jun 2019 11:47:10 +0200 Subject: [PATCH 57/64] CI: Actually push to correct repo. --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 939175218..58c9de167 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -121,8 +121,7 @@ review_app: - (ssh -t dokku@pleroma.online -- postgres:create $(echo $CI_ENVIRONMENT_SLUG | sed -e 's/-/_/g')_db) || true - (ssh -t dokku@pleroma.online -- postgres:link $(echo $CI_ENVIRONMENT_SLUG | sed -e 's/-/_/g')_db "$CI_ENVIRONMENT_SLUG") || true - (ssh -t dokku@pleroma.online -- certs:add "$CI_ENVIRONMENT_SLUG" /home/dokku/server.crt /home/dokku/server.key) || true - - (git remote add dokku dokku@pleroma.online:$CI_ENVIRONMENT_SLUG) || true - - git push -f dokku $CI_COMMIT_SHA:refs/heads/master + - git push -f dokku@pleroma.online:$CI_ENVIRONMENT_SLUG $CI_COMMIT_SHA:refs/heads/master stop_review_app: image: alpine:3.9 From 243d8ed94e031464a48621a55058882ba50de019 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Mon, 3 Jun 2019 18:00:32 +0300 Subject: [PATCH 58/64] Use workaround for the heavy checkmark symbol in iOS --- lib/pleroma/web/templates/layout/app.html.eex | 4 ++-- lib/pleroma/web/templates/o_auth/o_auth/show.html.eex | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index 818b3404b..98f7293bc 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -89,7 +89,7 @@ .scope:before { color: #b9b9ba; - content: "✔"; + content: "✔\fe0e"; margin-left: 1em; margin-right: 1em; } @@ -197,7 +197,7 @@ .scope:first-child:before { margin-left: 1em; - content: "✔"; + content: "✔\fe0e"; } .scope:after { diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index 8b69c3033..ed4fb5ce7 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -56,4 +56,9 @@ <%= hidden_input f, :response_type, value: @response_type %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> <%= hidden_input f, :state, value: @state %> + +<%= if Pleroma.Config.oauth_consumer_enabled?() do %> + <%= render @view_module, Pleroma.Web.Auth.Authenticator.oauth_consumer_template(), assigns %> +<% end %> + <% end %> From f2c4c99e0374a3657400d2e41fe8069f72a75337 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Mon, 3 Jun 2019 18:58:04 +0300 Subject: [PATCH 59/64] Remove repeated scope lists --- lib/pleroma/web/templates/layout/app.html.eex | 1 - .../templates/o_auth/o_auth/_scopes.html.eex | 16 ++++++++---- .../templates/o_auth/o_auth/consumer.html.eex | 4 ++- .../web/templates/o_auth/o_auth/show.html.eex | 25 ++----------------- 4 files changed, 16 insertions(+), 30 deletions(-) diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index 98f7293bc..b3cf9ed11 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -161,7 +161,6 @@ width: 100%; background-color: #931014; border: 1px solid #a06060; - color: #902020; border-radius: 4px; padding: 10px; margin-top: 20px; diff --git a/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex index e6cfe108b..c9ec1ecbf 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex @@ -1,13 +1,19 @@
- <%= label @form, :scope, "Permissions" %> - + <%= label @form, :scope, "The following permissions will be granted" %>
<%= for scope <- @available_scopes do %> <%# Note: using hidden input with `unchecked_value` in order to distinguish user's empty selection from `scope` param being omitted %> -
+ <%= if scope in @scopes do %> +
+ <%= checkbox @form, :"scope_#{scope}", value: scope in @scopes && scope, checked_value: scope, unchecked_value: "", name: "authorization[scope][]" %> + <%= label @form, :"scope_#{scope}", String.capitalize(scope) %> + <%= if scope in @scopes && scope do %> + <%= String.capitalize(scope) %> + <% end %> +
+ <% else %> <%= checkbox @form, :"scope_#{scope}", value: scope in @scopes && scope, checked_value: scope, unchecked_value: "", name: "authorization[scope][]" %> - <%= label @form, :"scope_#{scope}", String.capitalize(scope) %> -
+ <% end %> <% end %>
diff --git a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex index 4bcda7300..4a0718851 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex @@ -1,7 +1,9 @@

Sign in with external provider

<%= form_for @conn, o_auth_path(@conn, :prepare_request), [as: "authorization", method: "get"], fn f -> %> - <%= render @view_module, "_scopes.html", Map.put(assigns, :form, f) %> +
+ <%= render @view_module, "_scopes.html", Map.merge(assigns, %{form: f}) %> +
<%= hidden_input f, :client_id, value: @client_id %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index ed4fb5ce7..b17142ff8 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -18,7 +18,6 @@ <%= hidden_input f, :name, value: @params["name"] %> <%= hidden_input f, :password, value: @params["password"] %>
- <% else %>
<%= label f, :name, "Username" %> @@ -29,36 +28,16 @@ <%= password_input f, :password %>
<%= submit "Log In" %> -
- <%= label f, :scope, "The following permissions will be granted" %> -
- <%= for scope <- @available_scopes do %> - <%# Note: using hidden input with `unchecked_value` in order to distinguish user's empty selection from `scope` param being omitted %> - <%= if scope in @scopes do %> -
- <%= checkbox f, :"scope_#{scope}", value: scope in @scopes && scope, checked_value: scope, unchecked_value: "", name: "authorization[scope][]" %> - <%= label f, :"scope_#{scope}", String.capitalize(scope) %> - <%= if scope in @scopes && scope do %> - <%= String.capitalize(scope) %> - <% end %> -
- <% else %> - <%= checkbox f, :"scope_#{scope}", value: scope in @scopes && scope, checked_value: scope, unchecked_value: "", name: "authorization[scope][]" %> - <% end %> - <% end %> -
-
+ <%= render @view_module, "_scopes.html", Map.merge(assigns, %{form: f}) %> <% end %> -<%= render @view_module, "_scopes.html", Map.merge(assigns, %{form: f}) %> - <%= hidden_input f, :client_id, value: @client_id %> <%= hidden_input f, :response_type, value: @response_type %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> <%= hidden_input f, :state, value: @state %> +<% end %> <%= if Pleroma.Config.oauth_consumer_enabled?() do %> <%= render @view_module, Pleroma.Web.Auth.Authenticator.oauth_consumer_template(), assigns %> <% end %> -<% end %> From be56801ddab84d9e21af75c7752e5898fe0dbecb Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 3 Jun 2019 19:26:43 +0300 Subject: [PATCH 60/64] Add index on inReplyTo for objects Fixes the performance of `get_existing_votes` --- .../20190603162018_add_object_in_reply_to_index.exs | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 priv/repo/migrations/20190603162018_add_object_in_reply_to_index.exs diff --git a/priv/repo/migrations/20190603162018_add_object_in_reply_to_index.exs b/priv/repo/migrations/20190603162018_add_object_in_reply_to_index.exs new file mode 100644 index 000000000..df4ac7782 --- /dev/null +++ b/priv/repo/migrations/20190603162018_add_object_in_reply_to_index.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddObjectInReplyToIndex do + use Ecto.Migration + + def change do + create index(:objects, ["(data->>'inReplyTo')"], name: :objects_in_reply_to_index) + end +end From cfc3c62b2ffde1a2fbc7764ad1cbc292e7f20f74 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 3 Jun 2019 20:42:08 +0300 Subject: [PATCH 61/64] Add missing tag index on objects The previous activity index is useless because objects are not embedded anymore and instead a joined object is queried. --- .../20190603173419_add_tag_index_to_objects.exs | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs diff --git a/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs b/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs new file mode 100644 index 000000000..c915a0213 --- /dev/null +++ b/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs @@ -0,0 +1,8 @@ +defmodule Pleroma.Repo.Migrations.AddTagIndexToObjects do + use Ecto.Migration + + def change do + drop_if_exists index(:activities, ["(data #> '{\"object\",\"tag\"}')"], using: :gin, name: :activities_tags) + create index(:objects, ["(data->'tag')"], using: :gin, name: :objects_tags) + end +end From 7758148990184ac25e097dc7358a918b89aa3ca9 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 4 Jun 2019 05:37:31 +0000 Subject: [PATCH 62/64] update CHANGELOG for mrf_subchain --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff1fff876..532e76fbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - OAuth: added job to clean expired access tokens - MRF: Support for rejecting reports from specific instances (`mrf_simple`) - MRF: Support for stripping avatars and banner images from specific instances (`mrf_simple`) +- MRF: Support for running subchains. ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer From 84cc131b59ad6c8910735c982757fee598de8757 Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Tue, 4 Jun 2019 05:46:19 +0000 Subject: [PATCH 63/64] Add missing HTTP Request mocks --- test/object/containment_test.exs | 5 +++++ test/support/http_request_mock.ex | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/test/object/containment_test.exs b/test/object/containment_test.exs index 452064093..a7a046203 100644 --- a/test/object/containment_test.exs +++ b/test/object/containment_test.exs @@ -6,6 +6,11 @@ defmodule Pleroma.Object.ContainmentTest do import Pleroma.Factory + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + describe "general origin containment" do test "contain_origin_from_id() catches obvious spoofing attempts" do data = %{ diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 36b9265e7..67ef0928a 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -243,6 +243,14 @@ defmodule HttpRequestMock do }} end + def get("https://n1u.moe/users/rye", _, _, Accept: "application/activity+json") do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/httpoison_mock/rye.json") + }} + end + def get("http://mastodon.example.org/users/admin/statuses/100787282858396771", _, _, _) do {:ok, %Tesla.Env{ @@ -302,6 +310,10 @@ defmodule HttpRequestMock do }} end + def get("http://mastodon.example.org/users/gargron", _, _, Accept: "application/activity+json") do + {:error, :nxdomain} + end + def get( "http://mastodon.example.org/@admin/99541947525187367", _, @@ -546,6 +558,15 @@ defmodule HttpRequestMock do }} end + def get( + "http://gs.example.org:4040/index.php/user/1", + _, + _, + Accept: "application/activity+json" + ) do + {:ok, %Tesla.Env{status: 406, body: ""}} + end + def get("http://gs.example.org/index.php/api/statuses/user_timeline/1.atom", _, _, _) do {:ok, %Tesla.Env{ From 1c6cf0a348661612070d32c6f4f5d980d9254ac0 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 4 Jun 2019 06:19:44 +0000 Subject: [PATCH 64/64] nodeinfo: add pollLimits to metadata --- lib/pleroma/web/nodeinfo/nodeinfo_controller.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 59f3d4e11..57f5b61bb 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -97,6 +97,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do "pleroma_api", "mastodon_api", "mastodon_api_streaming", + "polls", if Config.get([:media_proxy, :enabled]) do "media_proxy" end, @@ -149,6 +150,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do }, staffAccounts: staff_accounts, federation: federation_response, + pollLimits: Config.get([:instance, :poll_limits]), postFormats: Config.get([:instance, :allowed_post_formats]), uploadLimits: %{ general: Config.get([:instance, :upload_limit]),