Merge branch 'fetch-posts-on-first-federation' into 'develop'

Fetch user's outbox posts on first federation with that user

Closes #131

See merge request pleroma/pleroma!907

config/config.exs

@@ -348,6 +348,10 @@ config :pleroma, Pleroma.Jobs,
   federator_outgoing: [max_jobs: 50],
   mailer: [max_jobs: 10]
 
+config :pleroma, :fetch_initial_posts,
+  enabled: false,
+  pages: 5
+
 config :auto_linker,
   opts: [
     scheme: true,

docs/config.md

@@ -285,6 +285,10 @@ This config contains two queues: `federator_incoming` and `federator_outgoing`.
 ## :rich_media
 * `enabled`: if enabled the instance will parse metadata from attached links to generate link previews
 
+## :fetch_initial_posts
+* `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts
+* `pages`: the amount of pages to fetch
+
 ## :hackney_pools
 
 Advanced. Tweaks Hackney (http client) connections pools.

lib/pleroma/user.ex

@@ -532,6 +532,10 @@ defmodule Pleroma.User do
       _e ->
         with [_nick, _domain] <- String.split(nickname, "@"),
              {:ok, user} <- fetch_by_nickname(nickname) do
+          if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do
+            {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
+          end
+
           user
         else
           _e -> nil
@@ -539,6 +543,17 @@ defmodule Pleroma.User do
     end
   end
 
+  @doc "Fetch some posts when the user has just been federated with"
+  def fetch_initial_posts(user) do
+    pages = Pleroma.Config.get!([:fetch_initial_posts, :pages])
+
+    Enum.each(
+      # Insert all the posts in reverse order, so they're in the right order on the timeline
+      Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)),
+      &Pleroma.Web.Federator.incoming_ap_doc/1
+    )
+  end
+
   def get_followers_query(%User{id: id, follower_address: follower_address}, nil) do
     from(
       u in User,
@@ -1121,24 +1136,36 @@ defmodule Pleroma.User do
 
   def html_filter_policy(_), do: @default_scrubbers
 
+  def fetch_by_ap_id(ap_id) do
+    ap_try = ActivityPub.make_user_from_ap_id(ap_id)
+
+    case ap_try do
+      {:ok, user} ->
+        user
+
+      _ ->
+        case OStatus.make_user(ap_id) do
+          {:ok, user} -> user
+          _ -> {:error, "Could not fetch by AP id"}
+        end
+    end
+  end
+
   def get_or_fetch_by_ap_id(ap_id) do
     user = get_by_ap_id(ap_id)
 
     if !is_nil(user) and !User.needs_update?(user) do
       user
     else
-      ap_try = ActivityPub.make_user_from_ap_id(ap_id)
+      user = fetch_by_ap_id(ap_id)
 
-      case ap_try do
+      if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do
-        {:ok, user} ->
+        with %User{} = user do
-          user
+          {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
-
+        end
-        _ ->
-          case OStatus.make_user(ap_id) do
-            {:ok, user} -> user
-            _ -> {:error, "Could not fetch by AP id"}
-          end
       end
+
+      user
     end
   end
 

lib/pleroma/web/activity_pub/utils.ex

@@ -633,4 +633,43 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     }
     |> Map.merge(additional)
   end
+
+  @doc """
+  Fetches the OrderedCollection/OrderedCollectionPage from `from`, limiting the amount of pages fetched after
+  the first one to `pages_left` pages.
+  If the amount of pages is higher than the collection has, it returns whatever was there.
+  """
+  def fetch_ordered_collection(from, pages_left, acc \\ []) do
+    with {:ok, response} <- Tesla.get(from),
+         {:ok, collection} <- Poison.decode(response.body) do
+      case collection["type"] do
+        "OrderedCollection" ->
+          # If we've encountered the OrderedCollection and not the page,
+          # just call the same function on the page address
+          fetch_ordered_collection(collection["first"], pages_left)
+
+        "OrderedCollectionPage" ->
+          if pages_left > 0 do
+            # There are still more pages
+            if Map.has_key?(collection, "next") do
+              # There are still more pages, go deeper saving what we have into the accumulator
+              fetch_ordered_collection(
+                collection["next"],
+                pages_left - 1,
+                acc ++ collection["orderedItems"]
+              )
+            else
+              # No more pages left, just return whatever we already have
+              acc ++ collection["orderedItems"]
+            end
+          else
+            # Got the amount of pages needed, add them all to the accumulator
+            acc ++ collection["orderedItems"]
+          end
+
+        _ ->
+          {:error, "Not an OrderedCollection or OrderedCollectionPage"}
+      end
+    end
+  end
 end

test/web/activity_pub/utils_test.exs

@@ -104,4 +104,69 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do
       assert Enum.sort(cc) == expected_cc
     end
   end
+
+  describe "fetch_ordered_collection" do
+    import Tesla.Mock
+
+    test "fetches the first OrderedCollectionPage when an OrderedCollection is encountered" do
+      mock(fn
+        %{method: :get, url: "http://mastodon.com/outbox"} ->
+          json(%{"type" => "OrderedCollection", "first" => "http://mastodon.com/outbox?page=true"})
+
+        %{method: :get, url: "http://mastodon.com/outbox?page=true"} ->
+          json(%{"type" => "OrderedCollectionPage", "orderedItems" => ["ok"]})
+      end)
+
+      assert Utils.fetch_ordered_collection("http://mastodon.com/outbox", 1) == ["ok"]
+    end
+
+    test "fetches several pages in the right order one after another, but only the specified amount" do
+      mock(fn
+        %{method: :get, url: "http://example.com/outbox"} ->
+          json(%{
+            "type" => "OrderedCollectionPage",
+            "orderedItems" => [0],
+            "next" => "http://example.com/outbox?page=1"
+          })
+
+        %{method: :get, url: "http://example.com/outbox?page=1"} ->
+          json(%{
+            "type" => "OrderedCollectionPage",
+            "orderedItems" => [1],
+            "next" => "http://example.com/outbox?page=2"
+          })
+
+        %{method: :get, url: "http://example.com/outbox?page=2"} ->
+          json(%{"type" => "OrderedCollectionPage", "orderedItems" => [2]})
+      end)
+
+      assert Utils.fetch_ordered_collection("http://example.com/outbox", 0) == [0]
+      assert Utils.fetch_ordered_collection("http://example.com/outbox", 1) == [0, 1]
+    end
+
+    test "returns an error if the url doesn't have an OrderedCollection/Page" do
+      mock(fn
+        %{method: :get, url: "http://example.com/not-an-outbox"} ->
+          json(%{"type" => "NotAnOutbox"})
+      end)
+
+      assert {:error, _} = Utils.fetch_ordered_collection("http://example.com/not-an-outbox", 1)
+    end
+
+    test "returns the what was collected if there are less pages than specified" do
+      mock(fn
+        %{method: :get, url: "http://example.com/outbox"} ->
+          json(%{
+            "type" => "OrderedCollectionPage",
+            "orderedItems" => [0],
+            "next" => "http://example.com/outbox?page=1"
+          })
+
+        %{method: :get, url: "http://example.com/outbox?page=1"} ->
+          json(%{"type" => "OrderedCollectionPage", "orderedItems" => [1]})
+      end)
+
+      assert Utils.fetch_ordered_collection("http://example.com/outbox", 5) == [0, 1]
+    end
+  end
 end