From f161a92cb1abd981e37367fcd5d315ac14510d12 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Wed, 23 Jan 2019 18:37:25 +0300
Subject: [PATCH 01/12] [#534] Initial implementation of unreachable federation
 targets retirement.

---
 lib/pleroma/instances.ex                      | 12 +++
 lib/pleroma/instances/instance.ex             | 77 +++++++++++++++++++
 lib/pleroma/web/activity_pub/activity_pub.ex  | 39 +++++++---
 lib/pleroma/web/salmon/salmon.ex              | 13 +++-
 lib/pleroma/web/websub/websub.ex              | 10 ++-
 .../20190123125546_create_instances.exs       | 16 ++++
 6 files changed, 153 insertions(+), 14 deletions(-)
 create mode 100644 lib/pleroma/instances.ex
 create mode 100644 lib/pleroma/instances/instance.ex
 create mode 100644 priv/repo/migrations/20190123125546_create_instances.exs

diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex
new file mode 100644
index 000000000..25b739520
--- /dev/null
+++ b/lib/pleroma/instances.ex
@@ -0,0 +1,12 @@
+defmodule Pleroma.Instances do
+  @moduledoc "Instances context."
+
+  @adapter Pleroma.Instances.Instance
+
+  defdelegate reachable?(url), to: @adapter
+  defdelegate set_reachable(url), to: @adapter
+  defdelegate set_unreachable(url, unreachable_since \\ nil), to: @adapter
+
+  def reachability_time_threshold,
+    do: NaiveDateTime.add(NaiveDateTime.utc_now(), -30 * 24 * 3600, :second)
+end
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
new file mode 100644
index 000000000..4507ef6d5
--- /dev/null
+++ b/lib/pleroma/instances/instance.ex
@@ -0,0 +1,77 @@
+defmodule Pleroma.Instances.Instance do
+  @moduledoc "Instance."
+
+  alias Pleroma.Instances
+  alias Pleroma.Instances.Instance
+
+  use Ecto.Schema
+
+  import Ecto.{Query, Changeset}
+
+  alias Pleroma.Repo
+
+  schema "instances" do
+    field(:host, :string)
+    field(:unreachable_since, :naive_datetime)
+    field(:reachability_checked_at, :naive_datetime)
+
+    timestamps()
+  end
+
+  def update_changeset(struct, params \\ %{}) do
+    struct
+    |> cast(params, [:host, :unreachable_since, :reachability_checked_at])
+    |> unique_constraint(:host)
+  end
+
+  def reachable?(url) do
+    !Repo.one(
+      from(i in Instance,
+        where:
+          i.host == ^host(url) and i.unreachable_since <= ^Instances.reachability_time_threshold(),
+        select: true
+      )
+    )
+  end
+
+  def set_reachable(url) do
+    Repo.update_all(
+      from(i in Instance, where: i.host == ^host(url)),
+      set: [
+        unreachable_since: nil,
+        reachability_checked_at: DateTime.utc_now()
+      ]
+    )
+  end
+
+  def set_unreachable(url, unreachable_since \\ nil) do
+    unreachable_since = unreachable_since || DateTime.utc_now()
+    host = host(url)
+    existing_record = Repo.get_by(Instance, %{host: host})
+
+    changes = %{
+      unreachable_since: unreachable_since,
+      reachability_checked_at: NaiveDateTime.utc_now()
+    }
+
+    if existing_record do
+      update_changes =
+        if existing_record.unreachable_since &&
+             NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt,
+           do: Map.delete(changes, :unreachable_since),
+           else: changes
+
+      {:ok, _instance} = Repo.update(update_changeset(existing_record, update_changes))
+    else
+      {:ok, _instance} = Repo.insert(update_changeset(%Instance{}, Map.put(changes, :host, host)))
+    end
+  end
+
+  defp host(url_or_host) do
+    if url_or_host =~ ~r/^http/i do
+      URI.parse(url_or_host).host
+    else
+      url_or_host
+    end
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 82fffd324..b14c91c18 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
-  alias Pleroma.{Activity, Repo, Object, Upload, User, Notification}
+  alias Pleroma.{Activity, Repo, Object, Upload, User, Notification, Instances}
   alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF}
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Federator
@@ -721,7 +721,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end)
   end
 
-  def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
+  def publish_one(%{inbox: inbox} = activity) do
+    if Instances.reachable?(inbox) do
+      do_publish_one(activity)
+    else
+      {:error, :noop}
+    end
+  end
+
+  defp do_publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
     Logger.info("Federating #{id} to #{inbox}")
     host = URI.parse(inbox).host
 
@@ -734,15 +742,24 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         digest: digest
       })
 
-    @httpoison.post(
-      inbox,
-      json,
-      [
-        {"Content-Type", "application/activity+json"},
-        {"signature", signature},
-        {"digest", digest}
-      ]
-    )
+    with {:ok, _} <-
+           result =
+             @httpoison.post(
+               inbox,
+               json,
+               [
+                 {"Content-Type", "application/activity+json"},
+                 {"signature", signature},
+                 {"digest", digest}
+               ]
+             ) do
+      Instances.set_reachable(inbox)
+      result
+    else
+      e ->
+        Instances.set_unreachable(inbox)
+        e
+    end
   end
 
   # TODO:
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index e41657da1..0a0b91433 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.Salmon do
   @httpoison Application.get_env(:pleroma, :httpoison)
 
   use Bitwise
+  alias Pleroma.Instances
   alias Pleroma.Web.XML
   alias Pleroma.Web.OStatus.ActivityRepresenter
   alias Pleroma.User
@@ -167,15 +168,23 @@ defmodule Pleroma.Web.Salmon do
     do: send_to_user(salmon, feed, poster)
 
   defp send_to_user(url, feed, poster) when is_binary(url) do
-    with {:ok, %{status: code}} <-
+    with {:reachable, true} <- {:reachable, Instances.reachable?(url)},
+         {:ok, %{status: code}} <-
            poster.(
              url,
              feed,
              [{"Content-Type", "application/magic-envelope+xml"}]
            ) do
+      Instances.set_reachable(url)
       Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
     else
-      e -> Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
+      {:reachable, false} ->
+        Logger.debug(fn -> "Pushing Salmon to #{url} skipped as marked unreachable)" end)
+        :noop
+
+      e ->
+        Instances.set_unreachable(url)
+        Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
     end
   end
 
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index 7ca62c83b..a6bbaef37 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.Websub do
   alias Ecto.Changeset
   alias Pleroma.Repo
+  alias Pleroma.Instances
   alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription}
   alias Pleroma.Web.OStatus.FeedRepresenter
   alias Pleroma.Web.{XML, Endpoint, OStatus}
@@ -267,7 +268,8 @@ defmodule Pleroma.Web.Websub do
     signature = sign(secret || "", xml)
     Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
 
-    with {:ok, %{status: code}} <-
+    with {:reachable, true} <- {:reachable, Instances.reachable?(callback)},
+         {:ok, %{status: code}} <-
            @httpoison.post(
              callback,
              xml,
@@ -276,10 +278,16 @@ defmodule Pleroma.Web.Websub do
                {"X-Hub-Signature", "sha1=#{signature}"}
              ]
            ) do
+      Instances.set_reachable(callback)
       Logger.info(fn -> "Pushed to #{callback}, code #{code}" end)
       {:ok, code}
     else
+      {:reachable, false} ->
+        Logger.debug(fn -> "Pushing to #{callback} skipped as marked unreachable)" end)
+        {:error, :noop}
+
       e ->
+        Instances.set_unreachable(callback)
         Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end)
         {:error, e}
     end
diff --git a/priv/repo/migrations/20190123125546_create_instances.exs b/priv/repo/migrations/20190123125546_create_instances.exs
new file mode 100644
index 000000000..4f5915fba
--- /dev/null
+++ b/priv/repo/migrations/20190123125546_create_instances.exs
@@ -0,0 +1,16 @@
+defmodule Pleroma.Repo.Migrations.CreateInstances do
+  use Ecto.Migration
+
+  def change do
+    create table(:instances) do
+      add :host, :string
+      add :unreachable_since, :naive_datetime
+      add :reachability_checked_at, :naive_datetime
+
+      timestamps()
+    end
+
+    create unique_index(:instances, [:host])
+    create index(:instances, [:unreachable_since])
+  end
+end

From 20b54366ee916677b3865acf36baeeb062dd550b Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 24 Jan 2019 11:54:52 +0300
Subject: [PATCH 02/12] [#534] Federation publish requests status control
 (enforced 2xx response code check).

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 2 +-
 lib/pleroma/web/salmon/salmon.ex             | 2 +-
 lib/pleroma/web/websub/websub.ex             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index b14c91c18..10155ff5a 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -742,7 +742,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         digest: digest
       })
 
-    with {:ok, _} <-
+    with {:ok, %{status: code}} when code in 200..299 <-
            result =
              @httpoison.post(
                inbox,
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index 0a0b91433..0423ccee0 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -169,7 +169,7 @@ defmodule Pleroma.Web.Salmon do
 
   defp send_to_user(url, feed, poster) when is_binary(url) do
     with {:reachable, true} <- {:reachable, Instances.reachable?(url)},
-         {:ok, %{status: code}} <-
+         {:ok, %{status: code}} when code in 200..299 <-
            poster.(
              url,
              feed,
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index a6bbaef37..9ceb5fbf7 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -269,7 +269,7 @@ defmodule Pleroma.Web.Websub do
     Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
 
     with {:reachable, true} <- {:reachable, Instances.reachable?(callback)},
-         {:ok, %{status: code}} <-
+         {:ok, %{status: code}} when code in 200..299 <-
            @httpoison.post(
              callback,
              xml,

From 8654a591f08c7d8d5d61f075906f0c6907e877bb Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 24 Jan 2019 17:37:23 +0300
Subject: [PATCH 03/12] [#534] Updating external instances reachability on
 incoming federation.

---
 lib/pleroma/instances/instance.ex                  | 14 +++++++++++---
 lib/pleroma/reverse_proxy.ex                       |  3 ++-
 lib/pleroma/web/activity_pub/activity_pub.ex       |  3 ++-
 .../web/activity_pub/activity_pub_controller.ex    |  7 +++++++
 lib/pleroma/web/controller_helper.ex               |  5 +++++
 lib/pleroma/web/ostatus/ostatus_controller.ex      |  7 +++++++
 lib/pleroma/web/salmon/salmon.ex                   |  5 ++++-
 lib/pleroma/web/websub/websub.ex                   |  3 ++-
 lib/pleroma/web/websub/websub_controller.ex        |  9 +++++++++
 9 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index 4507ef6d5..fe52331a3 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -24,7 +24,7 @@ defmodule Pleroma.Instances.Instance do
     |> unique_constraint(:host)
   end
 
-  def reachable?(url) do
+  def reachable?(url) when is_binary(url) do
     !Repo.one(
       from(i in Instance,
         where:
@@ -34,7 +34,9 @@ defmodule Pleroma.Instances.Instance do
     )
   end
 
-  def set_reachable(url) do
+  def reachable?(_), do: true
+
+  def set_reachable(url) when is_binary(url) do
     Repo.update_all(
       from(i in Instance, where: i.host == ^host(url)),
       set: [
@@ -44,7 +46,11 @@ defmodule Pleroma.Instances.Instance do
     )
   end
 
-  def set_unreachable(url, unreachable_since \\ nil) do
+  def set_reachable(_), do: {0, :noop}
+
+  def set_unreachable(url, unreachable_since \\ nil)
+
+  def set_unreachable(url, unreachable_since) when is_binary(url) do
     unreachable_since = unreachable_since || DateTime.utc_now()
     host = host(url)
     existing_record = Repo.get_by(Instance, %{host: host})
@@ -67,6 +73,8 @@ defmodule Pleroma.Instances.Instance do
     end
   end
 
+  def set_unreachable(_, _), do: {0, :noop}
+
   defp host(url_or_host) do
     if url_or_host =~ ~r/^http/i do
       URI.parse(url_or_host).host
diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex
index a25b5ea4e..d8b17212b 100644
--- a/lib/pleroma/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy.ex
@@ -3,7 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy do
-  @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-unmodified-since if-none-match if-range range)
+  @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-unmodified-since) ++
+                      ~w(if-none-match if-range range referer)
   @resp_cache_headers ~w(etag date last-modified cache-control)
   @keep_resp_headers @resp_cache_headers ++
                        ~w(content-type content-disposition content-encoding content-range accept-ranges vary)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 10155ff5a..44c295d65 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -750,7 +750,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
                [
                  {"Content-Type", "application/activity+json"},
                  {"signature", signature},
-                 {"digest", digest}
+                 {"digest", digest},
+                 {"referer", Pleroma.Web.Endpoint.url()}
                ]
              ) do
       Instances.set_reachable(inbox)
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 7eed0a600..dc353dff0 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   use Pleroma.Web, :controller
+
   alias Pleroma.{Activity, User, Object}
   alias Pleroma.Web.ActivityPub.{ObjectView, UserView}
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -18,6 +19,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay])
   plug(:relay_active? when action in [:relay])
+  plug(:set_requester_reachable when action in [:inbox])
 
   def relay_active?(conn, _) do
     if Keyword.get(Application.get_env(:pleroma, :instance), :allow_relay) do
@@ -289,4 +291,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     |> put_status(500)
     |> json("error")
   end
+
+  defp set_requester_reachable(conn, _) do
+    Pleroma.Web.ControllerHelper.set_requester_reachable(conn)
+    conn
+  end
 end
diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex
index 14e3d19fd..13cf1877f 100644
--- a/lib/pleroma/web/controller_helper.ex
+++ b/lib/pleroma/web/controller_helper.ex
@@ -10,4 +10,9 @@ defmodule Pleroma.Web.ControllerHelper do
     |> put_status(status)
     |> json(json)
   end
+
+  def set_requester_reachable(conn) do
+    with [referer] <- get_req_header(conn, "referer"),
+         do: Pleroma.Instances.set_reachable(referer)
+  end
 end
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index ce022bcc1..a89f16b94 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -15,6 +15,8 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Web.ActivityPub.ActivityPub
 
   plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming])
+  plug(:set_requester_reachable when action in [:salmon_incoming])
+
   action_fallback(:errors)
 
   def feed_redirect(conn, %{"nickname" => nickname}) do
@@ -201,4 +203,9 @@ defmodule Pleroma.Web.OStatus.OStatusController do
     |> put_status(500)
     |> text("Something went wrong")
   end
+
+  defp set_requester_reachable(conn, _) do
+    Pleroma.Web.ControllerHelper.set_requester_reachable(conn)
+    conn
+  end
 end
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index 0423ccee0..e4d2d9517 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -173,7 +173,10 @@ defmodule Pleroma.Web.Salmon do
            poster.(
              url,
              feed,
-             [{"Content-Type", "application/magic-envelope+xml"}]
+             [
+               {"Content-Type", "application/magic-envelope+xml"},
+               {"referer", Pleroma.Web.Endpoint.url()}
+             ]
            ) do
       Instances.set_reachable(url)
       Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index 9ceb5fbf7..ac8903913 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -275,7 +275,8 @@ defmodule Pleroma.Web.Websub do
              xml,
              [
                {"Content-Type", "application/atom+xml"},
-               {"X-Hub-Signature", "sha1=#{signature}"}
+               {"X-Hub-Signature", "sha1=#{signature}"},
+               {"referer", Pleroma.Web.Endpoint.url()}
              ]
            ) do
       Instances.set_reachable(callback)
diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex
index e58f144e5..02fe075d7 100644
--- a/lib/pleroma/web/websub/websub_controller.ex
+++ b/lib/pleroma/web/websub/websub_controller.ex
@@ -4,9 +4,11 @@
 
 defmodule Pleroma.Web.Websub.WebsubController do
   use Pleroma.Web, :controller
+
   alias Pleroma.{Repo, User}
   alias Pleroma.Web.{Websub, Federator}
   alias Pleroma.Web.Websub.WebsubClientSubscription
+
   require Logger
 
   plug(
@@ -18,6 +20,8 @@ defmodule Pleroma.Web.Websub.WebsubController do
          ]
   )
 
+  plug(:set_requester_reachable when action in [:websub_incoming])
+
   def websub_subscription_request(conn, %{"nickname" => nickname} = params) do
     user = User.get_cached_by_nickname(nickname)
 
@@ -92,4 +96,9 @@ defmodule Pleroma.Web.Websub.WebsubController do
         |> send_resp(500, "Error")
     end
   end
+
+  defp set_requester_reachable(conn, _) do
+    Pleroma.Web.ControllerHelper.set_requester_reachable(conn)
+    conn
+  end
 end

From 3e9399ec0b498c0c9783ccb0fea9f682c8b9d0ca Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 24 Jan 2019 19:15:23 +0300
Subject: [PATCH 04/12] [#534] Optimized bulk publish ops to filter on
 reachability early. `Instance` refactoring.

---
 lib/pleroma/instances.ex                     |  9 ++++
 lib/pleroma/instances/instance.ex            | 57 +++++++++++++-------
 lib/pleroma/web/activity_pub/activity_pub.ex |  5 +-
 lib/pleroma/web/salmon/salmon.ex             |  8 ++-
 lib/pleroma/web/websub/websub.ex             | 18 ++++---
 5 files changed, 69 insertions(+), 28 deletions(-)

diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex
index 25b739520..6d445d6b3 100644
--- a/lib/pleroma/instances.ex
+++ b/lib/pleroma/instances.ex
@@ -3,10 +3,19 @@ defmodule Pleroma.Instances do
 
   @adapter Pleroma.Instances.Instance
 
+  defdelegate filter_reachable(urls), to: @adapter
   defdelegate reachable?(url), to: @adapter
   defdelegate set_reachable(url), to: @adapter
   defdelegate set_unreachable(url, unreachable_since \\ nil), to: @adapter
 
   def reachability_time_threshold,
     do: NaiveDateTime.add(NaiveDateTime.utc_now(), -30 * 24 * 3600, :second)
+
+  def host(url_or_host) when is_binary(url_or_host) do
+    if url_or_host =~ ~r/^http/i do
+      URI.parse(url_or_host).host
+    else
+      url_or_host
+    end
+  end
 end
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index fe52331a3..a17c8dab1 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -18,12 +18,35 @@ defmodule Pleroma.Instances.Instance do
     timestamps()
   end
 
-  def update_changeset(struct, params \\ %{}) do
+  defdelegate host(url), to: Instances
+
+  def changeset(struct, params \\ %{}) do
     struct
     |> cast(params, [:host, :unreachable_since, :reachability_checked_at])
+    |> validate_required([:host])
     |> unique_constraint(:host)
   end
 
+  def filter_reachable([]), do: []
+
+  def filter_reachable(urls) when is_list(urls) do
+    hosts =
+      urls
+      |> Enum.map(&(&1 && host(&1)))
+      |> Enum.filter(&(to_string(&1) != ""))
+
+    unreachable_hosts =
+      Repo.all(
+        from(i in Instance,
+          where:
+            i.host in ^hosts and i.unreachable_since <= ^Instances.reachability_time_threshold(),
+          select: i.host
+        )
+      )
+
+    Enum.filter(urls, &(&1 && host(&1) not in unreachable_hosts))
+  end
+
   def reachable?(url) when is_binary(url) do
     !Repo.one(
       from(i in Instance,
@@ -37,13 +60,13 @@ defmodule Pleroma.Instances.Instance do
   def reachable?(_), do: true
 
   def set_reachable(url) when is_binary(url) do
-    Repo.update_all(
-      from(i in Instance, where: i.host == ^host(url)),
-      set: [
-        unreachable_since: nil,
-        reachability_checked_at: DateTime.utc_now()
-      ]
-    )
+    with host <- host(url),
+         %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do
+      {:ok, _instance} =
+        existing_record
+        |> changeset(%{unreachable_since: nil, reachability_checked_at: DateTime.utc_now()})
+        |> Repo.update()
+    end
   end
 
   def set_reachable(_), do: {0, :noop}
@@ -67,19 +90,17 @@ defmodule Pleroma.Instances.Instance do
            do: Map.delete(changes, :unreachable_since),
            else: changes
 
-      {:ok, _instance} = Repo.update(update_changeset(existing_record, update_changes))
+      {:ok, _instance} =
+        existing_record
+        |> changeset(update_changes)
+        |> Repo.update()
     else
-      {:ok, _instance} = Repo.insert(update_changeset(%Instance{}, Map.put(changes, :host, host)))
+      {:ok, _instance} =
+        %Instance{}
+        |> changeset(Map.put(changes, :host, host))
+        |> Repo.insert()
     end
   end
 
   def set_unreachable(_, _), do: {0, :noop}
-
-  defp host(url_or_host) do
-    if url_or_host =~ ~r/^http/i do
-      URI.parse(url_or_host).host
-    else
-      url_or_host
-    end
-  end
 end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 44c295d65..4b34334a0 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -689,7 +689,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def publish(actor, activity) do
-    followers =
+    remote_followers =
       if actor.follower_address in activity.recipients do
         {:ok, followers} = User.get_followers(actor)
         followers |> Enum.filter(&(!&1.local))
@@ -700,13 +700,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     public = is_public?(activity)
 
     remote_inboxes =
-      (Pleroma.Web.Salmon.remote_users(activity) ++ followers)
+      (Pleroma.Web.Salmon.remote_users(activity) ++ remote_followers)
       |> Enum.filter(fn user -> User.ap_enabled?(user) end)
       |> Enum.map(fn %{info: %{source_data: data}} ->
         (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"]
       end)
       |> Enum.uniq()
       |> Enum.filter(fn inbox -> should_federate?(inbox, public) end)
+      |> Instances.filter_reachable()
 
     {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
     json = Jason.encode!(data)
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index e4d2d9517..848131d52 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -221,7 +221,13 @@ defmodule Pleroma.Web.Salmon do
       {:ok, private, _} = keys_from_pem(keys)
       {:ok, feed} = encode(private, feed)
 
-      remote_users(activity)
+      remote_users = remote_users(activity)
+
+      salmon_urls = Enum.map(remote_users, & &1.info.salmon)
+      reachable_salmon_urls = Instances.filter_reachable(salmon_urls)
+
+      remote_users
+      |> Enum.filter(&(&1.info.salmon in reachable_salmon_urls))
       |> Enum.each(fn remote_user ->
         Task.start(fn ->
           Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index ac8903913..bb4442591 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -54,7 +54,12 @@ defmodule Pleroma.Web.Websub do
   ]
   def publish(topic, user, %{data: %{"type" => type}} = activity)
       when type in @supported_activities do
-    # TODO: Only send to still valid subscriptions.
+    response =
+      user
+      |> FeedRepresenter.to_simple_form([activity], [user])
+      |> :xmerl.export_simple(:xmerl_xml)
+      |> to_string
+
     query =
       from(
         sub in WebsubServerSubscription,
@@ -64,13 +69,12 @@ defmodule Pleroma.Web.Websub do
 
     subscriptions = Repo.all(query)
 
-    Enum.each(subscriptions, fn sub ->
-      response =
-        user
-        |> FeedRepresenter.to_simple_form([activity], [user])
-        |> :xmerl.export_simple(:xmerl_xml)
-        |> to_string
+    callbacks = Enum.map(subscriptions, & &1.callback)
+    reachable_callbacks = Instances.filter_reachable(callbacks)
 
+    subscriptions
+    |> Enum.filter(&(&1.callback in reachable_callbacks))
+    |> Enum.each(fn sub ->
       data = %{
         xml: response,
         topic: topic,

From 656ed7c84a5d8e423999457f66d8259ec8aa9a44 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 25 Jan 2019 15:10:21 +0300
Subject: [PATCH 05/12] [#534] Configurable outgoing federation reachability
 timeout.

---
 config/config.exs                 |  1 +
 docs/config.md                    |  1 +
 lib/pleroma/instances.ex          | 16 ++++++++++++++--
 lib/pleroma/instances/instance.ex |  6 ++++--
 4 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index d30b0aad0..7a1a875c9 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -125,6 +125,7 @@ config :pleroma, :instance,
   banner_upload_limit: 4_000_000,
   registrations_open: true,
   federating: true,
+  federation_reachability_timeout_days: 90,
   allow_relay: true,
   rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy,
   public: true,
diff --git a/docs/config.md b/docs/config.md
index 6bf7b9ea7..3f4588299 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -72,6 +72,7 @@ config :pleroma, Pleroma.Mailer,
 * `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`).
 * `account_activation_required`: Require users to confirm their emails before signing in.
 * `federating`: Enable federation with other instances
+* `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it.
 * `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance
 * `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default:
   * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default)
diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex
index 6d445d6b3..0b08f0eb8 100644
--- a/lib/pleroma/instances.ex
+++ b/lib/pleroma/instances.ex
@@ -8,8 +8,20 @@ defmodule Pleroma.Instances do
   defdelegate set_reachable(url), to: @adapter
   defdelegate set_unreachable(url, unreachable_since \\ nil), to: @adapter
 
-  def reachability_time_threshold,
-    do: NaiveDateTime.add(NaiveDateTime.utc_now(), -30 * 24 * 3600, :second)
+  def reachability_datetime_threshold do
+    federation_reachability_timeout_days =
+      Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 90
+
+    if federation_reachability_timeout_days > 0 do
+      NaiveDateTime.add(
+        NaiveDateTime.utc_now(),
+        -federation_reachability_timeout_days * 24 * 3600,
+        :second
+      )
+    else
+      ~N[0000-01-01 00:00:00]
+    end
+  end
 
   def host(url_or_host) when is_binary(url_or_host) do
     if url_or_host =~ ~r/^http/i do
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index a17c8dab1..60e8d0e21 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -39,7 +39,8 @@ defmodule Pleroma.Instances.Instance do
       Repo.all(
         from(i in Instance,
           where:
-            i.host in ^hosts and i.unreachable_since <= ^Instances.reachability_time_threshold(),
+            i.host in ^hosts and
+              i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
           select: i.host
         )
       )
@@ -51,7 +52,8 @@ defmodule Pleroma.Instances.Instance do
     !Repo.one(
       from(i in Instance,
         where:
-          i.host == ^host(url) and i.unreachable_since <= ^Instances.reachability_time_threshold(),
+          i.host == ^host(url) and
+            i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
         select: true
       )
     )

From 465adedb7cc457303278444d0f56960f87fde1e9 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 25 Jan 2019 18:29:43 +0300
Subject: [PATCH 06/12] [#534] Websub fix: made SQL use UTC time zone when
 comparing with `valid_until` (instead of postgresql-server default time
 zone).

---
 lib/pleroma/web/websub/websub.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index bb4442591..cbb7a5ac7 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -64,7 +64,7 @@ defmodule Pleroma.Web.Websub do
       from(
         sub in WebsubServerSubscription,
         where: sub.topic == ^topic and sub.state == "active",
-        where: fragment("? > NOW()", sub.valid_until)
+        where: fragment("? > (NOW() at time zone 'UTC')", sub.valid_until)
       )
 
     subscriptions = Repo.all(query)

From 060d280e64c201d3f8bec5615cc3b02cd460d3e1 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 25 Jan 2019 20:38:13 +0300
Subject: [PATCH 07/12] [#534] Made Salmon.send_to_user calls be handled
 through Federator.enqueue.

---
 lib/pleroma/web/federator/federator.ex |  6 +++++-
 lib/pleroma/web/salmon/salmon.ex       | 14 +++++++-------
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index f3a0e18b8..46f7a4973 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.Federator do
   use GenServer
   alias Pleroma.User
   alias Pleroma.Activity
-  alias Pleroma.Web.{WebFinger, Websub}
+  alias Pleroma.Web.{WebFinger, Websub, Salmon}
   alias Pleroma.Web.Federator.RetryQueue
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
@@ -124,6 +124,10 @@ defmodule Pleroma.Web.Federator do
     end
   end
 
+  def handle(:publish_single_salmon, {user_or_url, feed, poster}) do
+    Salmon.send_to_user(user_or_url, feed, poster)
+  end
+
   def handle(:publish_single_ap, params) do
     case ActivityPub.publish_one(params) do
       {:ok, _} ->
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index 848131d52..17ca7a6e8 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -164,10 +164,10 @@ defmodule Pleroma.Web.Salmon do
 
   # push an activity to remote accounts
   #
-  defp send_to_user(%{info: %{salmon: salmon}}, feed, poster),
+  def send_to_user(%{info: %{salmon: salmon}}, feed, poster),
     do: send_to_user(salmon, feed, poster)
 
-  defp send_to_user(url, feed, poster) when is_binary(url) do
+  def send_to_user(url, feed, poster) when is_binary(url) do
     with {:reachable, true} <- {:reachable, Instances.reachable?(url)},
          {:ok, %{status: code}} when code in 200..299 <-
            poster.(
@@ -180,6 +180,7 @@ defmodule Pleroma.Web.Salmon do
            ) do
       Instances.set_reachable(url)
       Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
+      :ok
     else
       {:reachable, false} ->
         Logger.debug(fn -> "Pushing Salmon to #{url} skipped as marked unreachable)" end)
@@ -188,10 +189,11 @@ defmodule Pleroma.Web.Salmon do
       e ->
         Instances.set_unreachable(url)
         Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
+        :error
     end
   end
 
-  defp send_to_user(_, _, _), do: nil
+  def send_to_user(_, _, _), do: :noop
 
   @supported_activities [
     "Create",
@@ -229,10 +231,8 @@ defmodule Pleroma.Web.Salmon do
       remote_users
       |> Enum.filter(&(&1.info.salmon in reachable_salmon_urls))
       |> Enum.each(fn remote_user ->
-        Task.start(fn ->
-          Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
-          send_to_user(remote_user, feed, poster)
-        end)
+        Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
+        Pleroma.Web.Federator.enqueue(:publish_single_salmon, {remote_user, feed, poster})
       end)
     end
   end

From d99650270b980c006690a7051a2d5cffe07779f1 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 25 Jan 2019 20:38:54 +0300
Subject: [PATCH 08/12] [#534] Federation reachability filtering tests.

---
 test/support/factory.ex                       |   2 +-
 .../activity_pub_controller_test.exs          |  44 +++++++-
 test/web/federator_test.exs                   | 101 +++++++++++++++++-
 test/web/ostatus/ostatus_controller_test.exs  |  90 ++++++++++------
 test/web/websub/websub_controller_test.exs    |  70 +++++++-----
 5 files changed, 243 insertions(+), 64 deletions(-)

diff --git a/test/support/factory.ex b/test/support/factory.ex
index 4ac77981a..964b2b61c 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -193,7 +193,7 @@ defmodule Pleroma.Factory do
   def websub_subscription_factory do
     %Pleroma.Web.Websub.WebsubServerSubscription{
       topic: "http://example.org",
-      callback: "http://example/org/callback",
+      callback: "http://example.org/callback",
       secret: "here's a secret",
       valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 100),
       state: "requested"
diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs
index 52e67f046..1b704330f 100644
--- a/test/web/activity_pub/activity_pub_controller_test.exs
+++ b/test/web/activity_pub/activity_pub_controller_test.exs
@@ -6,8 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
   use Pleroma.Web.ConnCase
   import Pleroma.Factory
   alias Pleroma.Web.ActivityPub.{UserView, ObjectView}
-  alias Pleroma.{Object, Repo, User}
-  alias Pleroma.Activity
+  alias Pleroma.{Object, Repo, Activity, User, Instances}
 
   setup_all do
     Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
@@ -144,6 +143,24 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
       :timer.sleep(500)
       assert Activity.get_by_ap_id(data["id"])
     end
+
+    test "it clears `unreachable` federation status of the sender", %{conn: conn} do
+      sender_url = "https://pleroma.soykaf.com"
+      Instances.set_unreachable(sender_url, Instances.reachability_datetime_threshold())
+      refute Instances.reachable?(sender_url)
+
+      data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!()
+
+      conn =
+        conn
+        |> assign(:valid_signature, true)
+        |> put_req_header("content-type", "application/activity+json")
+        |> put_req_header("referer", sender_url)
+        |> post("/inbox", data)
+
+      assert "ok" == json_response(conn, 200)
+      assert Instances.reachable?(sender_url)
+    end
   end
 
   describe "/users/:nickname/inbox" do
@@ -191,6 +208,29 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
 
       assert response(conn, 200) =~ note_activity.data["object"]["content"]
     end
+
+    test "it clears `unreachable` federation status of the sender", %{conn: conn} do
+      sender_host = "pleroma.soykaf.com"
+      Instances.set_unreachable(sender_host, Instances.reachability_datetime_threshold())
+      refute Instances.reachable?(sender_host)
+
+      user = insert(:user)
+
+      data =
+        File.read!("test/fixtures/mastodon-post-activity.json")
+        |> Poison.decode!()
+        |> Map.put("bcc", [user.ap_id])
+
+      conn =
+        conn
+        |> assign(:valid_signature, true)
+        |> put_req_header("content-type", "application/activity+json")
+        |> put_req_header("referer", "https://#{sender_host}")
+        |> post("/users/#{user.nickname}/inbox", data)
+
+      assert "ok" == json_response(conn, 200)
+      assert Instances.reachable?(sender_host)
+    end
   end
 
   describe "/users/:nickname/outbox" do
diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs
index a49265c0c..f4234aea8 100644
--- a/test/web/federator_test.exs
+++ b/test/web/federator_test.exs
@@ -3,8 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.FederatorTest do
-  alias Pleroma.Web.Federator
-  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.{CommonAPI, Federator}
+  alias Pleroma.Instances
   use Pleroma.DataCase
   import Pleroma.Factory
   import Mock
@@ -71,6 +71,103 @@ defmodule Pleroma.Web.FederatorTest do
     end
   end
 
+  describe "Targets reachability filtering in `publish`" do
+    test_with_mock "it federates only to reachable instances via AP",
+                   Federator,
+                   [:passthrough],
+                   [] do
+      user = insert(:user)
+
+      {inbox1, inbox2} =
+        {"https://domain.com/users/nick1/inbox", "https://domain2.com/users/nick2/inbox"}
+
+      insert(:user, %{
+        local: false,
+        nickname: "nick1@domain.com",
+        ap_id: "https://domain.com/users/nick1",
+        info: %{ap_enabled: true, source_data: %{"inbox" => inbox1}}
+      })
+
+      insert(:user, %{
+        local: false,
+        nickname: "nick2@domain2.com",
+        ap_id: "https://domain2.com/users/nick2",
+        info: %{ap_enabled: true, source_data: %{"inbox" => inbox2}}
+      })
+
+      Instances.set_unreachable(
+        URI.parse(inbox2).host,
+        Instances.reachability_datetime_threshold()
+      )
+
+      {:ok, _activity} =
+        CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"})
+
+      assert called(Federator.enqueue(:publish_single_ap, %{inbox: inbox1}))
+      refute called(Federator.enqueue(:publish_single_ap, %{inbox: inbox2}))
+    end
+
+    test_with_mock "it federates only to reachable instances via Websub",
+                   Federator,
+                   [:passthrough],
+                   [] do
+      user = insert(:user)
+      websub_topic = Pleroma.Web.OStatus.feed_path(user)
+
+      sub1 =
+        insert(:websub_subscription, %{
+          topic: websub_topic,
+          state: "active",
+          callback: "http://pleroma.soykaf.com/cb"
+        })
+
+      sub2 =
+        insert(:websub_subscription, %{
+          topic: websub_topic,
+          state: "active",
+          callback: "https://pleroma2.soykaf.com/cb"
+        })
+
+      Instances.set_unreachable(sub1.callback, Instances.reachability_datetime_threshold())
+
+      {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI"})
+
+      assert called(Federator.enqueue(:publish_single_websub, %{callback: sub2.callback}))
+      refute called(Federator.enqueue(:publish_single_websub, %{callback: sub1.callback}))
+    end
+
+    test_with_mock "it federates only to reachable instances via Salmon",
+                   Federator,
+                   [:passthrough],
+                   [] do
+      user = insert(:user)
+
+      remote_user1 =
+        insert(:user, %{
+          local: false,
+          nickname: "nick1@domain.com",
+          ap_id: "https://domain.com/users/nick1",
+          info: %{salmon: "https://domain.com/salmon"}
+        })
+
+      remote_user2 =
+        insert(:user, %{
+          local: false,
+          nickname: "nick2@domain2.com",
+          ap_id: "https://domain2.com/users/nick2",
+          info: %{salmon: "https://domain2.com/salmon"}
+        })
+
+      Instances.set_unreachable("domain.com", Instances.reachability_datetime_threshold())
+
+      {:ok, _activity} =
+        CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"})
+
+      assert called(Federator.enqueue(:publish_single_salmon, {remote_user2, :_, :_}))
+      refute called(Federator.enqueue(:publish_single_websub, {remote_user1, :_, :_}))
+    end
+  end
+
   describe "Receive an activity" do
     test "successfully processes incoming AP docs with correct origin" do
       params = %{
diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs
index 995cc00d6..ca447aa5d 100644
--- a/test/web/ostatus/ostatus_controller_test.exs
+++ b/test/web/ostatus/ostatus_controller_test.exs
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.OStatus.OStatusControllerTest do
   use Pleroma.Web.ConnCase
   import Pleroma.Factory
-  alias Pleroma.{User, Repo, Object}
+  alias Pleroma.{User, Repo, Object, Instances}
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.OStatus.ActivityRepresenter
 
@@ -14,49 +14,69 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do
     :ok
   end
 
-  test "decodes a salmon", %{conn: conn} do
-    user = insert(:user)
-    salmon = File.read!("test/fixtures/salmon.xml")
+  describe "salmon_incoming" do
+    test "decodes a salmon", %{conn: conn} do
+      user = insert(:user)
+      salmon = File.read!("test/fixtures/salmon.xml")
 
-    conn =
-      conn
-      |> put_req_header("content-type", "application/atom+xml")
-      |> post("/users/#{user.nickname}/salmon", salmon)
+      conn =
+        conn
+        |> put_req_header("content-type", "application/atom+xml")
+        |> post("/users/#{user.nickname}/salmon", salmon)
 
-    assert response(conn, 200)
-  end
+      assert response(conn, 200)
+    end
 
-  test "decodes a salmon with a changed magic key", %{conn: conn} do
-    user = insert(:user)
-    salmon = File.read!("test/fixtures/salmon.xml")
+    test "decodes a salmon with a changed magic key", %{conn: conn} do
+      user = insert(:user)
+      salmon = File.read!("test/fixtures/salmon.xml")
 
-    conn =
-      conn
-      |> put_req_header("content-type", "application/atom+xml")
-      |> post("/users/#{user.nickname}/salmon", salmon)
+      conn =
+        conn
+        |> put_req_header("content-type", "application/atom+xml")
+        |> post("/users/#{user.nickname}/salmon", salmon)
 
-    assert response(conn, 200)
+      assert response(conn, 200)
 
-    # Set a wrong magic-key for a user so it has to refetch
-    salmon_user = User.get_by_ap_id("http://gs.example.org:4040/index.php/user/1")
-    # Wrong key
-    info_cng =
-      User.Info.remote_user_creation(salmon_user.info, %{
-        magic_key:
-          "RSA.pu0s-halox4tu7wmES1FVSx6u-4wc0YrUFXcqWXZG4-27UmbCOpMQftRCldNRfyA-qLbz-eqiwrong1EwUvjsD4cYbAHNGHwTvDOyx5AKthQUP44ykPv7kjKGh3DWKySJvcs9tlUG87hlo7AvnMo9pwRS_Zz2CacQ-MKaXyDepk=.AQAB"
-      })
+      # Set a wrong magic-key for a user so it has to refetch
+      salmon_user = User.get_by_ap_id("http://gs.example.org:4040/index.php/user/1")
+      # Wrong key
+      info_cng =
+        User.Info.remote_user_creation(salmon_user.info, %{
+          magic_key:
+            "RSA.pu0s-halox4tu7wmES1FVSx6u-4wc0YrUFXcqWXZG4-27UmbCOpMQftRCldNRfyA-qLbz-eqiwrong1EwUvjsD4cYbAHNGHwTvDOyx5AKthQUP44ykPv7kjKGh3DWKySJvcs9tlUG87hlo7AvnMo9pwRS_Zz2CacQ-MKaXyDepk=.AQAB"
+        })
 
-    salmon_user
-    |> Ecto.Changeset.change()
-    |> Ecto.Changeset.put_embed(:info, info_cng)
-    |> Repo.update()
+      salmon_user
+      |> Ecto.Changeset.change()
+      |> Ecto.Changeset.put_embed(:info, info_cng)
+      |> Repo.update()
 
-    conn =
-      build_conn()
-      |> put_req_header("content-type", "application/atom+xml")
-      |> post("/users/#{user.nickname}/salmon", salmon)
+      conn =
+        build_conn()
+        |> put_req_header("content-type", "application/atom+xml")
+        |> post("/users/#{user.nickname}/salmon", salmon)
 
-    assert response(conn, 200)
+      assert response(conn, 200)
+    end
+
+    test "it clears `unreachable` federation status of the sender", %{conn: conn} do
+      sender_url = "https://pleroma.soykaf.com"
+      Instances.set_unreachable(sender_url, Instances.reachability_datetime_threshold())
+      refute Instances.reachable?(sender_url)
+
+      user = insert(:user)
+      salmon = File.read!("test/fixtures/salmon.xml")
+
+      conn =
+        conn
+        |> put_req_header("content-type", "application/atom+xml")
+        |> put_req_header("referer", sender_url)
+        |> post("/users/#{user.nickname}/salmon", salmon)
+
+      assert response(conn, 200)
+      assert Instances.reachable?(sender_url)
+    end
   end
 
   test "gets a feed", %{conn: conn} do
diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs
index 9cbcda063..c445ed676 100644
--- a/test/web/websub/websub_controller_test.exs
+++ b/test/web/websub/websub_controller_test.exs
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do
   use Pleroma.Web.ConnCase
   import Pleroma.Factory
   alias Pleroma.Web.Websub.WebsubClientSubscription
-  alias Pleroma.{Repo, Activity}
+  alias Pleroma.{Repo, Activity, Instances}
   alias Pleroma.Web.Websub
 
   test "websub subscription request", %{conn: conn} do
@@ -50,35 +50,57 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do
     assert_in_delta NaiveDateTime.diff(websub.valid_until, NaiveDateTime.utc_now()), 100, 5
   end
 
-  test "handles incoming feed updates", %{conn: conn} do
-    websub = insert(:websub_client_subscription)
-    doc = "some stuff"
-    signature = Websub.sign(websub.secret, doc)
+  describe "websub_incoming" do
+    test "handles incoming feed updates", %{conn: conn} do
+      websub = insert(:websub_client_subscription)
+      doc = "some stuff"
+      signature = Websub.sign(websub.secret, doc)
 
-    conn =
-      conn
-      |> put_req_header("x-hub-signature", "sha1=" <> signature)
-      |> put_req_header("content-type", "application/atom+xml")
-      |> post("/push/subscriptions/#{websub.id}", doc)
+      conn =
+        conn
+        |> put_req_header("x-hub-signature", "sha1=" <> signature)
+        |> put_req_header("content-type", "application/atom+xml")
+        |> post("/push/subscriptions/#{websub.id}", doc)
 
-    assert response(conn, 200) == "OK"
+      assert response(conn, 200) == "OK"
 
-    assert length(Repo.all(Activity)) == 1
-  end
+      assert length(Repo.all(Activity)) == 1
+    end
 
-  test "rejects incoming feed updates with the wrong signature", %{conn: conn} do
-    websub = insert(:websub_client_subscription)
-    doc = "some stuff"
-    signature = Websub.sign("wrong secret", doc)
+    test "rejects incoming feed updates with the wrong signature", %{conn: conn} do
+      websub = insert(:websub_client_subscription)
+      doc = "some stuff"
+      signature = Websub.sign("wrong secret", doc)
 
-    conn =
-      conn
-      |> put_req_header("x-hub-signature", "sha1=" <> signature)
-      |> put_req_header("content-type", "application/atom+xml")
-      |> post("/push/subscriptions/#{websub.id}", doc)
+      conn =
+        conn
+        |> put_req_header("x-hub-signature", "sha1=" <> signature)
+        |> put_req_header("content-type", "application/atom+xml")
+        |> post("/push/subscriptions/#{websub.id}", doc)
 
-    assert response(conn, 500) == "Error"
+      assert response(conn, 500) == "Error"
 
-    assert length(Repo.all(Activity)) == 0
+      assert length(Repo.all(Activity)) == 0
+    end
+
+    test "it clears `unreachable` federation status of the sender", %{conn: conn} do
+      sender_url = "https://pleroma.soykaf.com"
+      Instances.set_unreachable(sender_url, Instances.reachability_datetime_threshold())
+      refute Instances.reachable?(sender_url)
+
+      websub = insert(:websub_client_subscription)
+      doc = "some stuff"
+      signature = Websub.sign(websub.secret, doc)
+
+      conn =
+        conn
+        |> put_req_header("x-hub-signature", "sha1=" <> signature)
+        |> put_req_header("content-type", "application/atom+xml")
+        |> put_req_header("referer", sender_url)
+        |> post("/push/subscriptions/#{websub.id}", doc)
+
+      assert response(conn, 200) == "OK"
+      assert Instances.reachable?(sender_url)
+    end
   end
 end

From 9560abea102b8cd4927c9350bbd0a1a2f1800ea6 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Mon, 28 Jan 2019 11:03:52 +0300
Subject: [PATCH 09/12] [#534] Refactoring / tweaks per MR review.

---
 lib/pleroma/instances/instance.ex             | 34 ++++++++-----------
 .../plugs/set_requester_reachable_plug.ex     | 16 +++++++++
 lib/pleroma/web/activity_pub/activity_pub.ex  | 10 +-----
 .../activity_pub/activity_pub_controller.ex   |  7 +---
 lib/pleroma/web/controller_helper.ex          |  5 ---
 lib/pleroma/web/ostatus/ostatus_controller.ex |  7 +---
 lib/pleroma/web/salmon/salmon.ex              |  3 +-
 lib/pleroma/web/websub/websub.ex              |  3 +-
 lib/pleroma/web/websub/websub_controller.ex   |  7 +---
 .../20190123125546_create_instances.exs       |  1 -
 10 files changed, 36 insertions(+), 57 deletions(-)
 create mode 100644 lib/pleroma/plugs/set_requester_reachable_plug.ex

diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index 60e8d0e21..e3af4a8a7 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -13,7 +13,6 @@ defmodule Pleroma.Instances.Instance do
   schema "instances" do
     field(:host, :string)
     field(:unreachable_since, :naive_datetime)
-    field(:reachability_checked_at, :naive_datetime)
 
     timestamps()
   end
@@ -22,7 +21,7 @@ defmodule Pleroma.Instances.Instance do
 
   def changeset(struct, params \\ %{}) do
     struct
-    |> cast(params, [:host, :unreachable_since, :reachability_checked_at])
+    |> cast(params, [:host, :unreachable_since])
     |> validate_required([:host])
     |> unique_constraint(:host)
   end
@@ -66,7 +65,7 @@ defmodule Pleroma.Instances.Instance do
          %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do
       {:ok, _instance} =
         existing_record
-        |> changeset(%{unreachable_since: nil, reachability_checked_at: DateTime.utc_now()})
+        |> changeset(%{unreachable_since: nil})
         |> Repo.update()
     end
   end
@@ -80,27 +79,22 @@ defmodule Pleroma.Instances.Instance do
     host = host(url)
     existing_record = Repo.get_by(Instance, %{host: host})
 
-    changes = %{
-      unreachable_since: unreachable_since,
-      reachability_checked_at: NaiveDateTime.utc_now()
-    }
+    changes = %{unreachable_since: unreachable_since}
 
-    if existing_record do
-      update_changes =
-        if existing_record.unreachable_since &&
-             NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt,
-           do: Map.delete(changes, :unreachable_since),
-           else: changes
-
-      {:ok, _instance} =
-        existing_record
-        |> changeset(update_changes)
-        |> Repo.update()
-    else
-      {:ok, _instance} =
+    cond do
+      is_nil(existing_record) ->
         %Instance{}
         |> changeset(Map.put(changes, :host, host))
         |> Repo.insert()
+
+      existing_record.unreachable_since &&
+          NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt ->
+        {:noop, existing_record}
+
+      true ->
+        existing_record
+        |> changeset(changes)
+        |> Repo.update()
     end
   end
 
diff --git a/lib/pleroma/plugs/set_requester_reachable_plug.ex b/lib/pleroma/plugs/set_requester_reachable_plug.ex
new file mode 100644
index 000000000..88551be70
--- /dev/null
+++ b/lib/pleroma/plugs/set_requester_reachable_plug.ex
@@ -0,0 +1,16 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Plugs.SetRequesterReachablePlug do
+  import Plug.Conn
+
+  def init(_), do: []
+
+  def call(%Plug.Conn{} = conn, _) do
+    with [referer] <- get_req_header(conn, "referer"),
+         do: Pleroma.Instances.set_reachable(referer)
+
+    conn
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 4b34334a0..4232d6c0a 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -722,15 +722,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end)
   end
 
-  def publish_one(%{inbox: inbox} = activity) do
-    if Instances.reachable?(inbox) do
-      do_publish_one(activity)
-    else
-      {:error, :noop}
-    end
-  end
-
-  defp do_publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
+  def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
     Logger.info("Federating #{id} to #{inbox}")
     host = URI.parse(inbox).host
 
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index dc353dff0..fadb038a2 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -18,8 +18,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   action_fallback(:errors)
 
   plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay])
+  plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:inbox])
   plug(:relay_active? when action in [:relay])
-  plug(:set_requester_reachable when action in [:inbox])
 
   def relay_active?(conn, _) do
     if Keyword.get(Application.get_env(:pleroma, :instance), :allow_relay) do
@@ -291,9 +291,4 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     |> put_status(500)
     |> json("error")
   end
-
-  defp set_requester_reachable(conn, _) do
-    Pleroma.Web.ControllerHelper.set_requester_reachable(conn)
-    conn
-  end
 end
diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex
index 13cf1877f..14e3d19fd 100644
--- a/lib/pleroma/web/controller_helper.ex
+++ b/lib/pleroma/web/controller_helper.ex
@@ -10,9 +10,4 @@ defmodule Pleroma.Web.ControllerHelper do
     |> put_status(status)
     |> json(json)
   end
-
-  def set_requester_reachable(conn) do
-    with [referer] <- get_req_header(conn, "referer"),
-         do: Pleroma.Instances.set_reachable(referer)
-  end
 end
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index a89f16b94..e483447ed 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -15,7 +15,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Web.ActivityPub.ActivityPub
 
   plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming])
-  plug(:set_requester_reachable when action in [:salmon_incoming])
+  plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:salmon_incoming])
 
   action_fallback(:errors)
 
@@ -203,9 +203,4 @@ defmodule Pleroma.Web.OStatus.OStatusController do
     |> put_status(500)
     |> text("Something went wrong")
   end
-
-  defp set_requester_reachable(conn, _) do
-    Pleroma.Web.ControllerHelper.set_requester_reachable(conn)
-    conn
-  end
 end
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index 17ca7a6e8..80023127c 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -168,8 +168,7 @@ defmodule Pleroma.Web.Salmon do
     do: send_to_user(salmon, feed, poster)
 
   def send_to_user(url, feed, poster) when is_binary(url) do
-    with {:reachable, true} <- {:reachable, Instances.reachable?(url)},
-         {:ok, %{status: code}} when code in 200..299 <-
+    with {:ok, %{status: code}} when code in 200..299 <-
            poster.(
              url,
              feed,
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index cbb7a5ac7..64eba7221 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -272,8 +272,7 @@ defmodule Pleroma.Web.Websub do
     signature = sign(secret || "", xml)
     Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
 
-    with {:reachable, true} <- {:reachable, Instances.reachable?(callback)},
-         {:ok, %{status: code}} when code in 200..299 <-
+    with {:ok, %{status: code}} when code in 200..299 <-
            @httpoison.post(
              callback,
              xml,
diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex
index 02fe075d7..9da7e70a1 100644
--- a/lib/pleroma/web/websub/websub_controller.ex
+++ b/lib/pleroma/web/websub/websub_controller.ex
@@ -20,7 +20,7 @@ defmodule Pleroma.Web.Websub.WebsubController do
          ]
   )
 
-  plug(:set_requester_reachable when action in [:websub_incoming])
+  plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:websub_incoming])
 
   def websub_subscription_request(conn, %{"nickname" => nickname} = params) do
     user = User.get_cached_by_nickname(nickname)
@@ -96,9 +96,4 @@ defmodule Pleroma.Web.Websub.WebsubController do
         |> send_resp(500, "Error")
     end
   end
-
-  defp set_requester_reachable(conn, _) do
-    Pleroma.Web.ControllerHelper.set_requester_reachable(conn)
-    conn
-  end
 end
diff --git a/priv/repo/migrations/20190123125546_create_instances.exs b/priv/repo/migrations/20190123125546_create_instances.exs
index 4f5915fba..b527ad7ec 100644
--- a/priv/repo/migrations/20190123125546_create_instances.exs
+++ b/priv/repo/migrations/20190123125546_create_instances.exs
@@ -5,7 +5,6 @@ defmodule Pleroma.Repo.Migrations.CreateInstances do
     create table(:instances) do
       add :host, :string
       add :unreachable_since, :naive_datetime
-      add :reachability_checked_at, :naive_datetime
 
       timestamps()
     end

From 1d2f41642cfec5710055bcf8409778bb362beecb Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Mon, 28 Jan 2019 15:25:06 +0300
Subject: [PATCH 10/12] [#534] Various tweaks. Tests for Instances and
 Instance.

---
 config/config.exs                             |   2 +-
 lib/pleroma/instances.ex                      |  13 ++-
 lib/pleroma/instances/instance.ex             |  28 ++---
 lib/pleroma/web/activity_pub/activity_pub.ex  |   4 +-
 lib/pleroma/web/salmon/salmon.ex              |   4 -
 lib/pleroma/web/websub/websub.ex              |  10 +-
 test/support/factory.ex                       |   7 ++
 test/support/http_request_mock.ex             |  28 +++++
 .../activity_pub_controller_test.exs          |   4 +-
 test/web/activity_pub/activity_pub_test.exs   |  43 ++++++-
 test/web/federator_test.exs                   |   4 +-
 test/web/instances/instance_test.exs          | 107 ++++++++++++++++++
 test/web/instances/instances_test.exs         |  94 +++++++++++++++
 test/web/ostatus/ostatus_controller_test.exs  |   2 +-
 test/web/websub/websub_controller_test.exs    |   2 +-
 15 files changed, 312 insertions(+), 40 deletions(-)
 create mode 100644 test/web/instances/instance_test.exs
 create mode 100644 test/web/instances/instances_test.exs

diff --git a/config/config.exs b/config/config.exs
index 7a1a875c9..c0a936b17 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -125,7 +125,7 @@ config :pleroma, :instance,
   banner_upload_limit: 4_000_000,
   registrations_open: true,
   federating: true,
-  federation_reachability_timeout_days: 90,
+  federation_reachability_timeout_days: 7,
   allow_relay: true,
   rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy,
   public: true,
diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex
index 0b08f0eb8..5e107f4c9 100644
--- a/lib/pleroma/instances.ex
+++ b/lib/pleroma/instances.ex
@@ -3,14 +3,17 @@ defmodule Pleroma.Instances do
 
   @adapter Pleroma.Instances.Instance
 
-  defdelegate filter_reachable(urls), to: @adapter
-  defdelegate reachable?(url), to: @adapter
-  defdelegate set_reachable(url), to: @adapter
-  defdelegate set_unreachable(url, unreachable_since \\ nil), to: @adapter
+  defdelegate filter_reachable(urls_or_hosts), to: @adapter
+  defdelegate reachable?(url_or_host), to: @adapter
+  defdelegate set_reachable(url_or_host), to: @adapter
+  defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter
+
+  def set_consistently_unreachable(url_or_host),
+    do: set_unreachable(url_or_host, reachability_datetime_threshold())
 
   def reachability_datetime_threshold do
     federation_reachability_timeout_days =
-      Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 90
+      Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 0
 
     if federation_reachability_timeout_days > 0 do
       NaiveDateTime.add(
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index e3af4a8a7..a87590d8b 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -17,7 +17,7 @@ defmodule Pleroma.Instances.Instance do
     timestamps()
   end
 
-  defdelegate host(url), to: Instances
+  defdelegate host(url_or_host), to: Instances
 
   def changeset(struct, params \\ %{}) do
     struct
@@ -28,9 +28,9 @@ defmodule Pleroma.Instances.Instance do
 
   def filter_reachable([]), do: []
 
-  def filter_reachable(urls) when is_list(urls) do
+  def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
     hosts =
-      urls
+      urls_or_hosts
       |> Enum.map(&(&1 && host(&1)))
       |> Enum.filter(&(to_string(&1) != ""))
 
@@ -44,14 +44,14 @@ defmodule Pleroma.Instances.Instance do
         )
       )
 
-    Enum.filter(urls, &(&1 && host(&1) not in unreachable_hosts))
+    Enum.filter(urls_or_hosts, &(&1 && host(&1) not in unreachable_hosts))
   end
 
-  def reachable?(url) when is_binary(url) do
+  def reachable?(url_or_host) when is_binary(url_or_host) do
     !Repo.one(
       from(i in Instance,
         where:
-          i.host == ^host(url) and
+          i.host == ^host(url_or_host) and
             i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
         select: true
       )
@@ -60,8 +60,8 @@ defmodule Pleroma.Instances.Instance do
 
   def reachable?(_), do: true
 
-  def set_reachable(url) when is_binary(url) do
-    with host <- host(url),
+  def set_reachable(url_or_host) when is_binary(url_or_host) do
+    with host <- host(url_or_host),
          %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do
       {:ok, _instance} =
         existing_record
@@ -70,13 +70,13 @@ defmodule Pleroma.Instances.Instance do
     end
   end
 
-  def set_reachable(_), do: {0, :noop}
+  def set_reachable(_), do: {:error, nil}
 
-  def set_unreachable(url, unreachable_since \\ nil)
+  def set_unreachable(url_or_host, unreachable_since \\ nil)
 
-  def set_unreachable(url, unreachable_since) when is_binary(url) do
+  def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do
     unreachable_since = unreachable_since || DateTime.utc_now()
-    host = host(url)
+    host = host(url_or_host)
     existing_record = Repo.get_by(Instance, %{host: host})
 
     changes = %{unreachable_since: unreachable_since}
@@ -89,7 +89,7 @@ defmodule Pleroma.Instances.Instance do
 
       existing_record.unreachable_since &&
           NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt ->
-        {:noop, existing_record}
+        {:ok, existing_record}
 
       true ->
         existing_record
@@ -98,5 +98,5 @@ defmodule Pleroma.Instances.Instance do
     end
   end
 
-  def set_unreachable(_, _), do: {0, :noop}
+  def set_unreachable(_, _), do: {:error, nil}
 end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 4232d6c0a..6cad02da6 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -750,9 +750,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       Instances.set_reachable(inbox)
       result
     else
-      e ->
+      {_post_result, response} ->
         Instances.set_unreachable(inbox)
-        e
+        {:error, response}
     end
   end
 
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index 80023127c..e96455423 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -181,10 +181,6 @@ defmodule Pleroma.Web.Salmon do
       Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
       :ok
     else
-      {:reachable, false} ->
-        Logger.debug(fn -> "Pushing Salmon to #{url} skipped as marked unreachable)" end)
-        :noop
-
       e ->
         Instances.set_unreachable(url)
         Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index 64eba7221..abe148270 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -286,14 +286,10 @@ defmodule Pleroma.Web.Websub do
       Logger.info(fn -> "Pushed to #{callback}, code #{code}" end)
       {:ok, code}
     else
-      {:reachable, false} ->
-        Logger.debug(fn -> "Pushing to #{callback} skipped as marked unreachable)" end)
-        {:error, :noop}
-
-      e ->
+      {_post_result, response} ->
         Instances.set_unreachable(callback)
-        Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end)
-        {:error, e}
+        Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(response)}" end)
+        {:error, response}
     end
   end
 end
diff --git a/test/support/factory.ex b/test/support/factory.ex
index 964b2b61c..0c21093ce 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -220,4 +220,11 @@ defmodule Pleroma.Factory do
       client_secret: "aaa;/&bbb"
     }
   end
+
+  def instance_factory do
+    %Pleroma.Instances.Instance{
+      host: "domain.com",
+      unreachable_since: nil
+    }
+  end
 end
diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex
index e4279e14d..3d6efd52c 100644
--- a/test/support/http_request_mock.ex
+++ b/test/support/http_request_mock.ex
@@ -653,6 +653,14 @@ defmodule HttpRequestMock do
     {:ok, Tesla.Mock.json(%{"id" => "https://social.heldscal.la/user/23211"}, status: 200)}
   end
 
+  def get("http://404.site" <> _, _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 404,
+       body: ""
+     }}
+  end
+
   def get(url, query, body, headers) do
     {:error,
      "Not implemented the mock response for get #{inspect(url)}, #{query}, #{inspect(body)}, #{
@@ -673,6 +681,26 @@ defmodule HttpRequestMock do
      }}
   end
 
+  def post("http://200.site" <> _, _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: ""
+     }}
+  end
+
+  def post("http://connrefused.site" <> _, _, _, _) do
+    {:error, :connrefused}
+  end
+
+  def post("http://404.site" <> _, _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 404,
+       body: ""
+     }}
+  end
+
   def post(url, _query, _body, _headers) do
     {:error, "Not implemented the mock response for post #{inspect(url)}"}
   end
diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs
index 1b704330f..eca5c134d 100644
--- a/test/web/activity_pub/activity_pub_controller_test.exs
+++ b/test/web/activity_pub/activity_pub_controller_test.exs
@@ -146,7 +146,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
 
     test "it clears `unreachable` federation status of the sender", %{conn: conn} do
       sender_url = "https://pleroma.soykaf.com"
-      Instances.set_unreachable(sender_url, Instances.reachability_datetime_threshold())
+      Instances.set_consistently_unreachable(sender_url)
       refute Instances.reachable?(sender_url)
 
       data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!()
@@ -211,7 +211,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
 
     test "it clears `unreachable` federation status of the sender", %{conn: conn} do
       sender_host = "pleroma.soykaf.com"
-      Instances.set_unreachable(sender_host, Instances.reachability_datetime_threshold())
+      Instances.set_consistently_unreachable(sender_host)
       refute Instances.reachable?(sender_host)
 
       user = insert(:user)
diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index 18f094379..d517c7aa7 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -7,11 +7,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.{Activity, Object, User}
+  alias Pleroma.{Activity, Object, User, Instances}
   alias Pleroma.Builders.ActivityBuilder
 
   import Pleroma.Factory
   import Tesla.Mock
+  import Mock
 
   setup do
     mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
@@ -659,6 +660,46 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
     assert 3 = length(activities)
   end
 
+  describe "publish_one/1" do
+    test_with_mock "it calls `Instances.set_unreachable` on target inbox on non-2xx HTTP response code",
+                   Instances,
+                   [:passthrough],
+                   [] do
+      actor = insert(:user)
+      inbox = "http://404.site/users/nick1/inbox"
+
+      assert {:error, _} =
+               ActivityPub.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1})
+
+      assert called(Instances.set_unreachable(inbox))
+    end
+
+    test_with_mock "it calls `Instances.set_unreachable` on target inbox on request error of any kind",
+                   Instances,
+                   [:passthrough],
+                   [] do
+      actor = insert(:user)
+      inbox = "http://connrefused.site/users/nick1/inbox"
+
+      assert {:error, _} =
+               ActivityPub.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1})
+
+      assert called(Instances.set_unreachable(inbox))
+    end
+
+    test_with_mock "it does NOT call `Instances.set_unreachable` if target is reachable",
+                   Instances,
+                   [:passthrough],
+                   [] do
+      actor = insert(:user)
+      inbox = "http://200.site/users/nick1/inbox"
+
+      assert {:ok, _} = ActivityPub.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1})
+
+      refute called(Instances.set_unreachable(inbox))
+    end
+  end
+
   def data_uri do
     File.read!("test/fixtures/avatar_data_uri")
   end
diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs
index f4234aea8..c6d10ef78 100644
--- a/test/web/federator_test.exs
+++ b/test/web/federator_test.exs
@@ -128,7 +128,7 @@ defmodule Pleroma.Web.FederatorTest do
           callback: "https://pleroma2.soykaf.com/cb"
         })
 
-      Instances.set_unreachable(sub1.callback, Instances.reachability_datetime_threshold())
+      Instances.set_consistently_unreachable(sub1.callback)
 
       {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI"})
 
@@ -158,7 +158,7 @@ defmodule Pleroma.Web.FederatorTest do
           info: %{salmon: "https://domain2.com/salmon"}
         })
 
-      Instances.set_unreachable("domain.com", Instances.reachability_datetime_threshold())
+      Instances.set_consistently_unreachable("domain.com")
 
       {:ok, _activity} =
         CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"})
diff --git a/test/web/instances/instance_test.exs b/test/web/instances/instance_test.exs
new file mode 100644
index 000000000..a158c0a42
--- /dev/null
+++ b/test/web/instances/instance_test.exs
@@ -0,0 +1,107 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Instances.InstanceTest do
+  alias Pleroma.Repo
+  alias Pleroma.Instances.Instance
+
+  use Pleroma.DataCase
+
+  import Pleroma.Factory
+
+  setup_all do
+    config_path = [:instance, :federation_reachability_timeout_days]
+    initial_setting = Pleroma.Config.get(config_path)
+
+    Pleroma.Config.put(config_path, 1)
+    on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end)
+
+    :ok
+  end
+
+  describe "set_reachable/1" do
+    test "clears `unreachable_since` of existing matching Instance record having non-nil `unreachable_since`" do
+      instance = insert(:instance, unreachable_since: NaiveDateTime.utc_now())
+
+      assert {:ok, instance} = Instance.set_reachable(instance.host)
+      refute instance.unreachable_since
+    end
+
+    test "keeps nil `unreachable_since` of existing matching Instance record having nil `unreachable_since`" do
+      instance = insert(:instance, unreachable_since: nil)
+
+      assert {:ok, instance} = Instance.set_reachable(instance.host)
+      refute instance.unreachable_since
+    end
+
+    test "does NOT create an Instance record in case of no existing matching record" do
+      host = "domain.org"
+      assert nil == Instance.set_reachable(host)
+
+      assert [] = Repo.all(Ecto.Query.from(i in Instance))
+      assert Instance.reachable?(host)
+    end
+  end
+
+  describe "set_unreachable/1" do
+    test "creates new record having `unreachable_since` to current time if record does not exist" do
+      assert {:ok, instance} = Instance.set_unreachable("https://domain.com/path")
+
+      instance = Repo.get(Instance, instance.id)
+      assert instance.unreachable_since
+      assert "domain.com" == instance.host
+    end
+
+    test "sets `unreachable_since` of existing record having nil `unreachable_since`" do
+      instance = insert(:instance, unreachable_since: nil)
+      refute instance.unreachable_since
+
+      assert {:ok, _} = Instance.set_unreachable(instance.host)
+
+      instance = Repo.get(Instance, instance.id)
+      assert instance.unreachable_since
+    end
+
+    test "does NOT modify `unreachable_since` value of existing record in case it's present" do
+      instance =
+        insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))
+
+      assert instance.unreachable_since
+      initial_value = instance.unreachable_since
+
+      assert {:ok, _} = Instance.set_unreachable(instance.host)
+
+      instance = Repo.get(Instance, instance.id)
+      assert initial_value == instance.unreachable_since
+    end
+  end
+
+  describe "set_unreachable/2" do
+    test "sets `unreachable_since` value of existing record in case it's newer than supplied value" do
+      instance =
+        insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))
+
+      assert instance.unreachable_since
+
+      past_value = NaiveDateTime.add(NaiveDateTime.utc_now(), -100)
+      assert {:ok, _} = Instance.set_unreachable(instance.host, past_value)
+
+      instance = Repo.get(Instance, instance.id)
+      assert past_value == instance.unreachable_since
+    end
+
+    test "does NOT modify `unreachable_since` value of existing record in case it's equal to or older than supplied value" do
+      instance =
+        insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))
+
+      assert instance.unreachable_since
+      initial_value = instance.unreachable_since
+
+      assert {:ok, _} = Instance.set_unreachable(instance.host, NaiveDateTime.utc_now())
+
+      instance = Repo.get(Instance, instance.id)
+      assert initial_value == instance.unreachable_since
+    end
+  end
+end
diff --git a/test/web/instances/instances_test.exs b/test/web/instances/instances_test.exs
new file mode 100644
index 000000000..a2fdf1019
--- /dev/null
+++ b/test/web/instances/instances_test.exs
@@ -0,0 +1,94 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.InstancesTest do
+  alias Pleroma.Instances
+
+  use Pleroma.DataCase
+
+  setup_all do
+    config_path = [:instance, :federation_reachability_timeout_days]
+    initial_setting = Pleroma.Config.get(config_path)
+
+    Pleroma.Config.put(config_path, 1)
+    on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end)
+
+    :ok
+  end
+
+  describe "reachable?/1" do
+    test "returns `true` for host / url with unknown reachability status" do
+      assert Instances.reachable?("unknown.site")
+      assert Instances.reachable?("http://unknown.site")
+    end
+
+    test "returns `false` for host / url marked unreachable for at least `reachability_datetime_threshold()`" do
+      host = "consistently-unreachable.name"
+      Instances.set_consistently_unreachable(host)
+
+      refute Instances.reachable?(host)
+      refute Instances.reachable?("http://#{host}/path")
+    end
+
+    test "returns `true` for host / url marked unreachable for less than `reachability_datetime_threshold()`" do
+      url = "http://eventually-unreachable.name/path"
+
+      Instances.set_unreachable(url)
+
+      assert Instances.reachable?(url)
+      assert Instances.reachable?(URI.parse(url).host)
+    end
+  end
+
+  describe "filter_reachable/1" do
+    test "keeps only reachable elements of supplied list" do
+      host = "consistently-unreachable.name"
+      url1 = "http://eventually-unreachable.com/path"
+      url2 = "http://domain.com/path"
+
+      Instances.set_consistently_unreachable(host)
+      Instances.set_unreachable(url1)
+
+      assert [url1, url2] == Instances.filter_reachable([host, url1, url2])
+    end
+  end
+
+  describe "set_reachable/1" do
+    test "sets unreachable url or host reachable" do
+      host = "domain.com"
+      Instances.set_consistently_unreachable(host)
+      refute Instances.reachable?(host)
+
+      Instances.set_reachable(host)
+      assert Instances.reachable?(host)
+    end
+
+    test "keeps reachable url or host reachable" do
+      url = "https://site.name?q="
+      assert Instances.reachable?(url)
+
+      Instances.set_reachable(url)
+      assert Instances.reachable?(url)
+    end
+  end
+
+  describe "set_consistently_unreachable/1" do
+    test "sets reachable url or host unreachable" do
+      url = "http://domain.com?q="
+      assert Instances.reachable?(url)
+
+      Instances.set_consistently_unreachable(url)
+      refute Instances.reachable?(url)
+    end
+
+    test "keeps unreachable url or host unreachable" do
+      host = "site.name"
+      Instances.set_consistently_unreachable(host)
+      refute Instances.reachable?(host)
+
+      Instances.set_consistently_unreachable(host)
+      refute Instances.reachable?(host)
+    end
+  end
+end
diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs
index ca447aa5d..ad9bc418a 100644
--- a/test/web/ostatus/ostatus_controller_test.exs
+++ b/test/web/ostatus/ostatus_controller_test.exs
@@ -62,7 +62,7 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do
 
     test "it clears `unreachable` federation status of the sender", %{conn: conn} do
       sender_url = "https://pleroma.soykaf.com"
-      Instances.set_unreachable(sender_url, Instances.reachability_datetime_threshold())
+      Instances.set_consistently_unreachable(sender_url)
       refute Instances.reachable?(sender_url)
 
       user = insert(:user)
diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs
index c445ed676..cb19d6fe6 100644
--- a/test/web/websub/websub_controller_test.exs
+++ b/test/web/websub/websub_controller_test.exs
@@ -85,7 +85,7 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do
 
     test "it clears `unreachable` federation status of the sender", %{conn: conn} do
       sender_url = "https://pleroma.soykaf.com"
-      Instances.set_unreachable(sender_url, Instances.reachability_datetime_threshold())
+      Instances.set_consistently_unreachable(sender_url)
       refute Instances.reachable?(sender_url)
 
       websub = insert(:websub_client_subscription)

From 92753b0cd9cfcdc5edb64a5e55ad27f73079f9e0 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Tue, 29 Jan 2019 13:12:28 +0300
Subject: [PATCH 11/12] [#534] Made federation push sender be determined basing
 on content instead of `referer` header. Updated tests.

---
 .../plugs/set_requester_reachable_plug.ex     | 16 --------------
 lib/pleroma/reverse_proxy.ex                  |  3 +--
 lib/pleroma/web/activity_pub/activity_pub.ex  |  3 +--
 .../activity_pub/activity_pub_controller.ex   | 11 +++++++++-
 lib/pleroma/web/ostatus/ostatus.ex            |  3 +++
 lib/pleroma/web/ostatus/ostatus_controller.ex |  1 -
 lib/pleroma/web/salmon/salmon.ex              |  5 +----
 lib/pleroma/web/websub/websub.ex              |  3 +--
 lib/pleroma/web/websub/websub_controller.ex   |  2 --
 .../activity_pub_controller_test.exs          | 16 ++++++--------
 test/web/instances/instances_test.exs         | 18 +++++++++++++++
 .../delete_handling_test.exs                  |  7 ++++++
 test/web/ostatus/ostatus_controller_test.exs  | 20 +----------------
 test/web/ostatus/ostatus_test.exs             | 18 ++++++++++++++-
 test/web/websub/websub_controller_test.exs    | 22 +------------------
 15 files changed, 68 insertions(+), 80 deletions(-)
 delete mode 100644 lib/pleroma/plugs/set_requester_reachable_plug.ex

diff --git a/lib/pleroma/plugs/set_requester_reachable_plug.ex b/lib/pleroma/plugs/set_requester_reachable_plug.ex
deleted file mode 100644
index 88551be70..000000000
--- a/lib/pleroma/plugs/set_requester_reachable_plug.ex
+++ /dev/null
@@ -1,16 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.Plugs.SetRequesterReachablePlug do
-  import Plug.Conn
-
-  def init(_), do: []
-
-  def call(%Plug.Conn{} = conn, _) do
-    with [referer] <- get_req_header(conn, "referer"),
-         do: Pleroma.Instances.set_reachable(referer)
-
-    conn
-  end
-end
diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex
index d8b17212b..a25b5ea4e 100644
--- a/lib/pleroma/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy.ex
@@ -3,8 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy do
-  @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-unmodified-since) ++
-                      ~w(if-none-match if-range range referer)
+  @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-unmodified-since if-none-match if-range range)
   @resp_cache_headers ~w(etag date last-modified cache-control)
   @keep_resp_headers @resp_cache_headers ++
                        ~w(content-type content-disposition content-encoding content-range accept-ranges vary)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 22c7824fa..4016808e8 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -784,8 +784,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
                [
                  {"Content-Type", "application/activity+json"},
                  {"signature", signature},
-                 {"digest", digest},
-                 {"referer", Pleroma.Web.Endpoint.url()}
+                 {"digest", digest}
                ]
              ) do
       Instances.set_reachable(inbox)
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index fadb038a2..4dea6ab83 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -18,7 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   action_fallback(:errors)
 
   plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay])
-  plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:inbox])
+  plug(:set_requester_reachable when action in [:inbox])
   plug(:relay_active? when action in [:relay])
 
   def relay_active?(conn, _) do
@@ -291,4 +291,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     |> put_status(500)
     |> json("error")
   end
+
+  defp set_requester_reachable(%Plug.Conn{} = conn, _) do
+    with actor <- conn.params["actor"],
+         true <- is_binary(actor) do
+      Pleroma.Instances.set_reachable(actor)
+    end
+
+    conn
+  end
 end
diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex
index a3155b79d..a20ca17bb 100644
--- a/lib/pleroma/web/ostatus/ostatus.ex
+++ b/lib/pleroma/web/ostatus/ostatus.ex
@@ -48,6 +48,9 @@ defmodule Pleroma.Web.OStatus do
 
   def handle_incoming(xml_string) do
     with doc when doc != :error <- parse_document(xml_string) do
+      with {:ok, actor_user} <- find_make_or_update_user(doc),
+           do: Pleroma.Instances.set_reachable(actor_user.ap_id)
+
       entries = :xmerl_xpath.string('//entry', doc)
 
       activities =
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index 9392a97f0..302ff38a4 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -14,7 +14,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Web.ActivityPub.ActivityPub
 
   plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming])
-  plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:salmon_incoming])
 
   action_fallback(:errors)
 
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index e96455423..07ca42a5f 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -172,10 +172,7 @@ defmodule Pleroma.Web.Salmon do
            poster.(
              url,
              feed,
-             [
-               {"Content-Type", "application/magic-envelope+xml"},
-               {"referer", Pleroma.Web.Endpoint.url()}
-             ]
+             [{"Content-Type", "application/magic-envelope+xml"}]
            ) do
       Instances.set_reachable(url)
       Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index abe148270..8f7d53b03 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -278,8 +278,7 @@ defmodule Pleroma.Web.Websub do
              xml,
              [
                {"Content-Type", "application/atom+xml"},
-               {"X-Hub-Signature", "sha1=#{signature}"},
-               {"referer", Pleroma.Web.Endpoint.url()}
+               {"X-Hub-Signature", "sha1=#{signature}"}
              ]
            ) do
       Instances.set_reachable(callback)
diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex
index 9da7e70a1..a92dfe87b 100644
--- a/lib/pleroma/web/websub/websub_controller.ex
+++ b/lib/pleroma/web/websub/websub_controller.ex
@@ -20,8 +20,6 @@ defmodule Pleroma.Web.Websub.WebsubController do
          ]
   )
 
-  plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:websub_incoming])
-
   def websub_subscription_request(conn, %{"nickname" => nickname} = params) do
     user = User.get_cached_by_nickname(nickname)
 
diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs
index eca5c134d..d3dd160dd 100644
--- a/test/web/activity_pub/activity_pub_controller_test.exs
+++ b/test/web/activity_pub/activity_pub_controller_test.exs
@@ -145,17 +145,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
     end
 
     test "it clears `unreachable` federation status of the sender", %{conn: conn} do
-      sender_url = "https://pleroma.soykaf.com"
+      data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!()
+
+      sender_url = data["actor"]
       Instances.set_consistently_unreachable(sender_url)
       refute Instances.reachable?(sender_url)
 
-      data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!()
-
       conn =
         conn
         |> assign(:valid_signature, true)
         |> put_req_header("content-type", "application/activity+json")
-        |> put_req_header("referer", sender_url)
         |> post("/inbox", data)
 
       assert "ok" == json_response(conn, 200)
@@ -210,10 +209,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
     end
 
     test "it clears `unreachable` federation status of the sender", %{conn: conn} do
-      sender_host = "pleroma.soykaf.com"
-      Instances.set_consistently_unreachable(sender_host)
-      refute Instances.reachable?(sender_host)
-
       user = insert(:user)
 
       data =
@@ -221,11 +216,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
         |> Poison.decode!()
         |> Map.put("bcc", [user.ap_id])
 
+      sender_host = URI.parse(data["actor"]).host
+      Instances.set_consistently_unreachable(sender_host)
+      refute Instances.reachable?(sender_host)
+
       conn =
         conn
         |> assign(:valid_signature, true)
         |> put_req_header("content-type", "application/activity+json")
-        |> put_req_header("referer", "https://#{sender_host}")
         |> post("/users/#{user.nickname}/inbox", data)
 
       assert "ok" == json_response(conn, 200)
diff --git a/test/web/instances/instances_test.exs b/test/web/instances/instances_test.exs
index a2fdf1019..adb8560a7 100644
--- a/test/web/instances/instances_test.exs
+++ b/test/web/instances/instances_test.exs
@@ -39,6 +39,11 @@ defmodule Pleroma.InstancesTest do
       assert Instances.reachable?(url)
       assert Instances.reachable?(URI.parse(url).host)
     end
+
+    test "returns true on non-binary input" do
+      assert Instances.reachable?(nil)
+      assert Instances.reachable?(1)
+    end
   end
 
   describe "filter_reachable/1" do
@@ -71,6 +76,19 @@ defmodule Pleroma.InstancesTest do
       Instances.set_reachable(url)
       assert Instances.reachable?(url)
     end
+
+    test "returns error status on non-binary input" do
+      assert {:error, _} = Instances.set_reachable(nil)
+      assert {:error, _} = Instances.set_reachable(1)
+    end
+  end
+
+  # Note: implementation-specific (e.g. Instance) details of set_unreachable/1 should be tested in implementation-specific tests
+  describe "set_unreachable/1" do
+    test "returns error status on non-binary input" do
+      assert {:error, _} = Instances.set_unreachable(nil)
+      assert {:error, _} = Instances.set_unreachable(1)
+    end
   end
 
   describe "set_consistently_unreachable/1" do
diff --git a/test/web/ostatus/incoming_documents/delete_handling_test.exs b/test/web/ostatus/incoming_documents/delete_handling_test.exs
index c8fbff6cc..d97cd79f4 100644
--- a/test/web/ostatus/incoming_documents/delete_handling_test.exs
+++ b/test/web/ostatus/incoming_documents/delete_handling_test.exs
@@ -2,9 +2,16 @@ defmodule Pleroma.Web.OStatus.DeleteHandlingTest do
   use Pleroma.DataCase
 
   import Pleroma.Factory
+  import Tesla.Mock
+
   alias Pleroma.{Repo, Activity, Object}
   alias Pleroma.Web.OStatus
 
+  setup do
+    mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
   describe "deletions" do
     test "it removes the mentioned activity" do
       note = insert(:note_activity)
diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs
index cba12b3f7..3145ca9a1 100644
--- a/test/web/ostatus/ostatus_controller_test.exs
+++ b/test/web/ostatus/ostatus_controller_test.exs
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.OStatus.OStatusControllerTest do
   use Pleroma.Web.ConnCase
   import Pleroma.Factory
-  alias Pleroma.{User, Repo, Object, Instances}
+  alias Pleroma.{User, Repo, Object}
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.OStatus.ActivityRepresenter
 
@@ -59,24 +59,6 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do
 
       assert response(conn, 200)
     end
-
-    test "it clears `unreachable` federation status of the sender", %{conn: conn} do
-      sender_url = "https://pleroma.soykaf.com"
-      Instances.set_consistently_unreachable(sender_url)
-      refute Instances.reachable?(sender_url)
-
-      user = insert(:user)
-      salmon = File.read!("test/fixtures/salmon.xml")
-
-      conn =
-        conn
-        |> put_req_header("content-type", "application/atom+xml")
-        |> put_req_header("referer", sender_url)
-        |> post("/users/#{user.nickname}/salmon", salmon)
-
-      assert response(conn, 200)
-      assert Instances.reachable?(sender_url)
-    end
   end
 
   test "gets a feed", %{conn: conn} do
diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs
index 403cc7095..0c63dd84d 100644
--- a/test/web/ostatus/ostatus_test.exs
+++ b/test/web/ostatus/ostatus_test.exs
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.OStatusTest do
   use Pleroma.DataCase
   alias Pleroma.Web.OStatus
   alias Pleroma.Web.XML
-  alias Pleroma.{Object, Repo, User, Activity}
+  alias Pleroma.{Object, Repo, User, Activity, Instances}
   import Pleroma.Factory
   import ExUnit.CaptureLog
 
@@ -311,6 +311,22 @@ defmodule Pleroma.Web.OStatusTest do
     refute User.following?(follower, followed)
   end
 
+  test "it clears `unreachable` federation status of the sender" do
+    incoming_reaction_xml = File.read!("test/fixtures/share-gs.xml")
+    doc = XML.parse_document(incoming_reaction_xml)
+    actor_uri = XML.string_from_xpath("//author/uri[1]", doc)
+    reacted_to_author_uri = XML.string_from_xpath("//author/uri[2]", doc)
+
+    Instances.set_consistently_unreachable(actor_uri)
+    Instances.set_consistently_unreachable(reacted_to_author_uri)
+    refute Instances.reachable?(actor_uri)
+    refute Instances.reachable?(reacted_to_author_uri)
+
+    {:ok, _} = OStatus.handle_incoming(incoming_reaction_xml)
+    assert Instances.reachable?(actor_uri)
+    refute Instances.reachable?(reacted_to_author_uri)
+  end
+
   describe "new remote user creation" do
     test "returns local users" do
       local_user = insert(:user)
diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs
index cb19d6fe6..6492df2a0 100644
--- a/test/web/websub/websub_controller_test.exs
+++ b/test/web/websub/websub_controller_test.exs
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do
   use Pleroma.Web.ConnCase
   import Pleroma.Factory
   alias Pleroma.Web.Websub.WebsubClientSubscription
-  alias Pleroma.{Repo, Activity, Instances}
+  alias Pleroma.{Repo, Activity}
   alias Pleroma.Web.Websub
 
   test "websub subscription request", %{conn: conn} do
@@ -82,25 +82,5 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do
 
       assert length(Repo.all(Activity)) == 0
     end
-
-    test "it clears `unreachable` federation status of the sender", %{conn: conn} do
-      sender_url = "https://pleroma.soykaf.com"
-      Instances.set_consistently_unreachable(sender_url)
-      refute Instances.reachable?(sender_url)
-
-      websub = insert(:websub_client_subscription)
-      doc = "some stuff"
-      signature = Websub.sign(websub.secret, doc)
-
-      conn =
-        conn
-        |> put_req_header("x-hub-signature", "sha1=" <> signature)
-        |> put_req_header("content-type", "application/atom+xml")
-        |> put_req_header("referer", sender_url)
-        |> post("/push/subscriptions/#{websub.id}", doc)
-
-      assert response(conn, 200) == "OK"
-      assert Instances.reachable?(sender_url)
-    end
   end
 end

From a7c55c61bb581ddbd8159261e167b08486547b20 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 31 Jan 2019 23:23:53 +0100
Subject: [PATCH 12/12] docs/Pleroma-API.md: Add missing token field
 documentation

[ci skip]
---
 docs/Pleroma-API.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/Pleroma-API.md b/docs/Pleroma-API.md
index 3ebea565c..e1448d3f0 100644
--- a/docs/Pleroma-API.md
+++ b/docs/Pleroma-API.md
@@ -52,6 +52,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
     * `confirm`
     * `captcha_solution`: optional, contains provider-specific captcha solution,
     * `captcha_token`: optional, contains provider-specific captcha token
+    * `token`: invite token required when the registerations aren't public.
 * Response: JSON. Returns a user object on success, otherwise returns `{"error": "error_msg"}`
 * Example response:
 ```