Anon/pleroma
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'feature/admin-api-status-count-per-instance' into 'develop'

Browse Source 
admin api status count per instance

See merge request pleroma/pleroma!2500
This commit is contained in:
lain 2020-06-24 10:42:07 +00:00
commit 88ccade1cb
10 changed files with 324 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
CHANGELOG.md
View File

@ -15,9 +15,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
<details>
<summary>API Changes</summary>
- **Breaking:** Emoji API: changed methods and renamed routes.
</details>
<details>
<summary>Admin API Changes</summary>
- Status visibility stats: now can return stats per instance.
- Mix task to refresh counter cache (`mix pleroma.refresh_counter_cache`)
</details>
### Removed
- **Breaking:** removed `with_move` parameter from notifications timeline.
@ -98,6 +107,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
2. Run database migrations (inside Pleroma directory):
- OTP: `./bin/pleroma_ctl migrate`
- From Source: `mix ecto.migrate`
3. Reset status visibility counters (inside Pleroma directory):
- OTP: `./bin/pleroma_ctl refresh_counter_cache`
- From Source: `mix pleroma.refresh_counter_cache`
## [2.0.2] - 2020-04-08

4
docs/API/admin_api.md
View File

@ -1118,6 +1118,10 @@ Loads json generated from `config/descriptions.exs`.
### Stats
- Query Params:
- *optional* `instance`: **string** instance hostname (without protocol) to get stats for
- Example: `https://mypleroma.org/api/pleroma/admin/stats?instance=lain.com`
- Response:
```json

49
lib/mix/tasks/pleroma/refresh_counter_cache.ex
View File

@ -17,30 +17,53 @@ defmodule Mix.Tasks.Pleroma.RefreshCounterCache do
def run([]) do
Mix.Pleroma.start_pleroma()
["public", "unlisted", "private", "direct"]
|> Enum.each(fn visibility ->
count = status_visibility_count_query(visibility)
name = "status_visibility_#{visibility}"
CounterCache.set(name, count)
Mix.Pleroma.shell_info("Set #{name} to #{count}")
instances =
Activity
|> distinct([a], true)
|> select([a], fragment("split_part(?, '/', 3)", a.actor))
|> Repo.all()
instances
|> Enum.with_index(1)
|> Enum.each(fn {instance, i} ->
counters = instance_counters(instance)
CounterCache.set(instance, counters)
Mix.Pleroma.shell_info(
"[#{i}/#{length(instances)}] Setting #{instance} counters: #{inspect(counters)}"
)
end)
Mix.Pleroma.shell_info("Done")
end
defp status_visibility_count_query(visibility) do
defp instance_counters(instance) do
counters = %{"public" => 0, "unlisted" => 0, "private" => 0, "direct" => 0}
Activity
|> where(
|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
|> where([a], fragment("split_part(?, '/', 3) = ?", a.actor, ^instance))
|> select(
[a],
{fragment(
"activity_visibility(?, ?, ?)",
a.actor,
a.recipients,
a.data
), count(a.id)}
)
|> group_by(
[a],
fragment(
"activity_visibility(?, ?, ?) = ?",
"activity_visibility(?, ?, ?)",
a.actor,
a.recipients,
a.data,
^visibility
a.data
)
)
|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
|> Repo.aggregate(:count, :id, timeout: :timer.minutes(30))
|> Repo.all(timeout: :timer.minutes(30))
|> Enum.reduce(counters, fn {visibility, count}, acc ->
Map.put(acc, visibility, count)
end)
end
end

66
lib/pleroma/counter_cache.ex
View File

@ -10,32 +10,70 @@ defmodule Pleroma.CounterCache do
import Ecto.Query
schema "counter_cache" do
field(:name, :string)
field(:count, :integer)
field(:instance, :string)
field(:public, :integer)
field(:unlisted, :integer)
field(:private, :integer)
field(:direct, :integer)
end
def changeset(struct, params) do
struct
|> cast(params, [:name, :count])
|> validate_required([:name])
|> unique_constraint(:name)
|> cast(params, [:instance, :public, :unlisted, :private, :direct])
|> validate_required([:instance])
|> unique_constraint(:instance)
end
def get_as_map(names) when is_list(names) do
def get_by_instance(instance) do
CounterCache
|> where([cc], cc.name in ^names)
|> Repo.all()
|> Enum.group_by(& &1.name, & &1.count)
|> Map.new(fn {k, v} -> {k, hd(v)} end)
|> select([c], %{
"public" => c.public,
"unlisted" => c.unlisted,
"private" => c.private,
"direct" => c.direct
})
|> where([c], c.instance == ^instance)
|> Repo.one()
|> case do
nil -> %{"public" => 0, "unlisted" => 0, "private" => 0, "direct" => 0}
val -> val
end
end
def set(name, count) do
def get_sum do
CounterCache
|> select([c], %{
"public" => type(sum(c.public), :integer),
"unlisted" => type(sum(c.unlisted), :integer),
"private" => type(sum(c.private), :integer),
"direct" => type(sum(c.direct), :integer)
})
|> Repo.one()
end
def set(instance, values) do
params =
Enum.reduce(
["public", "private", "unlisted", "direct"],
%{"instance" => instance},
fn param, acc ->
Map.put_new(acc, param, Map.get(values, param, 0))
end
)
%CounterCache{}
|> changeset(%{"name" => name, "count" => count})
|> changeset(params)
|> Repo.insert(
on_conflict: [set: [count: count]],
on_conflict: [
set: [
public: params["public"],
private: params["private"],
unlisted: params["unlisted"],
direct: params["direct"]
]
],
returning: true,
conflict_target: :name
conflict_target: :instance
)
end
end

21
lib/pleroma/stats.ex
View File

@ -97,20 +97,11 @@ defmodule Pleroma.Stats do
}
end
def get_status_visibility_count do
counter_cache =
CounterCache.get_as_map([
"status_visibility_public",
"status_visibility_private",
"status_visibility_unlisted",
"status_visibility_direct"
])
%{
public: counter_cache["status_visibility_public"] || 0,
unlisted: counter_cache["status_visibility_unlisted"] || 0,
private: counter_cache["status_visibility_private"] || 0,
direct: counter_cache["status_visibility_direct"] || 0
}
def get_status_visibility_count(instance \\ nil) do
if is_nil(instance) do
CounterCache.get_sum()
else
CounterCache.get_by_instance(instance)
end
end
end

6
lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
View File

@ -643,10 +643,10 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
json(conn, "")
end
def stats(conn, _) do
count = Stats.get_status_visibility_count()
def stats(conn, params) do
counters = Stats.get_status_visibility_count(params["instance"])
json(conn, %{"status_visibility" => count})
json(conn, %{"status_visibility" => counters})
end
defp page_params(params) do

143
priv/repo/migrations/20200508092434_update_counter_cache_table.exs Normal file
View File

@ -0,0 +1,143 @@
defmodule Pleroma.Repo.Migrations.UpdateCounterCacheTable do
use Ecto.Migration
@function_name "update_status_visibility_counter_cache"
@trigger_name "status_visibility_counter_cache_trigger"
def up do
execute("drop trigger if exists #{@trigger_name} on activities")
execute("drop function if exists #{@function_name}()")
drop_if_exists(unique_index(:counter_cache, [:name]))
drop_if_exists(table(:counter_cache))
create_if_not_exists table(:counter_cache) do
add(:instance, :string, null: false)
add(:direct, :bigint, null: false, default: 0)
add(:private, :bigint, null: false, default: 0)
add(:unlisted, :bigint, null: false, default: 0)
add(:public, :bigint, null: false, default: 0)
end
create_if_not_exists(unique_index(:counter_cache, [:instance]))
"""
CREATE OR REPLACE FUNCTION #{@function_name}()
RETURNS TRIGGER AS
$$
DECLARE
hostname character varying(255);
visibility_new character varying(64);
visibility_old character varying(64);
actor character varying(255);
BEGIN
IF TG_OP = 'DELETE' THEN
actor := OLD.actor;
ELSE
actor := NEW.actor;
END IF;
hostname := split_part(actor, '/', 3);
IF TG_OP = 'INSERT' THEN
visibility_new := activity_visibility(NEW.actor, NEW.recipients, NEW.data);
IF NEW.data->>'type' = 'Create'
AND visibility_new IN ('public', 'unlisted', 'private', 'direct') THEN
EXECUTE format('INSERT INTO "counter_cache" ("instance", %1$I) VALUES ($1, 1)
ON CONFLICT ("instance") DO
UPDATE SET %1$I = "counter_cache".%1$I + 1', visibility_new)
USING hostname;
END IF;
RETURN NEW;
ELSIF TG_OP = 'UPDATE' THEN
visibility_new := activity_visibility(NEW.actor, NEW.recipients, NEW.data);
visibility_old := activity_visibility(OLD.actor, OLD.recipients, OLD.data);
IF (NEW.data->>'type' = 'Create')
AND (OLD.data->>'type' = 'Create')
AND visibility_new != visibility_old
AND visibility_new IN ('public', 'unlisted', 'private', 'direct') THEN
EXECUTE format('UPDATE "counter_cache" SET
%1$I = greatest("counter_cache".%1$I - 1, 0),
%2$I = "counter_cache".%2$I + 1
WHERE "instance" = $1', visibility_old, visibility_new)
USING hostname;
END IF;
RETURN NEW;
ELSIF TG_OP = 'DELETE' THEN
IF OLD.data->>'type' = 'Create' THEN
visibility_old := activity_visibility(OLD.actor, OLD.recipients, OLD.data);
EXECUTE format('UPDATE "counter_cache" SET
%1$I = greatest("counter_cache".%1$I - 1, 0)
WHERE "instance" = $1', visibility_old)
USING hostname;
END IF;
RETURN OLD;
END IF;
END;
$$
LANGUAGE 'plpgsql';
"""
|> execute()
execute("DROP TRIGGER IF EXISTS #{@trigger_name} ON activities")
"""
CREATE TRIGGER #{@trigger_name}
BEFORE
INSERT
OR UPDATE of recipients, data
OR DELETE
ON activities
FOR EACH ROW
EXECUTE PROCEDURE #{@function_name}();
"""
|> execute()
end
def down do
execute("DROP TRIGGER IF EXISTS #{@trigger_name} ON activities")
execute("DROP FUNCTION IF EXISTS #{@function_name}()")
drop_if_exists(unique_index(:counter_cache, [:instance]))
drop_if_exists(table(:counter_cache))
create_if_not_exists table(:counter_cache) do
add(:name, :string, null: false)
add(:count, :bigint, null: false, default: 0)
end
create_if_not_exists(unique_index(:counter_cache, [:name]))
"""
CREATE OR REPLACE FUNCTION #{@function_name}()
RETURNS TRIGGER AS
$$
DECLARE
BEGIN
IF TG_OP = 'INSERT' THEN
IF NEW.data->>'type' = 'Create' THEN
EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1';
END IF;
RETURN NEW;
ELSIF TG_OP = 'UPDATE' THEN
IF (NEW.data->>'type' = 'Create') and (OLD.data->>'type' = 'Create') and activity_visibility(NEW.actor, NEW.recipients, NEW.data) != activity_visibility(OLD.actor, OLD.recipients, OLD.data) THEN
EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1';
EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';';
END IF;
RETURN NEW;
ELSIF TG_OP = 'DELETE' THEN
IF OLD.data->>'type' = 'Create' THEN
EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';';
END IF;
RETURN OLD;
END IF;
END;
$$
LANGUAGE 'plpgsql';
"""
|> execute()
"""
CREATE TRIGGER #{@trigger_name} BEFORE INSERT OR UPDATE of recipients, data OR DELETE ON activities
FOR EACH ROW
EXECUTE PROCEDURE #{@function_name}();
"""
|> execute()
end
end

55
test/stats_test.exs
View File

@ -17,10 +17,11 @@ defmodule Pleroma.StatsTest do
end
end
describe "status visibility count" do
describe "status visibility sum count" do
test "on new status" do
instance2 = "instance2.tld"
user = insert(:user)
other_user = insert(:user)
other_user = insert(:user, %{ap_id: "https://#{instance2}/@actor"})
CommonAPI.post(user, %{visibility: "public", status: "hey"})
@ -45,24 +46,24 @@ defmodule Pleroma.StatsTest do
})
end)
assert %{direct: 3, private: 4, public: 1, unlisted: 2} =
assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} =
Pleroma.Stats.get_status_visibility_count()
end
test "on status delete" do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
assert %{public: 1} = Pleroma.Stats.get_status_visibility_count()
assert %{"public" => 1} = Pleroma.Stats.get_status_visibility_count()
CommonAPI.delete(activity.id, user)
assert %{public: 0} = Pleroma.Stats.get_status_visibility_count()
assert %{"public" => 0} = Pleroma.Stats.get_status_visibility_count()
end
test "on status visibility update" do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
assert %{public: 1, private: 0} = Pleroma.Stats.get_status_visibility_count()
assert %{"public" => 1, "private" => 0} = Pleroma.Stats.get_status_visibility_count()
{:ok, _} = CommonAPI.update_activity_scope(activity.id, %{visibility: "private"})
assert %{public: 0, private: 1} = Pleroma.Stats.get_status_visibility_count()
assert %{"public" => 0, "private" => 1} = Pleroma.Stats.get_status_visibility_count()
end
test "doesn't count unrelated activities" do
@ -73,8 +74,46 @@ defmodule Pleroma.StatsTest do
CommonAPI.favorite(other_user, activity.id)
CommonAPI.repeat(activity.id, other_user)
assert %{direct: 0, private: 0, public: 1, unlisted: 0} =
assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 0} =
Pleroma.Stats.get_status_visibility_count()
end
end
describe "status visibility by instance count" do
test "single instance" do
local_instance = Pleroma.Web.Endpoint.url() |> String.split("//") |> Enum.at(1)
instance2 = "instance2.tld"
user1 = insert(:user)
user2 = insert(:user, %{ap_id: "https://#{instance2}/@actor"})
CommonAPI.post(user1, %{visibility: "public", status: "hey"})
Enum.each(1..5, fn _ ->
CommonAPI.post(user1, %{
visibility: "unlisted",
status: "hey"
})
end)
Enum.each(1..10, fn _ ->
CommonAPI.post(user1, %{
visibility: "direct",
status: "hey @#{user2.nickname}"
})
end)
Enum.each(1..20, fn _ ->
CommonAPI.post(user2, %{
visibility: "private",
status: "hey"
})
end)
assert %{"direct" => 10, "private" => 0, "public" => 1, "unlisted" => 5} =
Pleroma.Stats.get_status_visibility_count(local_instance)
assert %{"direct" => 0, "private" => 20, "public" => 0, "unlisted" => 0} =
Pleroma.Stats.get_status_visibility_count(instance2)
end
end
end

2
test/tasks/refresh_counter_cache_test.exs
View File

@ -37,7 +37,7 @@ defmodule Mix.Tasks.Pleroma.RefreshCounterCacheTest do
assert capture_io(fn -> Mix.Tasks.Pleroma.RefreshCounterCache.run([]) end) =~ "Done\n"
assert %{direct: 3, private: 4, public: 1, unlisted: 2} =
assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} =
Pleroma.Stats.get_status_visibility_count()
end
end

20
test/web/admin_api/controllers/admin_api_controller_test.exs
View File

@ -1732,6 +1732,26 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 2} =
response["status_visibility"]
end
test "by instance", %{conn: conn} do
admin = insert(:user, is_admin: true)
user1 = insert(:user)
instance2 = "instance2.tld"
user2 = insert(:user, %{ap_id: "https://#{instance2}/@actor"})
CommonAPI.post(user1, %{visibility: "public", status: "hey"})
CommonAPI.post(user2, %{visibility: "unlisted", status: "hey"})
CommonAPI.post(user2, %{visibility: "private", status: "hey"})
response =
conn
|> assign(:user, admin)
|> get("/api/pleroma/admin/stats", instance: instance2)
|> json_response(200)
assert %{"direct" => 0, "private" => 1, "public" => 0, "unlisted" => 1} =
response["status_visibility"]
end
end
end