Anon/pleroma
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'feature/account-export' into 'develop'

Browse Source 
Add account export

Closes #847

See merge request pleroma/pleroma!2918
This commit is contained in:
feld 2020-10-31 17:03:40 +00:00
commit 38b481d112
19 changed files with 999 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Experimental websocket-based federation between Pleroma instances.
- Support pagination of blocks and mutes
- App metrics: ability to restrict access to specified IP whitelist.
- Account backup
- Configuration: Add `:instance, autofollowing_nicknames` setting to provide a way to make accounts automatically follow new users that register on the local Pleroma instance.
### Changed

6
config/config.exs
View File

@ -551,6 +551,7 @@ config :pleroma, Oban,
queues: [
activity_expiration: 10,
token_expiration: 5,
backup: 1,
federator_incoming: 50,
federator_outgoing: 50,
ingestion_queue: 50,
@ -835,6 +836,11 @@ config :floki, :html_parser, Floki.HTMLParser.FastHtml
config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
config :pleroma, Pleroma.User.Backup,
purge_after_days: 30,
limit_days: 7,
dir: nil
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"

26
config/description.exs
View File

@ -2297,6 +2297,12 @@ config :pleroma, :config_description, [
description: "Activity expiration queue",
suggestions: [10]
},
%{
key: :backup,
type: :integer,
description: "Backup queue",
suggestions: [1]
},
%{
key: :attachments_cleanup,
type: :integer,
@ -3732,6 +3738,26 @@ config :pleroma, :config_description, [
}
]
},
%{
group: :pleroma,
key: Pleroma.User.Backup,
type: :group,
description: "Account Backup",
children: [
%{
key: :purge_after_days,
type: :integer,
description: "Remove backup achives after N days",
suggestions: [30]
},
%{
key: :limit_days,
type: :integer,
description: "Limit user to export not more often than once per N days",
suggestions: [7]
}
]
},
%{
group: :prometheus,
key: Pleroma.Web.Endpoint.MetricsExporter,

38
docs/API/pleroma_api.md
View File

@ -615,3 +615,41 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
{"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]}
]
```
## `POST /api/v1/pleroma/backups`
### Create a user backup archive
* Method: `POST`
* Authentication: required
* Params: none
* Response: JSON
* Example response:
```json
[{
"content_type": "application/zip",
"file_size": 0,
"inserted_at": "2020-09-10T16:18:03.000Z",
"processed": false,
"url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
}]
```
## `GET /api/v1/pleroma/backups`
### Lists user backups
* Method: `GET`
* Authentication: not required
* Params: none
* Response: JSON
* Example response:
```json
[{
"content_type": "application/zip",
"file_size": 55457,
"inserted_at": "2020-09-10T16:18:03.000Z",
"processed": true,
"url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
}]
```

14
docs/configuration/cheatsheet.md
View File

@ -1078,6 +1078,20 @@ Control favicons for instances.
* `enabled`: Allow/disallow displaying and getting instances favicons
## Pleroma.User.Backup
!!! note
Requires enabled email
* `:purge_after_days` an integer, remove backup achives after N days.
* `:limit_days` an integer, limit user to export not more often than once per N days.
* `:dir` a string with a path to backup temporary directory or `nil` to let Pleroma choose temporary directory in the following order:
1. the directory named by the TMPDIR environment variable
2. the directory named by the TEMP environment variable
3. the directory named by the TMP environment variable
4. C:\TMP on Windows or /tmp on Unix-like operating systems
5. as a last resort, the current working directory
## Frontend management
Frontends in Pleroma are swappable - you can specify which one to use here.

26
lib/pleroma/emails/user_email.ex
View File

@ -189,4 +189,30 @@ defmodule Pleroma.Emails.UserEmail do
Router.Helpers.subscription_url(Endpoint, :unsubscribe, token)
end
def backup_is_ready_email(backup, admin_user_id \\ nil) do
%{user: user} = Pleroma.Repo.preload(backup, :user)
download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
html_body =
if is_nil(admin_user_id) do
"""
<p>You requested a full backup of your Pleroma account. It's ready for download:</p>
<p><a href="#{download_url}">#{download_url}</a></p>
"""
else
admin = Pleroma.Repo.get(User, admin_user_id)
"""
<p>Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:</p>
<p><a href="#{download_url}">#{download_url}</a></p>
"""
end
new()
|> to(recipient(user))
|> from(sender())
|> subject("Your account archive is ready")
|> html_body(html_body)
end
end

10
lib/pleroma/moderation_log.ex
View File

@ -655,6 +655,16 @@ defmodule Pleroma.ModerationLog do
"@#{actor_nickname} deleted chat message ##{subject_id}"
end
def get_log_entry_message(%ModerationLog{
data: %{
"actor" => %{"nickname" => actor_nickname},
"action" => "create_backup",
"subject" => %{"nickname" => user_nickname}
}
}) do
"@#{actor_nickname} requested account backup for @#{user_nickname}"
end
defp nicknames_to_string(nicknames) do
nicknames
|> Enum.map(&"@#{&1}")

258
lib/pleroma/user/backup.ex Normal file
View File

@ -0,0 +1,258 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.User.Backup do
use Ecto.Schema
import Ecto.Changeset
import Ecto.Query
import Pleroma.Web.Gettext
require Pleroma.Constants
alias Pleroma.Activity
alias Pleroma.Bookmark
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.UserView
alias Pleroma.Workers.BackupWorker
schema "backups" do
field(:content_type, :string)
field(:file_name, :string)
field(:file_size, :integer, default: 0)
field(:processed, :boolean, default: false)
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
timestamps()
end
def create(user, admin_id \\ nil) do
with :ok <- validate_email_enabled(),
:ok <- validate_user_email(user),
:ok <- validate_limit(user, admin_id),
{:ok, backup} <- user |> new() |> Repo.insert() do
BackupWorker.process(backup, admin_id)
end
end
def new(user) do
rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
%__MODULE__{
user_id: user.id,
content_type: "application/zip",
file_name: name
}
end
def delete(backup) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do
Repo.delete(backup)
end
end
defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok
defp validate_limit(user, nil) do
case get_last(user.id) do
%__MODULE__{inserted_at: inserted_at} ->
days = Pleroma.Config.get([__MODULE__, :limit_days])
diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
if diff > days do
:ok
else
{:error,
dngettext(
"errors",
"Last export was less than a day ago",
"Last export was less than %{days} days ago",
days,
days: days
)}
end
nil ->
:ok
end
end
defp validate_email_enabled do
if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
:ok
else
{:error, dgettext("errors", "Backups require enabled email")}
end
end
defp validate_user_email(%User{email: nil}) do
{:error, dgettext("errors", "Email is required")}
end
defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok
def get_last(user_id) do
__MODULE__
|> where(user_id: ^user_id)
|> order_by(desc: :id)
|> limit(1)
|> Repo.one()
end
def list(%User{id: user_id}) do
__MODULE__
|> where(user_id: ^user_id)
|> order_by(desc: :id)
|> Repo.all()
end
def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
__MODULE__
|> where(user_id: ^user_id)
|> where([b], b.id != ^latest_id)
|> Repo.all()
|> Enum.each(&BackupWorker.delete/1)
end
def get(id), do: Repo.get(__MODULE__, id)
def process(%__MODULE__{} = backup) do
with {:ok, zip_file} <- export(backup),
{:ok, %{size: size}} <- File.stat(zip_file),
{:ok, _upload} <- upload(backup, zip_file) do
backup
|> cast(%{file_size: size, processed: true}, [:file_size, :processed])
|> Repo.update()
end
end
@files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
def export(%__MODULE__{} = backup) do
backup = Repo.preload(backup, :user)
name = String.trim_trailing(backup.file_name, ".zip")
dir = dir(name)
with :ok <- File.mkdir(dir),
:ok <- actor(dir, backup.user),
:ok <- statuses(dir, backup.user),
:ok <- likes(dir, backup.user),
:ok <- bookmarks(dir, backup.user),
{:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
{:ok, _} <- File.rm_rf(dir) do
{:ok, to_string(zip_path)}
end
end
def dir(name) do
dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!()
Path.join(dir, name)
end
def upload(%__MODULE__{} = backup, zip_path) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
upload = %Pleroma.Upload{
name: backup.file_name,
tempfile: zip_path,
content_type: backup.content_type,
path: Path.join("backups", backup.file_name)
}
with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
:ok <- File.rm(zip_path) do
{:ok, upload}
end
end
defp actor(dir, user) do
with {:ok, json} <-
UserView.render("user.json", %{user: user})
|> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
|> Jason.encode() do
File.write(Path.join(dir, "actor.json"), json)
end
end
defp write_header(file, name) do
IO.write(
file,
"""
{
"@context": "https://www.w3.org/ns/activitystreams",
"id": "#{name}.json",
"type": "OrderedCollection",
"orderedItems": [
"""
)
end
defp write(query, dir, name, fun) do
path = Path.join(dir, "#{name}.json")
with {:ok, file} <- File.open(path, [:write, :utf8]),
:ok <- write_header(file, name) do
total =
query
|> Pleroma.Repo.chunk_stream(100)
|> Enum.reduce(0, fn i, acc ->
with {:ok, data} <- fun.(i),
{:ok, str} <- Jason.encode(data),
:ok <- IO.write(file, str <> ",\n") do
acc + 1
else
_ -> acc
end
end)
with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do
File.close(file)
end
end
end
defp bookmarks(dir, %{id: user_id} = _user) do
Bookmark
|> where(user_id: ^user_id)
|> join(:inner, [b], activity in assoc(b, :activity))
|> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
|> write(dir, "bookmarks", fn a -> {:ok, a.object} end)
end
defp likes(dir, user) do
user.ap_id
|> Activity.Queries.by_actor()
|> Activity.Queries.by_type("Like")
|> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
|> write(dir, "likes", fn a -> {:ok, a.object} end)
end
defp statuses(dir, user) do
opts =
%{}
|> Map.put(:type, ["Create", "Announce"])
|> Map.put(:actor_id, user.ap_id)
[
[Pleroma.Constants.as_public(), user.ap_id],
User.following(user),
Pleroma.List.memberships(user)
]
|> Enum.concat()
|> ActivityPub.fetch_activities_query(opts)
|> write(dir, "outbox", fn a ->
with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
{:ok, Map.delete(activity, "@context")}
end
end)
end
end

11
lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
View File

@ -26,7 +26,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
plug(
OAuthScopesPlug,
%{scopes: ["read:accounts"], admin: true}
when action in [:right_get, :show_user_credentials]
when action in [:right_get, :show_user_credentials, :create_backup]
)
plug(
@ -441,6 +441,15 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
json(conn, %{"status_visibility" => counters})
end
def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
with %User{} = user <- User.get_by_nickname(nickname),
{:ok, _} <- Pleroma.User.Backup.create(user, admin.id) do
ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"})
json(conn, "")
end
end
defp page_params(params) do
{
fetch_integer_param(params, "page", 1),

79
lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex Normal file
View File

@ -0,0 +1,79 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do
alias OpenApiSpex.Operation
alias OpenApiSpex.Schema
alias Pleroma.Web.ApiSpec.Schemas.ApiError
def open_api_operation(action) do
operation = String.to_existing_atom("#{action}_operation")
apply(__MODULE__, operation, [])
end
def index_operation do
%Operation{
tags: ["Backups"],
summary: "List backups",
security: [%{"oAuth" => ["read:account"]}],
operationId: "PleromaAPI.BackupController.index",
responses: %{
200 =>
Operation.response(
"An array of backups",
"application/json",
%Schema{
type: :array,
items: backup()
}
),
400 => Operation.response("Bad Request", "application/json", ApiError)
}
}
end
def create_operation do
%Operation{
tags: ["Backups"],
summary: "Create a backup",
security: [%{"oAuth" => ["read:account"]}],
operationId: "PleromaAPI.BackupController.create",
responses: %{
200 =>
Operation.response(
"An array of backups",
"application/json",
%Schema{
type: :array,
items: backup()
}
),
400 => Operation.response("Bad Request", "application/json", ApiError)
}
}
end
defp backup do
%Schema{
title: "Backup",
description: "Response schema for a backup",
type: :object,
properties: %{
inserted_at: %Schema{type: :string, format: :"date-time"},
content_type: %Schema{type: :string},
file_name: %Schema{type: :string},
file_size: %Schema{type: :integer},
processed: %Schema{type: :boolean}
},
example: %{
"content_type" => "application/zip",
"file_name" =>
"https://cofe.fe:4000/media/backups/archive-foobar-20200908T164207-Yr7vuT5Wycv-sN3kSN2iJ0k-9pMo60j9qmvRCdDqIew.zip",
"file_size" => 4105,
"inserted_at" => "2020-09-08T16:42:07.000Z",
"processed" => true
}
}
end
end

28
lib/pleroma/web/pleroma_api/controllers/backup_controller.ex Normal file
View File

@ -0,0 +1,28 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.PleromaAPI.BackupController do
use Pleroma.Web, :controller
alias Pleroma.User.Backup
alias Pleroma.Web.Plugs.OAuthScopesPlug
action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create])
plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError)
defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation
def index(%{assigns: %{user: user}} = conn, _params) do
backups = Backup.list(user)
render(conn, "index.json", backups: backups)
end
def create(%{assigns: %{user: user}} = conn, _params) do
with {:ok, _} <- Backup.create(user) do
backups = Backup.list(user)
render(conn, "index.json", backups: backups)
end
end
end

28
lib/pleroma/web/pleroma_api/views/backup_view.ex Normal file
View File

@ -0,0 +1,28 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.PleromaAPI.BackupView do
use Pleroma.Web, :view
alias Pleroma.User.Backup
alias Pleroma.Web.CommonAPI.Utils
def render("show.json", %{backup: %Backup{} = backup}) do
%{
content_type: backup.content_type,
url: download_url(backup),
file_size: backup.file_size,
processed: backup.processed,
inserted_at: Utils.to_masto_date(backup.inserted_at)
}
end
def render("index.json", %{backups: backups}) do
render_many(backups, __MODULE__, "show.json")
end
def download_url(%Backup{file_name: file_name}) do
Pleroma.Web.Endpoint.url() <> "/media/backups/" <> file_name
end
end

5
lib/pleroma/web/router.ex
View File

@ -243,6 +243,8 @@ defmodule Pleroma.Web.Router do
get("/chats/:id", ChatController, :show)
get("/chats/:id/messages", ChatController, :messages)
delete("/chats/:id/messages/:message_id", ChatController, :delete_message)
post("/backups", AdminAPIController, :create_backup)
end
scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do
@ -373,6 +375,9 @@ defmodule Pleroma.Web.Router do
put("/mascot", MascotController, :update)
post("/scrobble", ScrobbleController, :create)
get("/backups", BackupController, :index)
post("/backups", BackupController, :create)
end
scope [] do

54
lib/pleroma/workers/backup_worker.ex Normal file
View File

@ -0,0 +1,54 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.BackupWorker do
use Oban.Worker, queue: :backup, max_attempts: 1
alias Oban.Job
alias Pleroma.User.Backup
def process(backup, admin_user_id \\ nil) do
%{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id}
|> new()
|> Oban.insert()
end
def schedule_deletion(backup) do
days = Pleroma.Config.get([Backup, :purge_after_days])
time = 60 * 60 * 24 * days
scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time)
%{"op" => "delete", "backup_id" => backup.id}
|> new(scheduled_at: scheduled_at)
|> Oban.insert()
end
def delete(backup) do
%{"op" => "delete", "backup_id" => backup.id}
|> new()
|> Oban.insert()
end
def perform(%Job{
args: %{"op" => "process", "backup_id" => backup_id, "admin_user_id" => admin_user_id}
}) do
with {:ok, %Backup{} = backup} <-
backup_id |> Backup.get() |> Backup.process(),
{:ok, _job} <- schedule_deletion(backup),
:ok <- Backup.remove_outdated(backup),
{:ok, _} <-
backup
|> Pleroma.Emails.UserEmail.backup_is_ready_email(admin_user_id)
|> Pleroma.Emails.Mailer.deliver() do
{:ok, backup}
end
end
def perform(%Job{args: %{"op" => "delete", "backup_id" => backup_id}}) do
case Backup.get(backup_id) do
%Backup{} = backup -> Backup.delete(backup)
nil -> :ok
end
end
end

17
priv/repo/migrations/20200831192323_create_backups.exs Normal file
View File

@ -0,0 +1,17 @@
defmodule Pleroma.Repo.Migrations.CreateBackups do
use Ecto.Migration
def change do
create_if_not_exists table(:backups) do
add(:user_id, references(:users, type: :uuid, on_delete: :delete_all))
add(:file_name, :string, null: false)
add(:content_type, :string, null: false)
add(:processed, :boolean, null: false, default: false)
add(:file_size, :bigint)
timestamps()
end
create_if_not_exists(index(:backups, [:user_id]))
end
end

244
test/pleroma/user/backup_test.exs Normal file
View File

@ -0,0 +1,244 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.User.BackupTest do
use Oban.Testing, repo: Pleroma.Repo
use Pleroma.DataCase
import Mock
import Pleroma.Factory
import Swoosh.TestAssertions
alias Pleroma.Bookmark
alias Pleroma.Tests.ObanHelpers
alias Pleroma.User.Backup
alias Pleroma.Web.CommonAPI
alias Pleroma.Workers.BackupWorker
setup do
clear_config([Pleroma.Upload, :uploader])
clear_config([Backup, :limit_days])
clear_config([Pleroma.Emails.Mailer, :enabled], true)
end
test "it requries enabled email" do
Pleroma.Config.put([Pleroma.Emails.Mailer, :enabled], false)
user = insert(:user)
assert {:error, "Backups require enabled email"} == Backup.create(user)
end
test "it requries user's email" do
user = insert(:user, %{email: nil})
assert {:error, "Email is required"} == Backup.create(user)
end
test "it creates a backup record and an Oban job" do
%{id: user_id} = user = insert(:user)
assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
assert_enqueued(worker: BackupWorker, args: args)
backup = Backup.get(args["backup_id"])
assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
end
test "it return an error if the export limit is over" do
%{id: user_id} = user = insert(:user)
limit_days = Pleroma.Config.get([Backup, :limit_days])
assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
backup = Backup.get(args["backup_id"])
assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"}
end
test "it process a backup record" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
%{id: user_id} = user = insert(:user)
assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user)
assert {:ok, backup} = perform_job(BackupWorker, args)
assert backup.file_size > 0
assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
delete_job_args = %{"op" => "delete", "backup_id" => backup_id}
assert_enqueued(worker: BackupWorker, args: delete_job_args)
assert {:ok, backup} = perform_job(BackupWorker, delete_job_args)
refute Backup.get(backup_id)
email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup)
assert_email_sent(
to: {user.name, user.email},
html_body: email.html_body
)
end
test "it removes outdated backups after creating a fresh one" do
Pleroma.Config.put([Backup, :limit_days], -1)
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
user = insert(:user)
assert {:ok, job1} = Backup.create(user)
assert {:ok, %Backup{}} = ObanHelpers.perform(job1)
assert {:ok, job2} = Backup.create(user)
assert Pleroma.Repo.aggregate(Backup, :count) == 2
assert {:ok, backup2} = ObanHelpers.perform(job2)
ObanHelpers.perform_all()
assert [^backup2] = Pleroma.Repo.all(Backup)
end
test "it creates a zip archive with user data" do
user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
{:ok, %{object: %{data: %{"id" => id1}}} = status1} =
CommonAPI.post(user, %{status: "status1"})
{:ok, %{object: %{data: %{"id" => id2}}} = status2} =
CommonAPI.post(user, %{status: "status2"})
{:ok, %{object: %{data: %{"id" => id3}}} = status3} =
CommonAPI.post(user, %{status: "status3"})
CommonAPI.favorite(user, status1.id)
CommonAPI.favorite(user, status2.id)
Bookmark.create(user.id, status2.id)
Bookmark.create(user.id, status3.id)
assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
assert {:ok, path} = Backup.export(backup)
assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory])
assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
assert %{
"@context" => [
"https://www.w3.org/ns/activitystreams",
"http://localhost:4001/schemas/litepub-0.1.jsonld",
%{"@language" => "und"}
],
"bookmarks" => "bookmarks.json",
"followers" => "http://cofe.io/users/cofe/followers",
"following" => "http://cofe.io/users/cofe/following",
"id" => "http://cofe.io/users/cofe",
"inbox" => "http://cofe.io/users/cofe/inbox",
"likes" => "likes.json",
"name" => "Cofe",
"outbox" => "http://cofe.io/users/cofe/outbox",
"preferredUsername" => "cofe",
"publicKey" => %{
"id" => "http://cofe.io/users/cofe#main-key",
"owner" => "http://cofe.io/users/cofe"
},
"type" => "Person",
"url" => "http://cofe.io/users/cofe"
} = Jason.decode!(json)
assert {:ok, {'outbox.json', json}} = :zip.zip_get('outbox.json', zipfile)
assert %{
"@context" => "https://www.w3.org/ns/activitystreams",
"id" => "outbox.json",
"orderedItems" => [
%{
"object" => %{
"actor" => "http://cofe.io/users/cofe",
"content" => "status1",
"type" => "Note"
},
"type" => "Create"
},
%{
"object" => %{
"actor" => "http://cofe.io/users/cofe",
"content" => "status2"
}
},
%{
"actor" => "http://cofe.io/users/cofe",
"object" => %{
"content" => "status3"
}
}
],
"totalItems" => 3,
"type" => "OrderedCollection"
} = Jason.decode!(json)
assert {:ok, {'likes.json', json}} = :zip.zip_get('likes.json', zipfile)
assert %{
"@context" => "https://www.w3.org/ns/activitystreams",
"id" => "likes.json",
"orderedItems" => [^id1, ^id2],
"totalItems" => 2,
"type" => "OrderedCollection"
} = Jason.decode!(json)
assert {:ok, {'bookmarks.json', json}} = :zip.zip_get('bookmarks.json', zipfile)
assert %{
"@context" => "https://www.w3.org/ns/activitystreams",
"id" => "bookmarks.json",
"orderedItems" => [^id2, ^id3],
"totalItems" => 2,
"type" => "OrderedCollection"
} = Jason.decode!(json)
:zip.zip_close(zipfile)
File.rm!(path)
end
describe "it uploads and deletes a backup archive" do
setup do
clear_config(Pleroma.Uploaders.S3,
bucket: "test_bucket",
public_endpoint: "https://s3.amazonaws.com"
)
clear_config([Pleroma.Upload, :uploader])
user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
{:ok, status1} = CommonAPI.post(user, %{status: "status1"})
{:ok, status2} = CommonAPI.post(user, %{status: "status2"})
{:ok, status3} = CommonAPI.post(user, %{status: "status3"})
CommonAPI.favorite(user, status1.id)
CommonAPI.favorite(user, status2.id)
Bookmark.create(user.id, status2.id)
Bookmark.create(user.id, status3.id)
assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
assert {:ok, path} = Backup.export(backup)
[path: path, backup: backup]
end
test "S3", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
with_mock ExAws,
request: fn
%{http_method: :put} -> {:ok, :ok}
%{http_method: :delete} -> {:ok, %{status_code: 204}}
end do
assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
assert {:ok, _backup} = Backup.delete(backup)
end
with_mock ExAws, request: fn %{http_method: :delete} -> {:ok, %{status_code: 204}} end do
end
end
test "Local", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
assert {:ok, _backup} = Backup.delete(backup)
end
end
end

67
test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
View File

@ -977,6 +977,73 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
response["status_visibility"]
end
end
describe "/api/pleroma/backups" do
test "it creates a backup", %{conn: conn} do
admin = %{id: admin_id, nickname: admin_nickname} = insert(:user, is_admin: true)
token = insert(:oauth_admin_token, user: admin)
user = %{id: user_id, nickname: user_nickname} = insert(:user)
assert "" ==
conn
|> assign(:user, admin)
|> assign(:token, token)
|> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
|> json_response(200)
assert [backup] = Repo.all(Pleroma.User.Backup)
ObanHelpers.perform_all()
email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id)
assert String.contains?(email.html_body, "Admin @#{admin.nickname} requested a full backup")
assert_email_sent(to: {user.name, user.email}, html_body: email.html_body)
log_message = "@#{admin_nickname} requested account backup for @#{user_nickname}"
assert [
%{
data: %{
"action" => "create_backup",
"actor" => %{
"id" => ^admin_id,
"nickname" => ^admin_nickname
},
"message" => ^log_message,
"subject" => %{
"id" => ^user_id,
"nickname" => ^user_nickname
}
}
}
] = Pleroma.ModerationLog |> Repo.all()
end
test "it doesn't limit admins", %{conn: conn} do
admin = insert(:user, is_admin: true)
token = insert(:oauth_admin_token, user: admin)
user = insert(:user)
assert "" ==
conn
|> assign(:user, admin)
|> assign(:token, token)
|> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
|> json_response(200)
assert [_backup] = Repo.all(Pleroma.User.Backup)
assert "" ==
conn
|> assign(:user, admin)
|> assign(:token, token)
|> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
|> json_response(200)
assert Repo.aggregate(Pleroma.User.Backup, :count) == 2
end
end
end
# Needed for testing

85
test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs Normal file
View File

@ -0,0 +1,85 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do
use Pleroma.Web.ConnCase
alias Pleroma.User.Backup
alias Pleroma.Web.PleromaAPI.BackupView
setup do
clear_config([Pleroma.Upload, :uploader])
clear_config([Backup, :limit_days])
oauth_access(["read:accounts"])
end
test "GET /api/v1/pleroma/backups", %{user: user, conn: conn} do
assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}}} = Backup.create(user)
backup = Backup.get(backup_id)
response =
conn
|> get("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(:ok)
assert [
%{
"content_type" => "application/zip",
"url" => url,
"file_size" => 0,
"processed" => false,
"inserted_at" => _
}
] = response
assert url == BackupView.download_url(backup)
Pleroma.Tests.ObanHelpers.perform_all()
assert [
%{
"url" => ^url,
"processed" => true
}
] =
conn
|> get("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(:ok)
end
test "POST /api/v1/pleroma/backups", %{user: _user, conn: conn} do
assert [
%{
"content_type" => "application/zip",
"url" => url,
"file_size" => 0,
"processed" => false,
"inserted_at" => _
}
] =
conn
|> post("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(:ok)
Pleroma.Tests.ObanHelpers.perform_all()
assert [
%{
"url" => ^url,
"processed" => true
}
] =
conn
|> get("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(:ok)
days = Pleroma.Config.get([Backup, :limit_days])
assert %{"error" => "Last export was less than #{days} days ago"} ==
conn
|> post("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(400)
end
end

3
test/support/oban_helpers.ex
View File

@ -7,6 +7,8 @@ defmodule Pleroma.Tests.ObanHelpers do
Oban test helpers.
"""
require Ecto.Query
alias Pleroma.Repo
def wipe_all do
@ -15,6 +17,7 @@ defmodule Pleroma.Tests.ObanHelpers do
def perform_all do
Oban.Job
|> Ecto.Query.where(state: "available")
|> Repo.all()
|> perform()
end