Anon/pleroma
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

division emoji.ex on loader.ex and emoji.ex

Browse Source
This commit is contained in:
Maksim Pechnikov 2019-08-28 21:32:44 +03:00
parent a7202b52e0
commit cef2e980b1
5 changed files with 304 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/mix/tasks/pleroma/emoji.ex
View File

@ -235,7 +235,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
cwd: tmp_pack_dir
)
emoji_map = Pleroma.Emoji.make_shortcode_to_file_map(tmp_pack_dir, exts)
emoji_map = Pleroma.Emoji.Loader.make_shortcode_to_file_map(tmp_pack_dir, exts)
File.write!(files_name, Jason.encode!(emoji_map, pretty: true))

212
lib/pleroma/emoji.ex
View File

@ -4,24 +4,22 @@
defmodule Pleroma.Emoji do
@moduledoc """
The emojis are loaded from:
* emoji packs in INSTANCE-DIR/emoji
* the files: `config/emoji.txt` and `config/custom_emoji.txt`
* glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder
This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime.
This GenServer stores in an ETS table the list of the loaded emojis,
and also allows to reload the list at runtime.
"""
use GenServer
alias Pleroma.Emoji.Loader
require Logger
@type pattern :: Regex.t() | module() | String.t()
@type patterns :: pattern() | [pattern()]
@type group_patterns :: keyword(patterns())
@ets __MODULE__.Ets
@ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}]
@ets_options [
:ordered_set,
:protected,
:named_table,
{:read_concurrency, true}
]
@doc false
def start_link(_) do
@ -44,7 +42,7 @@ defmodule Pleroma.Emoji do
end
@doc "Returns all the emojos!!"
@spec get_all() :: [{String.t(), String.t()}, ...]
@spec get_all() :: list({String.t(), String.t(), String.t()})
def get_all do
:ets.tab2list(@ets)
end
@ -58,13 +56,13 @@ defmodule Pleroma.Emoji do
@doc false
def handle_cast(:reload, state) do
load()
update_emojis(Loader.load())
{:noreply, state}
end
@doc false
def handle_call(:reload, _from, state) do
load()
update_emojis(Loader.load())
{:reply, :ok, state}
end
@ -75,189 +73,11 @@ defmodule Pleroma.Emoji do
@doc false
def code_change(_old_vsn, state, _extra) do
load()
update_emojis(Loader.load())
{:ok, state}
end
defp load do
emoji_dir_path =
Path.join(
Pleroma.Config.get!([:instance, :static_dir]),
"emoji"
)
emoji_groups = Pleroma.Config.get([:emoji, :groups])
case File.ls(emoji_dir_path) do
{:error, :enoent} ->
# The custom emoji directory doesn't exist,
# don't do anything
nil
{:error, e} ->
# There was some other error
Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}")
{:ok, results} ->
grouped =
Enum.group_by(results, fn file -> File.dir?(Path.join(emoji_dir_path, file)) end)
packs = grouped[true] || []
files = grouped[false] || []
# Print the packs we've found
Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}")
if not Enum.empty?(files) do
Logger.warn(
"Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{
Enum.join(files, ", ")
}"
)
end
emojis =
Enum.flat_map(
packs,
fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end
)
true = :ets.insert(@ets, emojis)
end
# Compat thing for old custom emoji handling & default emoji,
# it should run even if there are no emoji packs
shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], [])
emojis =
(load_from_file("config/emoji.txt", emoji_groups) ++
load_from_file("config/custom_emoji.txt", emoji_groups) ++
load_from_globs(shortcode_globs, emoji_groups))
|> Enum.reject(fn value -> value == nil end)
true = :ets.insert(@ets, emojis)
:ok
end
defp load_pack(pack_dir, emoji_groups) do
pack_name = Path.basename(pack_dir)
emoji_txt = Path.join(pack_dir, "emoji.txt")
if File.exists?(emoji_txt) do
load_from_file(emoji_txt, emoji_groups)
else
extensions = Pleroma.Config.get([:emoji, :pack_extensions])
Logger.info(
"No emoji.txt found for pack \"#{pack_name}\", assuming all #{Enum.join(extensions, ", ")} files are emoji"
)
make_shortcode_to_file_map(pack_dir, extensions)
|> Enum.map(fn {shortcode, rel_file} ->
filename = Path.join("/emoji/#{pack_name}", rel_file)
{shortcode, filename, [to_string(match_extra(emoji_groups, filename))]}
end)
end
end
def make_shortcode_to_file_map(pack_dir, exts) do
find_all_emoji(pack_dir, exts)
|> Enum.map(&Path.relative_to(&1, pack_dir))
|> Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end)
|> Enum.into(%{})
end
def find_all_emoji(dir, exts) do
Enum.reduce(
File.ls!(dir),
[],
fn f, acc ->
filepath = Path.join(dir, f)
if File.dir?(filepath) do
acc ++ find_all_emoji(filepath, exts)
else
acc ++ [filepath]
end
end
)
|> Enum.filter(fn f -> Path.extname(f) in exts end)
end
defp load_from_file(file, emoji_groups) do
if File.exists?(file) do
load_from_file_stream(File.stream!(file), emoji_groups)
else
[]
end
end
defp load_from_file_stream(stream, emoji_groups) do
stream
|> Stream.map(&String.trim/1)
|> Stream.map(fn line ->
case String.split(line, ~r/,\s*/) do
[name, file] ->
{name, file, [to_string(match_extra(emoji_groups, file))]}
[name, file | tags] ->
{name, file, tags}
_ ->
nil
end
end)
|> Enum.to_list()
end
defp load_from_globs(globs, emoji_groups) do
static_path = Path.join(:code.priv_dir(:pleroma), "static")
paths =
Enum.map(globs, fn glob ->
Path.join(static_path, glob)
|> Path.wildcard()
end)
|> Enum.concat()
Enum.map(paths, fn path ->
tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path)))
shortcode = Path.basename(path, Path.extname(path))
external_path = Path.join("/", Path.relative_to(path, static_path))
{shortcode, external_path, [to_string(tag)]}
end)
end
@doc """
Finds a matching group for the given emoji filename
"""
@spec match_extra(group_patterns(), String.t()) :: atom() | nil
def match_extra(group_patterns, filename) do
match_group_patterns(group_patterns, fn pattern ->
case pattern do
%Regex{} = regex -> Regex.match?(regex, filename)
string when is_binary(string) -> filename == string
end
end)
end
defp match_group_patterns(group_patterns, matcher) do
Enum.find_value(group_patterns, fn {group, patterns} ->
patterns =
patterns
|> List.wrap()
|> Enum.map(fn pattern ->
if String.contains?(pattern, "*") do
~r(#{String.replace(pattern, "*", ".*")})
else
pattern
end
end)
Enum.any?(patterns, matcher) && group
end)
defp update_emojis(emojis) do
:ets.insert(@ets, emojis)
end
end

204
lib/pleroma/emoji/loader.ex Normal file
View File

@ -0,0 +1,204 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Emoji.Loader do
@moduledoc """
The Loader emoji from:
* emoji packs in INSTANCE-DIR/emoji
* the files: `config/emoji.txt` and `config/custom_emoji.txt`
* glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder
"""
alias Pleroma.Config
require Logger
@type pattern :: Regex.t() | module() | String.t()
@type patterns :: pattern() | [pattern()]
@type group_patterns :: keyword(patterns())
@type emoji :: {String.t(), String.t(), list(String.t())}
@doc """
Loads emojis from files/packs.
returns list emojis in format:
`{"000", "/emoji/freespeechextremist.com/000.png", ["Custom"]}`
"""
@spec load() :: list(emoji)
def load do
emoji_dir_path = Path.join(Config.get!([:instance, :static_dir]), "emoji")
emoji_groups = Config.get([:emoji, :groups])
emojis =
case File.ls(emoji_dir_path) do
{:error, :enoent} ->
# The custom emoji directory doesn't exist,
# don't do anything
[]
{:error, e} ->
# There was some other error
Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}")
[]
{:ok, results} ->
grouped =
Enum.group_by(results, fn file ->
File.dir?(Path.join(emoji_dir_path, file))
end)
packs = grouped[true] || []
files = grouped[false] || []
# Print the packs we've found
Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}")
if not Enum.empty?(files) do
Logger.warn(
"Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{
Enum.join(files, ", ")
}"
)
end
Enum.flat_map(packs, fn pack ->
load_pack(Path.join(emoji_dir_path, pack), emoji_groups)
end)
end
# Compat thing for old custom emoji handling & default emoji,
# it should run even if there are no emoji packs
shortcode_globs = Config.get([:emoji, :shortcode_globs], [])
emojis_txt =
(load_from_file("config/emoji.txt", emoji_groups) ++
load_from_file("config/custom_emoji.txt", emoji_groups) ++
load_from_globs(shortcode_globs, emoji_groups))
|> Enum.reject(fn value -> value == nil end)
emojis ++ emojis_txt
end
defp load_pack(pack_dir, emoji_groups) do
pack_name = Path.basename(pack_dir)
emoji_txt = Path.join(pack_dir, "emoji.txt")
if File.exists?(emoji_txt) do
load_from_file(emoji_txt, emoji_groups)
else
extensions = Config.get([:emoji, :pack_extensions])
Logger.info(
"No emoji.txt found for pack \"#{pack_name}\", assuming all #{Enum.join(extensions, ", ")} files are emoji"
)
make_shortcode_to_file_map(pack_dir, extensions)
|> Enum.map(fn {shortcode, rel_file} ->
filename = Path.join("/emoji/#{pack_name}", rel_file)
{shortcode, filename, [to_string(match_extra(emoji_groups, filename))]}
end)
end
end
def make_shortcode_to_file_map(pack_dir, exts) do
find_all_emoji(pack_dir, exts)
|> Enum.map(&Path.relative_to(&1, pack_dir))
|> Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end)
|> Enum.into(%{})
end
def find_all_emoji(dir, exts) do
Enum.reduce(
File.ls!(dir),
[],
fn f, acc ->
filepath = Path.join(dir, f)
if File.dir?(filepath) do
acc ++ find_all_emoji(filepath, exts)
else
acc ++ [filepath]
end
end
)
|> Enum.filter(fn f -> Path.extname(f) in exts end)
end
defp load_from_file(file, emoji_groups) do
if File.exists?(file) do
load_from_file_stream(File.stream!(file), emoji_groups)
else
[]
end
end
defp load_from_file_stream(stream, emoji_groups) do
stream
|> Stream.map(&String.trim/1)
|> Stream.map(fn line ->
case String.split(line, ~r/,\s*/) do
[name, file] ->
{name, file, [to_string(match_extra(emoji_groups, file))]}
[name, file | tags] ->
{name, file, tags}
_ ->
nil
end
end)
|> Enum.to_list()
end
defp load_from_globs(globs, emoji_groups) do
static_path = Path.join(:code.priv_dir(:pleroma), "static")
paths =
Enum.map(globs, fn glob ->
Path.join(static_path, glob)
|> Path.wildcard()
end)
|> Enum.concat()
Enum.map(paths, fn path ->
tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path)))
shortcode = Path.basename(path, Path.extname(path))
external_path = Path.join("/", Path.relative_to(path, static_path))
{shortcode, external_path, [to_string(tag)]}
end)
end
@doc """
Finds a matching group for the given emoji filename
"""
@spec match_extra(group_patterns(), String.t()) :: atom() | nil
def match_extra(group_patterns, filename) do
match_group_patterns(group_patterns, fn pattern ->
case pattern do
%Regex{} = regex -> Regex.match?(regex, filename)
string when is_binary(string) -> filename == string
end
end)
end
defp match_group_patterns(group_patterns, matcher) do
Enum.find_value(group_patterns, fn {group, patterns} ->
patterns =
patterns
|> List.wrap()
|> Enum.map(fn pattern ->
if String.contains?(pattern, "*") do
~r(#{String.replace(pattern, "*", ".*")})
else
pattern
end
end)
Enum.any?(patterns, matcher) && group
end)
end
end

83
test/emoji/loader_test.exs Normal file
View File

@ -0,0 +1,83 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Emoji.LoaderTest do
use ExUnit.Case, async: true
alias Pleroma.Emoji.Loader
describe "match_extra/2" do
setup do
groups = [
"list of files": ["/emoji/custom/first_file.png", "/emoji/custom/second_file.png"],
"wildcard folder": "/emoji/custom/*/file.png",
"wildcard files": "/emoji/custom/folder/*.png",
"special file": "/emoji/custom/special.png"
]
{:ok, groups: groups}
end
test "config for list of files", %{groups: groups} do
group =
groups
|> Loader.match_extra("/emoji/custom/first_file.png")
|> to_string()
assert group == "list of files"
end
test "config with wildcard folder", %{groups: groups} do
group =
groups
|> Loader.match_extra("/emoji/custom/some_folder/file.png")
|> to_string()
assert group == "wildcard folder"
end
test "config with wildcard folder and subfolders", %{groups: groups} do
group =
groups
|> Loader.match_extra("/emoji/custom/some_folder/another_folder/file.png")
|> to_string()
assert group == "wildcard folder"
end
test "config with wildcard files", %{groups: groups} do
group =
groups
|> Loader.match_extra("/emoji/custom/folder/some_file.png")
|> to_string()
assert group == "wildcard files"
end
test "config with wildcard files and subfolders", %{groups: groups} do
group =
groups
|> Loader.match_extra("/emoji/custom/folder/another_folder/some_file.png")
|> to_string()
assert group == "wildcard files"
end
test "config for special file", %{groups: groups} do
group =
groups
|> Loader.match_extra("/emoji/custom/special.png")
|> to_string()
assert group == "special file"
end
test "no mathing returns nil", %{groups: groups} do
group =
groups
|> Loader.match_extra("/emoji/some_undefined.png")
refute group
end
end
end

75
test/emoji_test.exs
View File

@ -32,79 +32,4 @@ defmodule Pleroma.EmojiTest do
assert is_list(tags)
end
end
describe "match_extra/2" do
setup do
groups = [
"list of files": ["/emoji/custom/first_file.png", "/emoji/custom/second_file.png"],
"wildcard folder": "/emoji/custom/*/file.png",
"wildcard files": "/emoji/custom/folder/*.png",
"special file": "/emoji/custom/special.png"
]
{:ok, groups: groups}
end
test "config for list of files", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/first_file.png")
|> to_string()
assert group == "list of files"
end
test "config with wildcard folder", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/some_folder/file.png")
|> to_string()
assert group == "wildcard folder"
end
test "config with wildcard folder and subfolders", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/some_folder/another_folder/file.png")
|> to_string()
assert group == "wildcard folder"
end
test "config with wildcard files", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/folder/some_file.png")
|> to_string()
assert group == "wildcard files"
end
test "config with wildcard files and subfolders", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/folder/another_folder/some_file.png")
|> to_string()
assert group == "wildcard files"
end
test "config for special file", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/special.png")
|> to_string()
assert group == "special file"
end
test "no mathing returns nil", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/some_undefined.png")
refute group
end
end
end