From dcd30e3cebc61c5b84cee123f450536003869dd0 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 9 Aug 2019 16:49:09 +0300 Subject: Mastodon API: Set follower/following counters to 0 when hiding followers/following is enabled We are already doing that in AP representation, so I think we should do it here as well for consistency. --- CHANGELOG.md | 4 ++++ lib/pleroma/web/mastodon_api/views/account_view.ex | 11 +++++++-- test/web/mastodon_api/account_view_test.exs | 27 ++++++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1008e6171..139af1fd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.0.5] - 2019-08-?? +### Fixed +- Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set + ## [1.0.4] - 2019-08-01 ### Fixed - Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 62c516f8e..edd37de2f 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -71,6 +71,13 @@ defp do_render("account.json", %{user: user} = opts) do image = User.avatar_url(user) |> MediaProxy.url() header = User.banner_url(user) |> MediaProxy.url() user_info = User.get_cached_user_info(user) + + following_count = + ((!user.info.hide_follows or opts[:for] == user) && user_info.following_count) || 0 + + followers_count = + ((!user.info.hide_followers or opts[:for] == user) && user_info.follower_count) || 0 + bot = (user.info.source_data["type"] || "Person") in ["Application", "Service"] emojis = @@ -101,8 +108,8 @@ defp do_render("account.json", %{user: user} = opts) do display_name: display_name, locked: user_info.locked, created_at: Utils.to_masto_date(user.inserted_at), - followers_count: user_info.follower_count, - following_count: user_info.following_count, + followers_count: followers_count, + following_count: following_count, statuses_count: user_info.note_count, note: bio || "", url: user.ap_id, diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index de6aeec72..f8a354272 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -275,4 +275,31 @@ test "sanitizes display names" do result = AccountView.render("account.json", %{user: user}) refute result.display_name == " username " end + + describe "hiding follows/following" do + test "shows when follows/following are hidden and sets follower/following count to 0" do + user = insert(:user, info: %{hide_followers: true, hide_follows: true}) + other_user = insert(:user) + {:ok, user, other_user, _activity} = CommonAPI.follow(user, other_user) + {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user) + + assert %{ + followers_count: 0, + following_count: 0, + pleroma: %{hide_follows: true, hide_followers: true} + } = AccountView.render("account.json", %{user: user}) + end + + test "shows actual follower/following count to the account owner" do + user = insert(:user, info: %{hide_followers: true, hide_follows: true}) + other_user = insert(:user) + {:ok, user, other_user, _activity} = CommonAPI.follow(user, other_user) + {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user) + + assert %{ + followers_count: 1, + following_count: 1 + } = AccountView.render("account.json", %{user: user, for: user}) + end + end end -- cgit v1.2.3 From ba21d515d6ace80109b5f38934b33def8243e797 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 10 Aug 2019 16:27:46 +0300 Subject: Mastodon API: Fix thread mute detection It was calling CommonAPI.thread_muted? with post author's account instead of viewer's one. --- CHANGELOG.md | 1 + lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- test/web/mastodon_api/mastodon_api_controller_test.exs | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 139af1fd6..8de2f2124 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [1.0.5] - 2019-08-?? ### Fixed - Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set +- Mastodon API: `muted` in the Status entity, using author's account to determine if the thread was muted ## [1.0.4] - 2019-08-01 ### Fixed diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index c183c9f4a..286fd08c7 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -160,7 +160,7 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity thread_muted? = case activity.thread_muted? do thread_muted? when is_boolean(thread_muted?) -> thread_muted? - nil -> CommonAPI.thread_muted?(user, activity) + nil -> (opts[:for] && CommonAPI.thread_muted?(opts[:for], activity)) || false end attachment_data = object.data["attachment"] || [] diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 152ba8137..7efdc7adb 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2684,8 +2684,10 @@ test "bookmarks" do describe "conversation muting" do setup do + post_user = insert(:user) user = insert(:user) - {:ok, activity} = CommonAPI.post(user, %{"status" => "HIE"}) + + {:ok, activity} = CommonAPI.post(post_user, %{"status" => "HIE"}) [user: user, activity: activity] end -- cgit v1.2.3 From 6525fbac95f83a6050fcea44ad819387c42aa368 Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Wed, 7 Aug 2019 20:29:30 +0000 Subject: Return profile URL when available instead of actor URI for MastodonAPI mention URL Fixes #1165 --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index edd37de2f..1cd8b62c8 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -28,7 +28,7 @@ def render("mention.json", %{user: user}) do id: to_string(user.id), acct: user.nickname, username: username_from_nickname(user.nickname), - url: user.ap_id + url: User.profile_url(user) || user.ap_id } end -- cgit v1.2.3 From b29dcc8c1b74a84e49d290cbea9b1c2f93f1cc5f Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Wed, 7 Aug 2019 20:55:37 +0000 Subject: Simplify logic to mention.js `url` field `User.profile_url` already fallbacks to ap_id --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 1cd8b62c8..b6cbb454b 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -28,7 +28,7 @@ def render("mention.json", %{user: user}) do id: to_string(user.id), acct: user.nickname, username: username_from_nickname(user.nickname), - url: User.profile_url(user) || user.ap_id + url: User.profile_url(user) } end -- cgit v1.2.3 From a3654d947967749211ff7d9d68c1365cfa439c52 Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Wed, 7 Aug 2019 21:40:53 +0000 Subject: Return profile URL in MastodonAPI's `url` field --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index b6cbb454b..69895fd71 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -112,7 +112,7 @@ defp do_render("account.json", %{user: user} = opts) do following_count: following_count, statuses_count: user_info.note_count, note: bio || "", - url: user.ap_id, + url: User.profile_url(user), avatar: image, avatar_static: image, header: header, -- cgit v1.2.3 From 951d8c28d105e8bff6d46af6088a250acf613ab0 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 10 Aug 2019 20:30:55 +0000 Subject: update changelog for thibg's change --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8de2f2124..35582f5fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set - Mastodon API: `muted` in the Status entity, using author's account to determine if the thread was muted +- Mastodon API: return the actual profile URL in the Account entity's `url` property when appropriate ## [1.0.4] - 2019-08-01 ### Fixed -- cgit v1.2.3 From 65bd927cf2c98a1e942f817b730f696d5f200cf8 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 3 Aug 2019 22:56:20 +0200 Subject: templates/layout/app.html.eex: Style anchors [ci skip] --- lib/pleroma/web/templates/layout/app.html.eex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index b3cf9ed11..5836ec1e0 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -36,6 +36,11 @@ margin-bottom: 20px; } + a { + color: color: #d8a070; + text-decoration: none; + } + form { width: 100%; } -- cgit v1.2.3 From d2c3c64da5b4d85f63e96e3b0fc752dc50259ddd Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 10 Aug 2019 20:31:49 +0000 Subject: update changelog for lanodan's change --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35582f5fc..624a348cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set - Mastodon API: `muted` in the Status entity, using author's account to determine if the thread was muted - Mastodon API: return the actual profile URL in the Account entity's `url` property when appropriate +- Templates: properly style anchor tags ## [1.0.4] - 2019-08-01 ### Fixed -- cgit v1.2.3 From f270e332082210c59e5a9024d929a8877bc8e9d1 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 Jul 2019 06:19:27 +0200 Subject: tasks/pleroma/instance.ex: Change :upload_dir to :uploads_dir Closes: https://git.pleroma.social/pleroma/pleroma/issues/1058 --- lib/mix/tasks/pleroma/instance.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index a27c4b897..2ae16adc0 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -149,7 +149,7 @@ def run(["gen" | rest]) do uploads_dir = get_option( options, - :upload_dir, + :uploads_dir, "What directory should media uploads go in (when using the local uploader)?", Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads]) ) -- cgit v1.2.3 From 54d4ceec5ce82ecea8bfd2fade7e618feca8a211 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 2 Aug 2019 23:30:47 +0200 Subject: tasks/pleroma/user.ex: Fix documentation of --max-use and --expire-at Closes: https://git.pleroma.social/pleroma/pleroma/issues/1155 [ci skip] --- lib/mix/tasks/pleroma/user.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 8a78b4fe6..6c7e34a1c 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -31,8 +31,8 @@ defmodule Mix.Tasks.Pleroma.User do mix pleroma.user invite [OPTION...] Options: - - `--expires_at DATE` - last day on which token is active (e.g. "2019-04-05") - - `--max_use NUMBER` - maximum numbers of token uses + - `--expires-at DATE` - last day on which token is active (e.g. "2019-04-05") + - `--max-use NUMBER` - maximum numbers of token uses ## List generated invites -- cgit v1.2.3 From 1310cdc24f31b8fb6858b0c70514a4e316638dbe Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Sat, 3 Aug 2019 18:12:38 +0000 Subject: Handle MRF rejections of incoming AP activities --- lib/pleroma/web/activity_pub/activity_pub.ex | 3 +++ test/web/federator_test.exs | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index fd5521c98..fcecbe21d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -274,6 +274,9 @@ def create(%{to: to, actor: actor, context: context, object: object} = params, f else {:fake, true, activity} -> {:ok, activity} + + {:error, message} -> + {:error, message} end end diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 0f43bc8f2..e42d376cb 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -213,5 +213,21 @@ test "rejects incoming AP docs with incorrect origin" do :error = Federator.incoming_ap_doc(params) end + + test "it does not crash if MRF rejects the post" do + policies = Pleroma.Config.get([:instance, :rewrite_policy]) + mrf_keyword_policy = Pleroma.Config.get(:mrf_keyword) + Pleroma.Config.put([:mrf_keyword, :reject], ["lain"]) + Pleroma.Config.put([:instance, :rewrite_policy], Pleroma.Web.ActivityPub.MRF.KeywordPolicy) + + params = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + + assert Federator.incoming_ap_doc(params) == :error + + Pleroma.Config.put([:instance, :rewrite_policy], policies) + Pleroma.Config.put(:mrf_keyword, mrf_keyword_policy) + end end end -- cgit v1.2.3 From 8a8fe57670db773d9f70a0248653661f93771457 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 3 Aug 2019 23:17:17 +0000 Subject: tasks: relay: add list task --- CHANGELOG.md | 3 +++ lib/mix/tasks/pleroma/relay.ex | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 624a348cd..655c01972 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: return the actual profile URL in the Account entity's `url` property when appropriate - Templates: properly style anchor tags +### Added +- Relays: Added a task to list relay subscriptions. + ## [1.0.4] - 2019-08-01 ### Fixed - Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex index 83ed0ed02..c7324fff6 100644 --- a/lib/mix/tasks/pleroma/relay.ex +++ b/lib/mix/tasks/pleroma/relay.ex @@ -5,6 +5,7 @@ defmodule Mix.Tasks.Pleroma.Relay do use Mix.Task import Mix.Pleroma + alias Pleroma.User alias Pleroma.Web.ActivityPub.Relay @shortdoc "Manages remote relays" @@ -22,6 +23,10 @@ defmodule Mix.Tasks.Pleroma.Relay do ``mix pleroma.relay unfollow `` Example: ``mix pleroma.relay unfollow https://example.org/relay`` + + ## List relay subscriptions + + ``mix pleroma.relay list`` """ def run(["follow", target]) do start_pleroma() @@ -44,4 +49,19 @@ def run(["unfollow", target]) do {:error, e} -> shell_error("Error while following #{target}: #{inspect(e)}") end end + + def run(["list"]) do + start_pleroma() + + with %User{} = user <- Relay.get_actor() do + user.following + |> Enum.each(fn entry -> + URI.parse(entry) + |> Map.get(:host) + |> shell_info() + end) + else + e -> shell_error("Error while fetching relay subscription list: #{inspect(e)}") + end + end end -- cgit v1.2.3 From e08b97853f20c6d0571cf451d18defd0bdb2393f Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Wed, 10 Jul 2019 01:42:41 +0545 Subject: add listener port and ip option for 'pleroma.instance gen' and enable its test --- lib/mix/tasks/pleroma/instance.ex | 26 ++++++++++++-- priv/templates/sample_config.eex | 1 + test/tasks/instance.exs | 65 --------------------------------- test/tasks/instance_test.exs | 76 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 67 deletions(-) delete mode 100644 test/tasks/instance.exs create mode 100644 test/tasks/instance_test.exs diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 2ae16adc0..9080adb52 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -34,6 +34,8 @@ defmodule Mix.Tasks.Pleroma.Instance do - `--db-configurable Y/N` - Allow/disallow configuring instance from admin part - `--uploads-dir` - the directory uploads go in when using a local uploader - `--static-dir` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.) + - `--listen-ip` - the ip the app should listen to, defaults to 127.0.0.1 + - `--listen-port` - the port the app should listen to, defaults to 4000 """ def run(["gen" | rest]) do @@ -56,7 +58,9 @@ def run(["gen" | rest]) do indexable: :string, db_configurable: :string, uploads_dir: :string, - static_dir: :string + static_dir: :string, + listen_ip: :string, + listen_port: :string ], aliases: [ o: :output, @@ -146,6 +150,22 @@ def run(["gen" | rest]) do "n" ) === "y" + listen_port = + get_option( + options, + :listen_port, + "What port will the app listen to (leave it if you are using the default setup with nginx)?", + 4000 + ) + + listen_ip = + get_option( + options, + :listen_ip, + "What ip will the app listen to (leave it if you are using the default setup with nginx)?", + "127.0.0.1" + ) + uploads_dir = get_option( options, @@ -186,7 +206,9 @@ def run(["gen" | rest]) do db_configurable?: db_configurable?, static_dir: static_dir, uploads_dir: uploads_dir, - rum_enabled: rum_enabled + rum_enabled: rum_enabled, + listen_ip: listen_ip, + listen_port: listen_port ) result_psql = diff --git a/priv/templates/sample_config.eex b/priv/templates/sample_config.eex index 2d4a49328..054aa6c22 100644 --- a/priv/templates/sample_config.eex +++ b/priv/templates/sample_config.eex @@ -11,6 +11,7 @@ end %> config :pleroma, Pleroma.Web.Endpoint, url: [host: "<%= domain %>", scheme: "https", port: <%= port %>], + http: [ip: {<%= String.replace(listen_ip, ".", ", ") %>}, port: <%= listen_port %>], secret_key_base: "<%= secret %>", signing_salt: "<%= signing_salt %>" diff --git a/test/tasks/instance.exs b/test/tasks/instance.exs deleted file mode 100644 index 1875f52a3..000000000 --- a/test/tasks/instance.exs +++ /dev/null @@ -1,65 +0,0 @@ -defmodule Pleroma.InstanceTest do - use ExUnit.Case, async: true - - setup do - File.mkdir_p!(tmp_path()) - on_exit(fn -> File.rm_rf(tmp_path()) end) - :ok - end - - defp tmp_path do - "/tmp/generated_files/" - end - - test "running gen" do - mix_task = fn -> - Mix.Tasks.Pleroma.Instance.run([ - "gen", - "--output", - tmp_path() <> "generated_config.exs", - "--output-psql", - tmp_path() <> "setup.psql", - "--domain", - "test.pleroma.social", - "--instance-name", - "Pleroma", - "--admin-email", - "admin@example.com", - "--notify-email", - "notify@example.com", - "--dbhost", - "dbhost", - "--dbname", - "dbname", - "--dbuser", - "dbuser", - "--dbpass", - "dbpass", - "--indexable", - "y", - "--db-configurable", - "y" - ]) - end - - ExUnit.CaptureIO.capture_io(fn -> - mix_task.() - end) - - generated_config = File.read!(tmp_path() <> "generated_config.exs") - assert generated_config =~ "host: \"test.pleroma.social\"" - assert generated_config =~ "name: \"Pleroma\"" - assert generated_config =~ "email: \"admin@example.com\"" - assert generated_config =~ "notify_email: \"notify@example.com\"" - assert generated_config =~ "hostname: \"dbhost\"" - assert generated_config =~ "database: \"dbname\"" - assert generated_config =~ "username: \"dbuser\"" - assert generated_config =~ "password: \"dbpass\"" - assert generated_config =~ "dynamic_configuration: true" - assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql() - end - - defp generated_setup_psql do - ~s(CREATE USER dbuser WITH ENCRYPTED PASSWORD 'dbpass';\nCREATE DATABASE dbname OWNER dbuser;\n\\c dbname;\n--Extensions made by ecto.migrate that need superuser access\nCREATE EXTENSION IF NOT EXISTS citext;\nCREATE EXTENSION IF NOT EXISTS pg_trgm;\nCREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";\n) - end -end diff --git a/test/tasks/instance_test.exs b/test/tasks/instance_test.exs new file mode 100644 index 000000000..229ecc9c1 --- /dev/null +++ b/test/tasks/instance_test.exs @@ -0,0 +1,76 @@ +defmodule Pleroma.InstanceTest do + use ExUnit.Case, async: true + + setup do + File.mkdir_p!(tmp_path()) + on_exit(fn -> File.rm_rf(tmp_path()) end) + :ok + end + + defp tmp_path do + "/tmp/generated_files/" + end + + test "running gen" do + mix_task = fn -> + Mix.Tasks.Pleroma.Instance.run([ + "gen", + "--output", + tmp_path() <> "generated_config.exs", + "--output-psql", + tmp_path() <> "setup.psql", + "--domain", + "test.pleroma.social", + "--instance-name", + "Pleroma", + "--admin-email", + "admin@example.com", + "--notify-email", + "notify@example.com", + "--dbhost", + "dbhost", + "--dbname", + "dbname", + "--dbuser", + "dbuser", + "--dbpass", + "dbpass", + "--indexable", + "y", + "--db-configurable", + "y", + "--rum", + "y", + "--listen-port", + "4000", + "--listen-ip", + "127.0.0.1", + "--uploads-dir", + "test/uploads", + "--static-dir", + "instance/static/" + ]) + end + + ExUnit.CaptureIO.capture_io(fn -> + mix_task.() + end) + + generated_config = File.read!(tmp_path() <> "generated_config.exs") + assert generated_config =~ "host: \"test.pleroma.social\"" + assert generated_config =~ "name: \"Pleroma\"" + assert generated_config =~ "email: \"admin@example.com\"" + assert generated_config =~ "notify_email: \"notify@example.com\"" + assert generated_config =~ "hostname: \"dbhost\"" + assert generated_config =~ "database: \"dbname\"" + assert generated_config =~ "username: \"dbuser\"" + assert generated_config =~ "password: \"dbpass\"" + assert generated_config =~ "dynamic_configuration: true" + assert generated_config =~ "http: [ip: {127, 0, 0, 1}, port: 4000]" + assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql() + end + + defp generated_setup_psql do + ~s(CREATE USER dbuser WITH ENCRYPTED PASSWORD 'dbpass';\nCREATE DATABASE dbname OWNER dbuser;\n\\c dbname;\n--Extensions made by ecto.migrate that need superuser access\nCREATE EXTENSION IF NOT EXISTS citext;\nCREATE EXTENSION IF NOT EXISTS pg_trgm;\nCREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";\nCREATE EXTENSION IF NOT EXISTS rum;\n) + end +end -- cgit v1.2.3 From b9def3758ae97490f8c46a77a03c344568106376 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 22 Jul 2019 14:33:58 +0000 Subject: Feature/1087 wildcard option for blocks --- lib/pleroma/user.ex | 12 +-- lib/pleroma/web/activity_pub/mrf.ex | 10 +++ lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 55 ++++++++++---- lib/pleroma/web/activity_pub/publisher.ex | 9 ++- test/user_test.exs | 42 ++++++++++ test/web/activity_pub/mrf/mrf_test.exs | 46 +++++++++++ test/web/activity_pub/mrf/simple_policy_test.exs | 93 +++++++++++++++++++++++ 7 files changed, 245 insertions(+), 22 deletions(-) create mode 100644 test/web/activity_pub/mrf/mrf_test.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f7191762f..ed5403219 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -836,15 +836,15 @@ def unblock(blocker, %{ap_id: ap_id}) do def mutes?(nil, _), do: false def mutes?(user, %{ap_id: ap_id}), do: Enum.member?(user.info.mutes, ap_id) - def blocks?(user, %{ap_id: ap_id}) do - blocks = user.info.blocks - domain_blocks = user.info.domain_blocks + def blocks?(%User{info: info} = _user, %{ap_id: ap_id}) do + blocks = info.blocks + + domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(info.domain_blocks) + %{host: host} = URI.parse(ap_id) Enum.member?(blocks, ap_id) || - Enum.any?(domain_blocks, fn domain -> - host == domain - end) + Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, host) end def subscribed_to?(user, %{ap_id: ap_id}) do diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 10ceef715..dd204b21c 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -25,4 +25,14 @@ def get_policies do defp get_policies(policy) when is_atom(policy), do: [policy] defp get_policies(policies) when is_list(policies), do: policies defp get_policies(_), do: [] + + @spec subdomains_regex([String.t()]) :: [Regex.t()] + def subdomains_regex(domains) when is_list(domains) do + for domain <- domains, do: ~r(^#{String.replace(domain, "*.", "(.*\\.)*")}$) + end + + @spec subdomain_match?([Regex.t()], String.t()) :: boolean() + def subdomain_match?(domains, host) do + Enum.any?(domains, fn domain -> Regex.match?(domain, host) end) + end end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 433d23c5f..2cf63d3db 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -4,22 +4,29 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do alias Pleroma.User + alias Pleroma.Web.ActivityPub.MRF @moduledoc "Filter activities depending on their origin instance" - @behaviour Pleroma.Web.ActivityPub.MRF + @behaviour MRF defp check_accept(%{host: actor_host} = _actor_info, object) do - accepts = Pleroma.Config.get([:mrf_simple, :accept]) + accepts = + Pleroma.Config.get([:mrf_simple, :accept]) + |> MRF.subdomains_regex() cond do accepts == [] -> {:ok, object} actor_host == Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, object} - Enum.member?(accepts, actor_host) -> {:ok, object} + MRF.subdomain_match?(accepts, actor_host) -> {:ok, object} true -> {:reject, nil} end end defp check_reject(%{host: actor_host} = _actor_info, object) do - if Enum.member?(Pleroma.Config.get([:mrf_simple, :reject]), actor_host) do + rejects = + Pleroma.Config.get([:mrf_simple, :reject]) + |> MRF.subdomains_regex() + + if MRF.subdomain_match?(rejects, actor_host) do {:reject, nil} else {:ok, object} @@ -31,8 +38,12 @@ defp check_media_removal( %{"type" => "Create", "object" => %{"attachment" => child_attachment}} = object ) when length(child_attachment) > 0 do + media_removal = + Pleroma.Config.get([:mrf_simple, :media_removal]) + |> MRF.subdomains_regex() + object = - if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_removal]), actor_host) do + if MRF.subdomain_match?(media_removal, actor_host) do child_object = Map.delete(object["object"], "attachment") Map.put(object, "object", child_object) else @@ -51,8 +62,12 @@ defp check_media_nsfw( "object" => child_object } = object ) do + media_nsfw = + Pleroma.Config.get([:mrf_simple, :media_nsfw]) + |> MRF.subdomains_regex() + object = - if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_nsfw]), actor_host) do + if MRF.subdomain_match?(media_nsfw, actor_host) do tags = (child_object["tag"] || []) ++ ["nsfw"] child_object = Map.put(child_object, "tag", tags) child_object = Map.put(child_object, "sensitive", true) @@ -67,12 +82,12 @@ defp check_media_nsfw( defp check_media_nsfw(_actor_info, object), do: {:ok, object} defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do + timeline_removal = + Pleroma.Config.get([:mrf_simple, :federated_timeline_removal]) + |> MRF.subdomains_regex() + object = - with true <- - Enum.member?( - Pleroma.Config.get([:mrf_simple, :federated_timeline_removal]), - actor_host - ), + with true <- MRF.subdomain_match?(timeline_removal, actor_host), user <- User.get_cached_by_ap_id(object["actor"]), true <- "https://www.w3.org/ns/activitystreams#Public" in object["to"] do to = @@ -94,7 +109,11 @@ defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do end defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do - if actor_host in Pleroma.Config.get([:mrf_simple, :report_removal]) do + report_removal = + Pleroma.Config.get([:mrf_simple, :report_removal]) + |> MRF.subdomains_regex() + + if MRF.subdomain_match?(report_removal, actor_host) do {:reject, nil} else {:ok, object} @@ -104,7 +123,11 @@ defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} defp check_report_removal(_actor_info, object), do: {:ok, object} defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do - if actor_host in Pleroma.Config.get([:mrf_simple, :avatar_removal]) do + avatar_removal = + Pleroma.Config.get([:mrf_simple, :avatar_removal]) + |> MRF.subdomains_regex() + + if MRF.subdomain_match?(avatar_removal, actor_host) do {:ok, Map.delete(object, "icon")} else {:ok, object} @@ -114,7 +137,11 @@ defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} defp check_avatar_removal(_actor_info, object), do: {:ok, object} defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do - if actor_host in Pleroma.Config.get([:mrf_simple, :banner_removal]) do + banner_removal = + Pleroma.Config.get([:mrf_simple, :banner_removal]) + |> MRF.subdomains_regex() + + if MRF.subdomain_match?(banner_removal, actor_host) do {:ok, Map.delete(object, "image")} else {:ok, object} diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index a05e03263..3d181f57c 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -87,8 +87,13 @@ defp should_federate?(inbox, public) do if public do true else - inbox_info = URI.parse(inbox) - !Enum.member?(Config.get([:instance, :quarantined_instances], []), inbox_info.host) + %{host: host} = URI.parse(inbox) + + quarantined_instances = + Config.get([:instance, :quarantined_instances], []) + |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() + + !Pleroma.Web.ActivityPub.MRF.subdomain_match?(quarantined_instances, host) end end diff --git a/test/user_test.exs b/test/user_test.exs index 198a97fae..7bd745104 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -799,6 +799,48 @@ test "blocks domains" do assert User.blocks?(user, collateral_user) end + test "does not block domain with same end" do + user = insert(:user) + + collateral_user = + insert(:user, %{ap_id: "https://another-awful-and-rude-instance.com/user/bully"}) + + {:ok, user} = User.block_domain(user, "awful-and-rude-instance.com") + + refute User.blocks?(user, collateral_user) + end + + test "does not block domain with same end if wildcard added" do + user = insert(:user) + + collateral_user = + insert(:user, %{ap_id: "https://another-awful-and-rude-instance.com/user/bully"}) + + {:ok, user} = User.block_domain(user, "*.awful-and-rude-instance.com") + + refute User.blocks?(user, collateral_user) + end + + test "blocks domain with wildcard for subdomain" do + user = insert(:user) + + user_from_subdomain = + insert(:user, %{ap_id: "https://subdomain.awful-and-rude-instance.com/user/bully"}) + + user_with_two_subdomains = + insert(:user, %{ + ap_id: "https://subdomain.second_subdomain.awful-and-rude-instance.com/user/bully" + }) + + user_domain = insert(:user, %{ap_id: "https://awful-and-rude-instance.com/user/bully"}) + + {:ok, user} = User.block_domain(user, "*.awful-and-rude-instance.com") + + assert User.blocks?(user, user_from_subdomain) + assert User.blocks?(user, user_with_two_subdomains) + assert User.blocks?(user, user_domain) + end + test "unblocks domains" do user = insert(:user) collateral_user = insert(:user, %{ap_id: "https://awful-and-rude-instance.com/user/bully"}) diff --git a/test/web/activity_pub/mrf/mrf_test.exs b/test/web/activity_pub/mrf/mrf_test.exs new file mode 100644 index 000000000..a9cdf5317 --- /dev/null +++ b/test/web/activity_pub/mrf/mrf_test.exs @@ -0,0 +1,46 @@ +defmodule Pleroma.Web.ActivityPub.MRFTest do + use ExUnit.Case, async: true + alias Pleroma.Web.ActivityPub.MRF + + test "subdomains_regex/1" do + assert MRF.subdomains_regex(["unsafe.tld", "*.unsafe.tld"]) == [ + ~r/^unsafe.tld$/, + ~r/^(.*\.)*unsafe.tld$/ + ] + end + + describe "subdomain_match/2" do + test "common domains" do + regexes = MRF.subdomains_regex(["unsafe.tld", "unsafe2.tld"]) + + assert regexes == [~r/^unsafe.tld$/, ~r/^unsafe2.tld$/] + + assert MRF.subdomain_match?(regexes, "unsafe.tld") + assert MRF.subdomain_match?(regexes, "unsafe2.tld") + + refute MRF.subdomain_match?(regexes, "example.com") + end + + test "wildcard domains with one subdomain" do + regexes = MRF.subdomains_regex(["*.unsafe.tld"]) + + assert regexes == [~r/^(.*\.)*unsafe.tld$/] + + assert MRF.subdomain_match?(regexes, "unsafe.tld") + assert MRF.subdomain_match?(regexes, "sub.unsafe.tld") + refute MRF.subdomain_match?(regexes, "anotherunsafe.tld") + refute MRF.subdomain_match?(regexes, "unsafe.tldanother") + end + + test "wildcard domains with two subdomains" do + regexes = MRF.subdomains_regex(["*.unsafe.tld"]) + + assert regexes == [~r/^(.*\.)*unsafe.tld$/] + + assert MRF.subdomain_match?(regexes, "unsafe.tld") + assert MRF.subdomain_match?(regexes, "sub.sub.unsafe.tld") + refute MRF.subdomain_match?(regexes, "sub.anotherunsafe.tld") + refute MRF.subdomain_match?(regexes, "sub.unsafe.tldanother") + end + end +end diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs index 0fd68e103..8e86d2219 100644 --- a/test/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -49,6 +49,19 @@ test "has a matching host" do assert SimplePolicy.filter(local_message) == {:ok, local_message} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :media_removal], ["*.remote.instance"]) + media_message = build_media_message() + local_message = build_local_message() + + assert SimplePolicy.filter(media_message) == + {:ok, + media_message + |> Map.put("object", Map.delete(media_message["object"], "attachment"))} + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end end describe "when :media_nsfw" do @@ -74,6 +87,20 @@ test "has a matching host" do assert SimplePolicy.filter(local_message) == {:ok, local_message} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :media_nsfw], ["*.remote.instance"]) + media_message = build_media_message() + local_message = build_local_message() + + assert SimplePolicy.filter(media_message) == + {:ok, + media_message + |> put_in(["object", "tag"], ["foo", "nsfw"]) + |> put_in(["object", "sensitive"], true)} + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end end defp build_media_message do @@ -106,6 +133,15 @@ test "has a matching host" do assert SimplePolicy.filter(report_message) == {:reject, nil} assert SimplePolicy.filter(local_message) == {:ok, local_message} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :report_removal], ["*.remote.instance"]) + report_message = build_report_message() + local_message = build_local_message() + + assert SimplePolicy.filter(report_message) == {:reject, nil} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end end defp build_report_message do @@ -146,6 +182,27 @@ test "has a matching host" do assert SimplePolicy.filter(local_message) == {:ok, local_message} end + test "match with wildcard domain" do + {actor, ftl_message} = build_ftl_actor_and_message() + + ftl_message_actor_host = + ftl_message + |> Map.fetch!("actor") + |> URI.parse() + |> Map.fetch!(:host) + + Config.put([:mrf_simple, :federated_timeline_removal], ["*." <> ftl_message_actor_host]) + local_message = build_local_message() + + assert {:ok, ftl_message} = SimplePolicy.filter(ftl_message) + assert actor.follower_address in ftl_message["to"] + refute actor.follower_address in ftl_message["cc"] + refute "https://www.w3.org/ns/activitystreams#Public" in ftl_message["to"] + assert "https://www.w3.org/ns/activitystreams#Public" in ftl_message["cc"] + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + test "has a matching host but only as:Public in to" do {_actor, ftl_message} = build_ftl_actor_and_message() @@ -192,6 +249,14 @@ test "has a matching host" do assert SimplePolicy.filter(remote_message) == {:reject, nil} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :reject], ["*.remote.instance"]) + + remote_message = build_remote_message() + + assert SimplePolicy.filter(remote_message) == {:reject, nil} + end end describe "when :accept" do @@ -224,6 +289,16 @@ test "has a matching host" do assert SimplePolicy.filter(local_message) == {:ok, local_message} assert SimplePolicy.filter(remote_message) == {:ok, remote_message} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :accept], ["*.remote.instance"]) + + local_message = build_local_message() + remote_message = build_remote_message() + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + end end describe "when :avatar_removal" do @@ -251,6 +326,15 @@ test "has a matching host" do refute filtered["icon"] end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :avatar_removal], ["*.remote.instance"]) + + remote_user = build_remote_user() + {:ok, filtered} = SimplePolicy.filter(remote_user) + + refute filtered["icon"] + end end describe "when :banner_removal" do @@ -278,6 +362,15 @@ test "has a matching host" do refute filtered["image"] end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :banner_removal], ["*.remote.instance"]) + + remote_user = build_remote_user() + {:ok, filtered} = SimplePolicy.filter(remote_user) + + refute filtered["image"] + end end defp build_local_message do -- cgit v1.2.3 From a7b947534cc998486b82c721849bfc9aceb5178f Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 10 Aug 2019 20:38:04 +0000 Subject: chase changelog updates --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 655c01972..8c3666cc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - Relays: Added a task to list relay subscriptions. +- MRF (Simple Policy): Support for wildcard domains. +- Support for wildcard domains in user domain blocks setting. +- Configuration: `quarantined_instances` support wildcard domains. ## [1.0.4] - 2019-08-01 ### Fixed -- cgit v1.2.3 From b85840b536f9be391120db5d3e0e7934c8ed9390 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 7 Aug 2019 00:12:42 +0300 Subject: Stop depending on the embedded object in restrict_favorited_by --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index fcecbe21d..f9ab06bf8 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -733,8 +733,8 @@ defp restrict_state(query, _), do: query defp restrict_favorited_by(query, %{"favorited_by" => ap_id}) do from( - activity in query, - where: fragment(~s(? <@ (? #> '{"object","likes"}'\)), ^ap_id, activity.data) + [_activity, object] in query, + where: fragment("(?)->'likes' \\? (?)", object.data, ^ap_id) ) end -- cgit v1.2.3 From 1807d3a115edf12510b2ca735c7aea3d3f57e5f8 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 7 Aug 2019 00:23:58 +0300 Subject: OStatus Announce Representer: Do not depend on the object being embedded in the Create activity --- lib/pleroma/web/ostatus/activity_representer.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 95037125d..2f23e5730 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -182,6 +182,7 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] retweeted_activity = Activity.get_create_by_object_ap_id(activity.data["object"]) + retweeted_object = Object.normalize(retweeted_activity) retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"]) retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true) @@ -196,7 +197,7 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho {:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']}, {:id, h.(activity.data["id"])}, {:title, ['#{user.nickname} repeated a notice']}, - {:content, [type: 'html'], ['RT #{retweeted_activity.data["object"]["content"]}']}, + {:content, [type: 'html'], ['RT #{retweeted_object.data["content"]}']}, {:published, h.(inserted_at)}, {:updated, h.(updated_at)}, {:"ostatus:conversation", [ref: h.(activity.data["context"])], -- cgit v1.2.3 From e5161961bdf41a0c0a6f051940b31c0b4f258473 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 7 Aug 2019 00:36:13 +0300 Subject: ActivityPub tests: remove assertions of embedded object being updated, because the objects are no longer supposed to be embedded --- test/web/activity_pub/activity_pub_test.exs | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 76586ee4a..97c946924 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -679,9 +679,6 @@ test "adds a like activity to the db" do assert like_activity == same_like_activity assert object.data["likes"] == [user.ap_id] - [note_activity] = Activity.get_all_create_by_object_ap_id(object.data["id"]) - assert note_activity.data["object"]["like_count"] == 1 - {:ok, _like_activity, object} = ActivityPub.like(user_two, object) assert object.data["like_count"] == 2 end -- cgit v1.2.3 From f0c32364b7678ce9c234f2df994b9107c922a23d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 7 Aug 2019 00:58:48 +0300 Subject: OStatus tests: stop relying on embedded objects --- test/web/ostatus/ostatus_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 77bccdaa1..41cf188ca 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -196,7 +196,7 @@ test "handle incoming retweets - GS, subscription - local message" do assert retweeted_activity.data["type"] == "Create" assert retweeted_activity.data["actor"] == user.ap_id assert retweeted_activity.local - assert retweeted_activity.data["object"]["announcement_count"] == 1 + assert Object.normalize(retweeted_activity).data["announcement_count"] == 1 end test "handle incoming retweets - Mastodon, salmon" do -- cgit v1.2.3 From 7e412fd88a57643bd3826d84d7bd46beec980188 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 7 Aug 2019 01:02:29 +0300 Subject: Do not rembed the object after updating it --- CHANGELOG.md | 5 +---- lib/pleroma/web/activity_pub/utils.ex | 17 +---------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c3666cc5..23319d4e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: `muted` in the Status entity, using author's account to determine if the thread was muted - Mastodon API: return the actual profile URL in the Account entity's `url` property when appropriate - Templates: properly style anchor tags +- Objects being re-embedded to activities after being updated (e.g faved/reposted). Running 'mix pleroma.database prune_objects' again is advised. ### Added - Relays: Added a task to list relay subscriptions. @@ -32,10 +33,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [1.0.2] - 2019-07-28 ### Fixed - Not being able to pin unlisted posts -- Mastodon API: represent poll IDs as strings -- MediaProxy: fix matching filenames -- MediaProxy: fix filename encoding -- Migrations: fix a sporadic migration failure - Metadata rendering errors resulting in the entire page being inaccessible - Federation/MediaProxy not working with instances that have wrong certificate order - ActivityPub S2S: remote user deletions now work the same as local user deletions. diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 514266cee..b9fcae192 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -251,20 +251,6 @@ def insert_full_object(%{"object" => %{"type" => type} = object_data} = map) def insert_full_object(map), do: {:ok, map, nil} - def update_object_in_activities(%{data: %{"id" => id}} = object) do - # TODO - # Update activities that already had this. Could be done in a seperate process. - # Alternatively, just don't do this and fetch the current object each time. Most - # could probably be taken from cache. - relevant_activities = Activity.get_all_create_by_object_ap_id(id) - - Enum.map(relevant_activities, fn activity -> - new_activity_data = activity.data |> Map.put("object", object.data) - changeset = Changeset.change(activity, data: new_activity_data) - Repo.update(changeset) - end) - end - #### Like-related helpers @doc """ @@ -347,8 +333,7 @@ def update_element_in_object(property, element, object) do |> Map.put("#{property}_count", length(element)) |> Map.put("#{property}s", element), changeset <- Changeset.change(object, data: new_data), - {:ok, object} <- Object.update_and_set_cache(changeset), - _ <- update_object_in_activities(object) do + {:ok, object} <- Object.update_and_set_cache(changeset) do {:ok, object} end end -- cgit v1.2.3 From 48bd4ee933e549daa3637ac4741aa4439af543ef Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Sat, 10 Aug 2019 18:47:40 +0000 Subject: Strip internal fields including likes from incoming and outgoing activities --- CHANGELOG.md | 2 ++ lib/mix/tasks/pleroma/database.ex | 36 ++++++++++++++++++++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 38 ++++---------------------- test/tasks/database_test.exs | 36 ++++++++++++++++++++++++ test/web/activity_pub/transmogrifier_test.exs | 30 ++++++++++++++------ 5 files changed, 101 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23319d4e8..9ea0052c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF (Simple Policy): Support for wildcard domains. - Support for wildcard domains in user domain blocks setting. - Configuration: `quarantined_instances` support wildcard domains. +- Mix Tasks: `mix pleroma.database fix_likes_collections` +- Federation: Remove `likes` from objects. ## [1.0.4] - 2019-08-01 ### Fixed diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index e91fb31d1..990651da4 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -35,6 +35,10 @@ defmodule Mix.Tasks.Pleroma.Database do ## Remove duplicated items from following and update followers count for all users mix pleroma.database update_users_following_followers_counts + + ## Fix the pre-existing "likes" collections for all objects + + mix pleroma.database fix_likes_collections """ def run(["remove_embedded_objects" | args]) do {options, [], []} = @@ -119,4 +123,36 @@ def run(["prune_objects" | args]) do ) end end + + def run(["fix_likes_collections"]) do + import Ecto.Query + + start_pleroma() + + from(object in Object, + where: fragment("(?)->>'likes' is not null", object.data), + select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)} + ) + |> Pleroma.RepoStreamer.chunk_stream(100) + |> Stream.each(fn objects -> + ids = + objects + |> Enum.filter(fn object -> object.likes |> Jason.decode!() |> is_map() end) + |> Enum.map(& &1.id) + + Object + |> where([object], object.id in ^ids) + |> update([object], + set: [ + data: + fragment( + "jsonb_set(?, '{likes}', '[]'::jsonb, true)", + object.data + ) + ] + ) + |> Repo.update_all([], timeout: :infinity) + end) + |> Stream.run() + end end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 591227af2..65c1f9d9f 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -24,6 +24,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do """ def fix_object(object) do object + |> strip_internal_fields |> fix_actor |> fix_url |> fix_attachments @@ -32,7 +33,6 @@ def fix_object(object) do |> fix_emoji |> fix_tag |> fix_content_map - |> fix_likes |> fix_addressing |> fix_summary |> fix_type @@ -150,21 +150,9 @@ def fix_actor(%{"attributedTo" => actor} = object) do |> Map.put("actor", Containment.get_actor(%{"actor" => actor})) end - # Check for standardisation - # This is what Peertube does - # curl -H 'Accept: application/activity+json' $likes | jq .totalItems - # Prismo returns only an integer (count) as "likes" - def fix_likes(%{"likes" => likes} = object) when not is_map(likes) do - object - |> Map.put("likes", []) - |> Map.put("like_count", 0) - end + def fix_in_reply_to(object, options \\ []) - def fix_likes(object) do - object - end - - def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object) + def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, _options) when not is_nil(in_reply_to) do in_reply_to_id = cond do @@ -203,7 +191,7 @@ def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object) end end - def fix_in_reply_to(object), do: object + def fix_in_reply_to(object, _options), do: object def fix_context(object) do context = object["context"] || object["conversation"] || Utils.generate_context_id() @@ -752,7 +740,6 @@ def prepare_object(object) do |> add_mention_tags |> add_emoji_tags |> add_attributed_to - |> add_likes |> prepare_attachments |> set_conversation |> set_reply_to_uri @@ -936,22 +923,6 @@ def add_attributed_to(object) do |> Map.put("attributedTo", attributed_to) end - def add_likes(%{"id" => id, "like_count" => likes} = object) do - likes = %{ - "id" => "#{id}/likes", - "first" => "#{id}/likes?page=1", - "type" => "OrderedCollection", - "totalItems" => likes - } - - object - |> Map.put("likes", likes) - end - - def add_likes(object) do - object - end - def prepare_attachments(object) do attachments = (object["attachment"] || []) @@ -967,6 +938,7 @@ def prepare_attachments(object) do defp strip_internal_fields(object) do object |> Map.drop([ + "likes", "like_count", "announcements", "announcement_count", diff --git a/test/tasks/database_test.exs b/test/tasks/database_test.exs index 579130b05..a8f25f500 100644 --- a/test/tasks/database_test.exs +++ b/test/tasks/database_test.exs @@ -3,8 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.DatabaseTest do + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Web.CommonAPI + use Pleroma.DataCase import Pleroma.Factory @@ -46,4 +49,37 @@ test "following and followers count are updated" do assert user.info.follower_count == 0 end end + + describe "running fix_likes_collections" do + test "it turns OrderedCollection likes into empty arrays" do + [user, user2] = insert_pair(:user) + + {:ok, %{id: id, object: object}} = CommonAPI.post(user, %{"status" => "test"}) + {:ok, %{object: object2}} = CommonAPI.post(user, %{"status" => "test test"}) + + CommonAPI.favorite(id, user2) + + likes = %{ + "first" => + "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1", + "id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes", + "totalItems" => 3, + "type" => "OrderedCollection" + } + + new_data = Map.put(object2.data, "likes", likes) + + object2 + |> Ecto.Changeset.change(%{data: new_data}) + |> Repo.update() + + assert length(Object.get_by_id(object.id).data["likes"]) == 1 + assert is_map(Object.get_by_id(object2.id).data["likes"]) + + assert :ok == Mix.Tasks.Pleroma.Database.run(["fix_likes_collections"]) + + assert length(Object.get_by_id(object.id).data["likes"]) == 1 + assert Enum.empty?(Object.get_by_id(object2.id).data["likes"]) + end + end end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 5a7b1ed80..b8b998acd 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -424,6 +424,27 @@ test "it ensures that address fields become lists" do assert !is_nil(data["cc"]) end + test "it strips internal likes" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + + likes = %{ + "first" => + "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1", + "id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes", + "totalItems" => 3, + "type" => "OrderedCollection" + } + + object = Map.put(data["object"], "likes", likes) + data = Map.put(data, "object", object) + + {:ok, %Activity{object: object}} = Transmogrifier.handle_incoming(data) + + refute Map.has_key?(object.data, "likes") + end + test "it works for incoming update activities" do data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() @@ -1010,14 +1031,7 @@ test "it strips internal fields of article" do assert is_nil(modified["object"]["announcements"]) assert is_nil(modified["object"]["announcement_count"]) assert is_nil(modified["object"]["context_id"]) - end - - test "it adds like collection to object" do - activity = insert(:note_activity) - {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) - - assert modified["object"]["likes"]["type"] == "OrderedCollection" - assert modified["object"]["likes"]["totalItems"] == 0 + assert is_nil(modified["object"]["likes"]) end test "the directMessage flag is present" do -- cgit v1.2.3 From f7028ae8acc8a1a5af426725778b84c7648f5854 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 10 Aug 2019 20:54:17 +0000 Subject: tests: mastodon account views: add missing CommonAPI alias --- test/web/mastodon_api/account_view_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index f8a354272..592a941a9 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do use Pleroma.DataCase import Pleroma.Factory alias Pleroma.User + alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.AccountView test "Represent a user account" do -- cgit v1.2.3 From 25c818ed6fd8fb66143a544e8e745b3d0703c51b Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Mon, 29 Jul 2019 16:17:22 +0000 Subject: Redirect not logged-in users to the MastoFE login page on private instances --- CHANGELOG.md | 5 +++++ lib/pleroma/web/router.ex | 2 +- test/web/mastodon_api/mastodon_api_controller_test.exs | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ea0052c6..eb47be7cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: return the actual profile URL in the Account entity's `url` property when appropriate - Templates: properly style anchor tags - Objects being re-embedded to activities after being updated (e.g faved/reposted). Running 'mix pleroma.database prune_objects' again is advised. +- Not being able to access the Mastodon FE login page on private instances ### Added - Relays: Added a task to list relay subscriptions. @@ -35,6 +36,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [1.0.2] - 2019-07-28 ### Fixed - Not being able to pin unlisted posts +- Mastodon API: represent poll IDs as strings +- MediaProxy: fix matching filenames +- MediaProxy: fix filename encoding +- Migrations: fix a sporadic migration failure - Metadata rendering errors resulting in the entire page being inaccessible - Federation/MediaProxy not working with instances that have wrong certificate order - ActivityPub S2S: remote user deletions now work the same as local user deletions. diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ff9ed1640..3a8dda772 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -684,7 +684,7 @@ defmodule Pleroma.Web.Router do delete("/auth/sign_out", MastodonAPIController, :logout) scope [] do - pipe_through(:oauth_read_or_public) + pipe_through(:oauth_read) get("/web/*path", MastodonAPIController, :index) end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 7efdc7adb..de6f044e7 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2879,6 +2879,21 @@ test "redirects not logged-in users to the login page", %{conn: conn, path: path assert redirected_to(conn) == "/web/login" end + test "redirects not logged-in users to the login page on private instances", %{ + conn: conn, + path: path + } do + is_public = Pleroma.Config.get([:instance, :public]) + Pleroma.Config.put([:instance, :public], false) + + conn = get(conn, path) + + assert conn.status == 302 + assert redirected_to(conn) == "/web/login" + + Pleroma.Config.put([:instance, :public], is_public) + end + test "does not redirect logged in users to the login page", %{conn: conn, path: path} do token = insert(:oauth_token) -- cgit v1.2.3 From 4d0dd046533efb94af5974e9c7ef027155aafbdc Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 10 Aug 2019 21:18:26 +0000 Subject: MRF: ensure that subdomain_match calls are case-insensitive --- CHANGELOG.md | 1 + lib/pleroma/web/activity_pub/mrf.ex | 2 +- test/web/activity_pub/mrf/mrf_test.exs | 24 +++++++++++++++++++----- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb47be7cf..0a8f1e363 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Templates: properly style anchor tags - Objects being re-embedded to activities after being updated (e.g faved/reposted). Running 'mix pleroma.database prune_objects' again is advised. - Not being able to access the Mastodon FE login page on private instances +- MRF: ensure that subdomain_match calls are case-insensitive ### Added - Relays: Added a task to list relay subscriptions. diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index dd204b21c..caa2a3231 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -28,7 +28,7 @@ defp get_policies(_), do: [] @spec subdomains_regex([String.t()]) :: [Regex.t()] def subdomains_regex(domains) when is_list(domains) do - for domain <- domains, do: ~r(^#{String.replace(domain, "*.", "(.*\\.)*")}$) + for domain <- domains, do: ~r(^#{String.replace(domain, "*.", "(.*\\.)*")}$)i end @spec subdomain_match?([Regex.t()], String.t()) :: boolean() diff --git a/test/web/activity_pub/mrf/mrf_test.exs b/test/web/activity_pub/mrf/mrf_test.exs index a9cdf5317..1a888e18f 100644 --- a/test/web/activity_pub/mrf/mrf_test.exs +++ b/test/web/activity_pub/mrf/mrf_test.exs @@ -4,8 +4,8 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "subdomains_regex/1" do assert MRF.subdomains_regex(["unsafe.tld", "*.unsafe.tld"]) == [ - ~r/^unsafe.tld$/, - ~r/^(.*\.)*unsafe.tld$/ + ~r/^unsafe.tld$/i, + ~r/^(.*\.)*unsafe.tld$/i ] end @@ -13,7 +13,7 @@ test "subdomains_regex/1" do test "common domains" do regexes = MRF.subdomains_regex(["unsafe.tld", "unsafe2.tld"]) - assert regexes == [~r/^unsafe.tld$/, ~r/^unsafe2.tld$/] + assert regexes == [~r/^unsafe.tld$/i, ~r/^unsafe2.tld$/i] assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "unsafe2.tld") @@ -24,7 +24,7 @@ test "common domains" do test "wildcard domains with one subdomain" do regexes = MRF.subdomains_regex(["*.unsafe.tld"]) - assert regexes == [~r/^(.*\.)*unsafe.tld$/] + assert regexes == [~r/^(.*\.)*unsafe.tld$/i] assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "sub.unsafe.tld") @@ -35,12 +35,26 @@ test "wildcard domains with one subdomain" do test "wildcard domains with two subdomains" do regexes = MRF.subdomains_regex(["*.unsafe.tld"]) - assert regexes == [~r/^(.*\.)*unsafe.tld$/] + assert regexes == [~r/^(.*\.)*unsafe.tld$/i] assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "sub.sub.unsafe.tld") refute MRF.subdomain_match?(regexes, "sub.anotherunsafe.tld") refute MRF.subdomain_match?(regexes, "sub.unsafe.tldanother") end + + test "matches are case-insensitive" do + regexes = MRF.subdomains_regex(["UnSafe.TLD", "UnSAFE2.Tld"]) + + assert regexes == [~r/^UnSafe.TLD$/i, ~r/^UnSAFE2.Tld$/i] + + assert MRF.subdomain_match?(regexes, "UNSAFE.TLD") + assert MRF.subdomain_match?(regexes, "UNSAFE2.TLD") + assert MRF.subdomain_match?(regexes, "unsafe.tld") + assert MRF.subdomain_match?(regexes, "unsafe2.tld") + + refute MRF.subdomain_match?(regexes, "EXAMPLE.COM") + refute MRF.subdomain_match?(regexes, "example.com") + end end end -- cgit v1.2.3 From 14efbcf1f91b1b758397d3445695bf72b43dcb53 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 14 Aug 2019 01:05:50 +0000 Subject: add removed section to changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a8f1e363..70fce5882 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Support for wildcard domains in user domain blocks setting. - Configuration: `quarantined_instances` support wildcard domains. - Mix Tasks: `mix pleroma.database fix_likes_collections` + +### Removed - Federation: Remove `likes` from objects. ## [1.0.4] - 2019-08-01 -- cgit v1.2.3 From 60c75d6740e5a5194cac4ae690a7b84cfa104efa Mon Sep 17 00:00:00 2001 From: Maksim Date: Fri, 19 Jul 2019 16:20:23 +0000 Subject: #1110 fixed /api/pleroma/healthcheck --- .../web/twitter_api/controllers/util_controller.ex | 32 ++++++----- test/web/twitter_api/util_controller_test.exs | 64 +++++++++++++++++++++- 2 files changed, 79 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 39bc2efce..b37e51a3d 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -9,7 +9,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Comeonin.Pbkdf2 alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.Emoji + alias Pleroma.Healthcheck alias Pleroma.Notification alias Pleroma.User alias Pleroma.Web @@ -22,7 +24,8 @@ def help_test(conn, _params) do end def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do - with %User{} = user <- User.get_cached_by_nickname(nick), avatar = User.avatar_url(user) do + with %User{} = user <- User.get_cached_by_nickname(nick), + avatar = User.avatar_url(user) do conn |> render("subscribe.html", %{nickname: nick, avatar: avatar, error: false}) else @@ -335,20 +338,21 @@ def captcha(conn, _params) do end def healthcheck(conn, _params) do - info = - if Pleroma.Config.get([:instance, :healthcheck]) do - Pleroma.Healthcheck.system_info() - else - %{} - end + with true <- Config.get([:instance, :healthcheck]), + %{healthy: true} = info <- Healthcheck.system_info() do + json(conn, info) + else + %{healthy: false} = info -> + service_unavailable(conn, info) - conn = - if info[:healthy] do - conn - else - Plug.Conn.put_status(conn, :service_unavailable) - end + _ -> + service_unavailable(conn, %{}) + end + end - json(conn, info) + defp service_unavailable(conn, info) do + conn + |> put_status(:service_unavailable) + |> json(info) end end diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index cab9e5d90..9e4a3094c 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do alias Pleroma.User alias Pleroma.Web.CommonAPI import Pleroma.Factory + import Mock setup do Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -227,10 +228,67 @@ test "show follow account page if the `acct` is a account link", %{conn: conn} d end end - test "GET /api/pleroma/healthcheck", %{conn: conn} do - conn = get(conn, "/api/pleroma/healthcheck") + describe "GET /api/pleroma/healthcheck" do + setup do + config_healthcheck = Pleroma.Config.get([:instance, :healthcheck]) - assert conn.status in [200, 503] + on_exit(fn -> + Pleroma.Config.put([:instance, :healthcheck], config_healthcheck) + end) + + :ok + end + + test "returns 503 when healthcheck disabled", %{conn: conn} do + Pleroma.Config.put([:instance, :healthcheck], false) + + response = + conn + |> get("/api/pleroma/healthcheck") + |> json_response(503) + + assert response == %{} + end + + test "returns 200 when healthcheck enabled and all ok", %{conn: conn} do + Pleroma.Config.put([:instance, :healthcheck], true) + + with_mock Pleroma.Healthcheck, + system_info: fn -> %Pleroma.Healthcheck{healthy: true} end do + response = + conn + |> get("/api/pleroma/healthcheck") + |> json_response(200) + + assert %{ + "active" => _, + "healthy" => true, + "idle" => _, + "memory_used" => _, + "pool_size" => _ + } = response + end + end + + test "returns 503 when healthcheck enabled and health is false", %{conn: conn} do + Pleroma.Config.put([:instance, :healthcheck], true) + + with_mock Pleroma.Healthcheck, + system_info: fn -> %Pleroma.Healthcheck{healthy: false} end do + response = + conn + |> get("/api/pleroma/healthcheck") + |> json_response(503) + + assert %{ + "active" => _, + "healthy" => false, + "idle" => _, + "memory_used" => _, + "pool_size" => _ + } = response + end + end end describe "POST /api/pleroma/disable_account" do -- cgit v1.2.3 From 5af3c00072b27e4f23567e0defbc3c021ed11b2b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 11 Aug 2019 22:49:55 +0300 Subject: Do not fetch the reply object in `fix_type` unless the object has the `name` key and use a depth limit when fetching it --- lib/pleroma/web/activity_pub/transmogrifier.ex | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 65c1f9d9f..af756c21e 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -324,17 +324,24 @@ def fix_content_map(%{"contentMap" => content_map} = object) do def fix_content_map(object), do: object - def fix_type(%{"inReplyTo" => reply_id} = object) when is_binary(reply_id) do - reply = Object.normalize(reply_id) + def fix_type(object, options \\ []) - if reply && (reply.data["type"] == "Question" and object["name"]) do + def fix_type(%{"inReplyTo" => reply_id, "name" => _} = object, options) + when is_binary(reply_id) do + reply = + with true <- Federator.allowed_incoming_reply_depth?(options[:depth]), + {:ok, object} <- get_obj_helper(reply_id) do + object + end + + if reply && reply.data["type"] == "Question" do Map.put(object, "type", "Answer") else object end end - def fix_type(object), do: object + def fix_type(object, _), do: object defp mastodon_follow_hack(%{"id" => id, "actor" => follower_id}, followed) do with true <- id =~ "follows", -- cgit v1.2.3 From 54405919d86ce803bf1a17a55c382c2bc59db3b0 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 14 Aug 2019 01:12:42 +0000 Subject: changelog updates --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70fce5882..2a1cfccf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Objects being re-embedded to activities after being updated (e.g faved/reposted). Running 'mix pleroma.database prune_objects' again is advised. - Not being able to access the Mastodon FE login page on private instances - MRF: ensure that subdomain_match calls are case-insensitive +- Fix internal server error when using the healthcheck API. +- `federation_incoming_replies_max_depth` option being ignored in certain cases. ### Added - Relays: Added a task to list relay subscriptions. -- cgit v1.2.3 From 29db8dc79945026d2024e2462f86b1795859c574 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 02:15:21 +0000 Subject: config: remove legacy activitypub accept_blocks setting Anyone who is interested in dropping blocks can write their own MRF policy at this point. This setting predated the MRF framework. Disabling the side effect (unsubscription) is still a config option per policy. --- CHANGELOG.md | 1 + config/config.exs | 1 - docs/config.md | 1 - lib/pleroma/web/activity_pub/transmogrifier.ex | 6 ++---- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a1cfccf2..35dcd3401 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Removed - Federation: Remove `likes` from objects. +- ActivityPub: The `accept_blocks` configuration setting. ## [1.0.4] - 2019-08-01 ### Fixed diff --git a/config/config.exs b/config/config.exs index 872ee3b00..b145cab22 100644 --- a/config/config.exs +++ b/config/config.exs @@ -300,7 +300,6 @@ default_mascot: :pleroma_fox_tan config :pleroma, :activitypub, - accept_blocks: true, unfollow_blocked: true, outgoing_blocks: true, follow_handshake_timeout: 500 diff --git a/docs/config.md b/docs/config.md index 22799af23..3b9be73ec 100644 --- a/docs/config.md +++ b/docs/config.md @@ -319,7 +319,6 @@ config :pleroma, Pleroma.Web.Endpoint, This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls starting with `https://example.com:2020` ## :activitypub -* ``accept_blocks``: Whether to accept incoming block activities from other instances * ``unfollow_blocked``: Whether blocks result in people getting unfollowed * ``outgoing_blocks``: Whether to federate blocks to other instances * ``deny_follow_blocked``: Whether to disallow following an account that has blocked the user in question diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index af756c21e..ca6940026 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -678,8 +678,7 @@ def handle_incoming( "id" => id } = _data ) do - with true <- Pleroma.Config.get([:activitypub, :accept_blocks]), - %User{local: true} = blocked <- User.get_cached_by_ap_id(blocked), + with %User{local: true} = blocked <- User.get_cached_by_ap_id(blocked), {:ok, %User{} = blocker} <- User.get_or_fetch_by_ap_id(blocker), {:ok, activity} <- ActivityPub.unblock(blocker, blocked, id, false) do User.unblock(blocker, blocked) @@ -692,8 +691,7 @@ def handle_incoming( def handle_incoming( %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data ) do - with true <- Pleroma.Config.get([:activitypub, :accept_blocks]), - %User{local: true} = blocked = User.get_cached_by_ap_id(blocked), + with %User{local: true} = blocked = User.get_cached_by_ap_id(blocked), {:ok, %User{} = blocker} = User.get_or_fetch_by_ap_id(blocker), {:ok, activity} <- ActivityPub.block(blocker, blocked, id, false) do User.unfollow(blocker, blocked) -- cgit v1.2.3 From e13edc2d3b83bdb063a991edcf56721eef084f6d Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 21:26:24 +0000 Subject: MRF: add describe() for gathering and describing the MRF configuration --- lib/pleroma/web/activity_pub/mrf.ex | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index caa2a3231..d43a8760b 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -35,4 +35,20 @@ def subdomains_regex(domains) when is_list(domains) do def subdomain_match?(domains, host) do Enum.any?(domains, fn domain -> Regex.match?(domain, host) end) end + + @callback describe() :: {:ok | :error, Map.t()} + + def describe(policies) do + policies + |> Enum.reduce({:ok, %{}}, fn + policy, {:ok, data} -> + {:ok, policy_data} = policy.describe() + {:ok, Map.merge(data, policy_data)} + + _, error -> + error + end) + end + + def describe(), do: get_policies() |> describe() end -- cgit v1.2.3 From b8186c26fa319eb9fa3d8417a9a3fbe387f40b9a Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 21:29:15 +0000 Subject: test: add mock MRF module for describe() testing --- test/support/mrf_module_mock.ex | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 test/support/mrf_module_mock.ex diff --git a/test/support/mrf_module_mock.ex b/test/support/mrf_module_mock.ex new file mode 100644 index 000000000..e5ae21ad8 --- /dev/null +++ b/test/support/mrf_module_mock.ex @@ -0,0 +1,13 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule MRFModuleMock do + @behaviour Pleroma.Web.ActivityPub.MRF + + @impl true + def filter(message), do: {:ok, message} + + @impl true + def describe(), do: %{"mrf_module_mock" => "some config data"} +end -- cgit v1.2.3 From 7e2bc39f3c51bc7f25238d4a349f2c39c8b9f107 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 21:52:54 +0000 Subject: MRF: add describe() to all modules, add base MRF configuration to base describe() --- lib/pleroma/web/activity_pub/mrf.ex | 32 ++++++++++++++++------ .../web/activity_pub/mrf/anti_followbot_policy.ex | 3 ++ .../web/activity_pub/mrf/anti_link_spam_policy.ex | 6 ++++ lib/pleroma/web/activity_pub/mrf/drop_policy.ex | 3 ++ .../web/activity_pub/mrf/ensure_re_prepended.ex | 2 ++ .../web/activity_pub/mrf/hellthread_policy.ex | 3 ++ lib/pleroma/web/activity_pub/mrf/keyword_policy.ex | 32 ++++++++++++++++++++++ .../activity_pub/mrf/no_placeholder_text_policy.ex | 3 ++ lib/pleroma/web/activity_pub/mrf/noop_policy.ex | 3 ++ .../web/activity_pub/mrf/normalize_markup.ex | 2 ++ .../web/activity_pub/mrf/reject_non_public.ex | 3 ++ lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 12 ++++++++ .../web/activity_pub/mrf/subchain_policy.ex | 3 ++ lib/pleroma/web/activity_pub/mrf/tag_policy.ex | 3 ++ lib/pleroma/web/activity_pub/mrf/user_allowlist.ex | 9 ++++++ 15 files changed, 111 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index d43a8760b..7533552d5 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -39,15 +39,31 @@ def subdomain_match?(domains, host) do @callback describe() :: {:ok | :error, Map.t()} def describe(policies) do - policies - |> Enum.reduce({:ok, %{}}, fn - policy, {:ok, data} -> - {:ok, policy_data} = policy.describe() - {:ok, Map.merge(data, policy_data)} + {:ok, policy_configs} = + policies + |> Enum.reduce({:ok, %{}}, fn + policy, {:ok, data} -> + {:ok, policy_data} = policy.describe() + {:ok, Map.merge(data, policy_data)} - _, error -> - error - end) + _, error -> + error + end) + + mrf_policies = + get_policies() + |> Enum.map(fn policy -> to_string(policy) |> String.split(".") |> List.last() end) + + exclusions = Pleroma.Config.get([:instance, :mrf_transparency_exclusions]) + + base = + %{ + mrf_policies: mrf_policies, + exclusions: length(exclusions) > 0, + } + |> Map.merge(policy_configs) + + {:ok, base} end def describe(), do: get_policies() |> describe() diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 87fa514c3..ad2d9bf54 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -62,4 +62,7 @@ def filter(%{"type" => "Follow", "actor" => actor_id} = message) do @impl true def filter(message), do: {:ok, message} + + @impl true + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index 2da3eac2f..d27386591 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -5,6 +5,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do alias Pleroma.User + @behaviour Pleroma.Web.ActivityPub.MRF + require Logger # has the user successfully posted before? @@ -22,6 +24,7 @@ defp contains_links?(%{"content" => content} = _object) do defp contains_links?(_), do: false + @impl true def filter(%{"type" => "Create", "actor" => actor, "object" => object} = message) do with {:ok, %User{} = u} <- User.get_or_fetch_by_ap_id(actor), {:contains_links, true} <- {:contains_links, contains_links?(object)}, @@ -45,4 +48,7 @@ def filter(%{"type" => "Create", "actor" => actor, "object" => object} = message # in all other cases, pass through def filter(message), do: {:ok, message} + + @impl true + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex index b8d38aae6..dcb640b12 100644 --- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex @@ -12,4 +12,7 @@ def filter(object) do Logger.info("REJECTING #{inspect(object)}") {:reject, object} end + + @impl true + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index 15d8514be..259920cff 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -42,4 +42,6 @@ def filter(%{"type" => activity_type} = object) when activity_type == "Create" d end def filter(object), do: {:ok, object} + + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index a699f6a7e..deb4995dd 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -87,4 +87,7 @@ def filter(%{"type" => "Create"} = message) do @impl true def filter(message), do: {:ok, message} + + @impl true + def describe(), do: {:ok, %{mrf_hellthread: Pleroma.Config.get([:mrf_hellthread])}} end diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index d5c341433..dfa1704c4 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -94,4 +94,36 @@ def filter(%{"type" => "Create", "object" => %{"content" => _content}} = message @impl true def filter(message), do: {:ok, message} + + @impl true + def describe() do + # This horror is needed to convert regex sigils to strings + mrf_keyword = + Pleroma.Config.get(:mrf_keyword, []) + |> Enum.map(fn {key, value} -> + {key, + Enum.map(value, fn + {pattern, replacement} -> + %{ + "pattern" => + if not is_binary(pattern) do + inspect(pattern) + else + pattern + end, + "replacement" => replacement + } + + pattern -> + if not is_binary(pattern) do + inspect(pattern) + else + pattern + end + end)} + end) + |> Enum.into(%{}) + + {:ok, %{mrf_keyword: mrf_keyword}} + end end diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index f30fee0d5..a94cda856 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -27,4 +27,7 @@ def filter( @impl true def filter(object), do: {:ok, object} + + @impl true + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/noop_policy.ex b/lib/pleroma/web/activity_pub/mrf/noop_policy.ex index c47cb3298..19890ef0c 100644 --- a/lib/pleroma/web/activity_pub/mrf/noop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/noop_policy.ex @@ -10,4 +10,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do def filter(object) do {:ok, object} end + + @impl true + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex index 9c87c6963..36b0763be 100644 --- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex +++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex @@ -25,4 +25,6 @@ def filter(%{"type" => activity_type} = object) when activity_type == "Create" d end def filter(object), do: {:ok, object} + + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index ea3df1b4d..52ffc4ca3 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -48,4 +48,7 @@ def filter(%{"type" => "Create"} = object) do @impl true def filter(object), do: {:ok, object} + + @impl true + def describe(), do: {:ok, %{mrf_rejectnonpublic: Pleroma.Config.get([:mrf_rejectnonpublic])}} end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 2cf63d3db..07bef362f 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -179,4 +179,16 @@ def filter(%{"id" => actor, "type" => obj_type} = object) end def filter(object), do: {:ok, object} + + @impl true + def describe() do + exclusions = Pleroma.Config.get([:instance, :mrf_transparency_exclusions]) + + mrf_simple = + Pleroma.Config.get(:mrf_simple) + |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn v -> v in exclusions end)} end) + |> Enum.into(%{}) + + {:ok, %{mrf_simple: mrf_simple}} + end end diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex index 765704389..b69410ca8 100644 --- a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex @@ -37,4 +37,7 @@ def filter(%{"actor" => actor} = message) do @impl true def filter(message), do: {:ok, message} + + @impl true + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index 6683b8d8e..de4e53cb8 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -149,4 +149,7 @@ def filter(%{"actor" => actor, "type" => "Create"} = message), @impl true def filter(message), do: {:ok, message} + + @impl true + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex index 47663414a..2b5f7d516 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex @@ -27,4 +27,13 @@ def filter(%{"actor" => actor} = object) do end def filter(object), do: {:ok, object} + + @impl true + def describe() do + mrf_user_allowlist = + Config.get([:mrf_user_allowlist], []) + |> Enum.into(%{}, fn {k, v} -> {k, length(v)} end) + + {:ok, %{mrf_user_allowlist: mrf_user_allowlist}} + end end -- cgit v1.2.3 From 4ccd3410a859e0bb2365aee9967c7d8e88487591 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 21:57:39 +0000 Subject: nodeinfo: use MRF.describe() instead of hardcoded MRF transparency stuff --- lib/pleroma/web/nodeinfo/nodeinfo_controller.ex | 54 ++----------------------- 1 file changed, 4 insertions(+), 50 deletions(-) diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index dc4858e43..066e9f5ba 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -34,64 +34,18 @@ def schemas(conn, _params) do def raw_nodeinfo do stats = Stats.get_stats() - exclusions = Config.get([:instance, :mrf_transparency_exclusions]) - - mrf_simple = - Config.get(:mrf_simple) - |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn v -> v in exclusions end)} end) - |> Enum.into(%{}) - - # This horror is needed to convert regex sigils to strings - mrf_keyword = - Config.get(:mrf_keyword, []) - |> Enum.map(fn {key, value} -> - {key, - Enum.map(value, fn - {pattern, replacement} -> - %{ - "pattern" => - if not is_binary(pattern) do - inspect(pattern) - else - pattern - end, - "replacement" => replacement - } - - pattern -> - if not is_binary(pattern) do - inspect(pattern) - else - pattern - end - end)} - end) - |> Enum.into(%{}) - - mrf_policies = - MRF.get_policies() - |> Enum.map(fn policy -> to_string(policy) |> String.split(".") |> List.last() end) - quarantined = Config.get([:instance, :quarantined_instances], []) staff_accounts = User.all_superusers() |> Enum.map(fn u -> u.ap_id end) - mrf_user_allowlist = - Config.get([:mrf_user_allowlist], []) - |> Enum.into(%{}, fn {k, v} -> {k, length(v)} end) - federation_response = if Config.get([:instance, :mrf_transparency]) do - %{ - mrf_policies: mrf_policies, - mrf_simple: mrf_simple, - mrf_keyword: mrf_keyword, - mrf_user_allowlist: mrf_user_allowlist, - quarantined_instances: quarantined, - exclusions: length(exclusions) > 0 - } + {:ok, data} = MRF.describe() + + data + |> Map.merge(%{quarantined_instances: quarantined}) else %{} end -- cgit v1.2.3 From c9468c9a597f768e0b236b65e5da7979df4ad42a Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 22:19:15 +0000 Subject: tests: add tests for MRF.describe() --- test/support/mrf_module_mock.ex | 2 +- test/web/activity_pub/mrf/mrf_test.exs | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/test/support/mrf_module_mock.ex b/test/support/mrf_module_mock.ex index e5ae21ad8..573eb0147 100644 --- a/test/support/mrf_module_mock.ex +++ b/test/support/mrf_module_mock.ex @@ -9,5 +9,5 @@ defmodule MRFModuleMock do def filter(message), do: {:ok, message} @impl true - def describe(), do: %{"mrf_module_mock" => "some config data"} + def describe(), do: {:ok, %{mrf_module_mock: "some config data"}} end diff --git a/test/web/activity_pub/mrf/mrf_test.exs b/test/web/activity_pub/mrf/mrf_test.exs index 1a888e18f..19e172939 100644 --- a/test/web/activity_pub/mrf/mrf_test.exs +++ b/test/web/activity_pub/mrf/mrf_test.exs @@ -57,4 +57,30 @@ test "matches are case-insensitive" do refute MRF.subdomain_match?(regexes, "example.com") end end + + describe "describe/0" do + test "it works as expected with noop policy" do + expected = %{ + mrf_policies: ["NoOpPolicy"], + exclusions: false + } + + {:ok, ^expected} = MRF.describe() + end + + test "it works as expected with mock policy" do + config = Pleroma.Config.get([:instance, :rewrite_policy]) + Pleroma.Config.put([:instance, :rewrite_policy], [MRFModuleMock]) + + expected = %{ + mrf_policies: ["MRFModuleMock"], + mrf_module_mock: "some config data", + exclusions: false + } + + {:ok, ^expected} = MRF.describe() + + Pleroma.Config.put([:instance, :rewrite_policy], config) + end + end end -- cgit v1.2.3 From 3af51afdd7773db566ed8c92c19d1605705f8208 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 22:32:40 +0000 Subject: tests: fix up nodeinfo tests --- test/web/node_info_test.exs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/web/node_info_test.exs b/test/web/node_info_test.exs index d7f848bfa..f6147c286 100644 --- a/test/web/node_info_test.exs +++ b/test/web/node_info_test.exs @@ -85,6 +85,9 @@ test "it returns the safe_dm_mentions feature if enabled", %{conn: conn} do end test "it shows MRF transparency data if enabled", %{conn: conn} do + config = Pleroma.Config.get([:instance, :rewrite_policy]) + Pleroma.Config.put([:instance, :rewrite_policy], [Pleroma.Web.ActivityPub.MRF.SimplePolicy]) + option = Pleroma.Config.get([:instance, :mrf_transparency]) Pleroma.Config.put([:instance, :mrf_transparency], true) @@ -98,11 +101,15 @@ test "it shows MRF transparency data if enabled", %{conn: conn} do assert response["metadata"]["federation"]["mrf_simple"] == simple_config + Pleroma.Config.put([:instance, :rewrite_policy], config) Pleroma.Config.put([:instance, :mrf_transparency], option) Pleroma.Config.put(:mrf_simple, %{}) end test "it performs exclusions from MRF transparency data if configured", %{conn: conn} do + config = Pleroma.Config.get([:instance, :rewrite_policy]) + Pleroma.Config.put([:instance, :rewrite_policy], [Pleroma.Web.ActivityPub.MRF.SimplePolicy]) + option = Pleroma.Config.get([:instance, :mrf_transparency]) Pleroma.Config.put([:instance, :mrf_transparency], true) @@ -122,6 +129,7 @@ test "it performs exclusions from MRF transparency data if configured", %{conn: assert response["metadata"]["federation"]["mrf_simple"] == expected_config assert response["metadata"]["federation"]["exclusions"] == true + Pleroma.Config.put([:instance, :rewrite_policy], config) Pleroma.Config.put([:instance, :mrf_transparency], option) Pleroma.Config.put([:instance, :mrf_transparency_exclusions], exclusions) Pleroma.Config.put(:mrf_simple, %{}) -- cgit v1.2.3 From fb77dc50aae4fb9c8bdb7b20dc874e6a85ce322b Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 22:36:24 +0000 Subject: fix credo --- lib/pleroma/web/activity_pub/mrf.ex | 4 ++-- lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/drop_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex | 2 +- lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/keyword_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/noop_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/normalize_markup.ex | 2 +- lib/pleroma/web/activity_pub/mrf/reject_non_public.ex | 2 +- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/subchain_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/tag_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/user_allowlist.ex | 2 +- test/support/mrf_module_mock.ex | 2 +- 16 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 7533552d5..263ed11af 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -59,12 +59,12 @@ def describe(policies) do base = %{ mrf_policies: mrf_policies, - exclusions: length(exclusions) > 0, + exclusions: length(exclusions) > 0 } |> Map.merge(policy_configs) {:ok, base} end - def describe(), do: get_policies() |> describe() + def describe, do: get_policies() |> describe() end diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index ad2d9bf54..de1eb4aa5 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -64,5 +64,5 @@ def filter(%{"type" => "Follow", "actor" => actor_id} = message) do def filter(message), do: {:ok, message} @impl true - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index d27386591..b90193ca0 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -50,5 +50,5 @@ def filter(%{"type" => "Create", "actor" => actor, "object" => object} = message def filter(message), do: {:ok, message} @impl true - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex index dcb640b12..f7831bc3e 100644 --- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex @@ -14,5 +14,5 @@ def filter(object) do end @impl true - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index 259920cff..4cc45233f 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -43,5 +43,5 @@ def filter(%{"type" => activity_type} = object) when activity_type == "Create" d def filter(object), do: {:ok, object} - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index deb4995dd..621c9fa24 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -89,5 +89,5 @@ def filter(%{"type" => "Create"} = message) do def filter(message), do: {:ok, message} @impl true - def describe(), do: {:ok, %{mrf_hellthread: Pleroma.Config.get([:mrf_hellthread])}} + def describe, do: {:ok, %{mrf_hellthread: Pleroma.Config.get([:mrf_hellthread])}} end diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index dfa1704c4..4852355ac 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -96,7 +96,7 @@ def filter(%{"type" => "Create", "object" => %{"content" => _content}} = message def filter(message), do: {:ok, message} @impl true - def describe() do + def describe do # This horror is needed to convert regex sigils to strings mrf_keyword = Pleroma.Config.get(:mrf_keyword, []) diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index a94cda856..3ae81e025 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -29,5 +29,5 @@ def filter( def filter(object), do: {:ok, object} @impl true - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/noop_policy.ex b/lib/pleroma/web/activity_pub/mrf/noop_policy.ex index 19890ef0c..878c57925 100644 --- a/lib/pleroma/web/activity_pub/mrf/noop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/noop_policy.ex @@ -12,5 +12,5 @@ def filter(object) do end @impl true - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex index 36b0763be..62ed379f1 100644 --- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex +++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex @@ -26,5 +26,5 @@ def filter(%{"type" => activity_type} = object) when activity_type == "Create" d def filter(object), do: {:ok, object} - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index 52ffc4ca3..adaeb1c91 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -50,5 +50,5 @@ def filter(%{"type" => "Create"} = object) do def filter(object), do: {:ok, object} @impl true - def describe(), do: {:ok, %{mrf_rejectnonpublic: Pleroma.Config.get([:mrf_rejectnonpublic])}} + def describe, do: {:ok, %{mrf_rejectnonpublic: Pleroma.Config.get([:mrf_rejectnonpublic])}} end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 07bef362f..56ff47304 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -181,7 +181,7 @@ def filter(%{"id" => actor, "type" => obj_type} = object) def filter(object), do: {:ok, object} @impl true - def describe() do + def describe do exclusions = Pleroma.Config.get([:instance, :mrf_transparency_exclusions]) mrf_simple = diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex index b69410ca8..566c1e191 100644 --- a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex @@ -39,5 +39,5 @@ def filter(%{"actor" => actor} = message) do def filter(message), do: {:ok, message} @impl true - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index de4e53cb8..2e2292aff 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -151,5 +151,5 @@ def filter(%{"actor" => actor, "type" => "Create"} = message), def filter(message), do: {:ok, message} @impl true - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex index 2b5f7d516..f77d1a3a3 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex @@ -29,7 +29,7 @@ def filter(%{"actor" => actor} = object) do def filter(object), do: {:ok, object} @impl true - def describe() do + def describe do mrf_user_allowlist = Config.get([:mrf_user_allowlist], []) |> Enum.into(%{}, fn {k, v} -> {k, length(v)} end) diff --git a/test/support/mrf_module_mock.ex b/test/support/mrf_module_mock.ex index 573eb0147..12c7e22bc 100644 --- a/test/support/mrf_module_mock.ex +++ b/test/support/mrf_module_mock.ex @@ -9,5 +9,5 @@ defmodule MRFModuleMock do def filter(message), do: {:ok, message} @impl true - def describe(), do: {:ok, %{mrf_module_mock: "some config data"}} + def describe, do: {:ok, %{mrf_module_mock: "some config data"}} end -- cgit v1.2.3 From eed36278a60e1366635343754bf0c660c49b8ad4 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 20:28:59 +0000 Subject: MRF: add vocabulary policy module --- config/config.exs | 4 +++ .../web/activity_pub/mrf/vocabulary_policy.ex | 34 ++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex diff --git a/config/config.exs b/config/config.exs index b145cab22..66a8c18ea 100644 --- a/config/config.exs +++ b/config/config.exs @@ -333,6 +333,10 @@ config :pleroma, :mrf_subchain, match_actor: %{} +config :pleroma, :mrf_vocabulary, + accept: [], + reject: [] + config :pleroma, :rich_media, enabled: true, ignore_hosts: [], diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex new file mode 100644 index 000000000..de00b23da --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -0,0 +1,34 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do + @moduledoc "Filter messages which belong to certain activity vocabularies" + + @behaviour Pleroma.Web.ActivityPub.MRF + + def filter(%{"type" => "Undo", "object" => child_message} = message) do + with {:ok, _} <- filter(child_message) do + {:ok, message} + else + {:reject, nil} -> + {:reject, nil} + end + end + + def filter(%{"type" => message_type} = message) do + with accepted_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :accept]), + rejected_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :reject]), + true <- + length(accepted_vocabulary) == 0 || Enum.member?(accepted_vocabulary, message_type), + false <- + length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type), + {:ok, _} <- filter(message["object"]) do + {:ok, message} + else + _ -> {:reject, nil} + end + end + + def filter(message), do: {:ok, message} +end -- cgit v1.2.3 From 7a5e1d64a53f719c085cbff30228318311651a14 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 20:32:43 +0000 Subject: docs: document mrf_vocabulary module settings --- docs/config.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/config.md b/docs/config.md index 3b9be73ec..a4461521b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -98,6 +98,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links. + * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (see `:mrf_vocabulary` section) * `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. * `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. * `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` @@ -266,6 +267,10 @@ config :pleroma, :mrf_subchain, * `federated_timeline_removal`: A list of patterns which result in message being removed from federated timelines (a.k.a unlisted), each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) * `replace`: A list of tuples containing `{pattern, replacement}`, `pattern` can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) +## :mrf_vocabulary +* `accept`: A list of ActivityStreams terms to accept. If empty, all messages are accepted. +* `reject`: A list of ActivityStreams terms to reject. If empty, no messages are rejected. + ## :media_proxy * `enabled`: Enables proxying of remote media to the instance’s proxy * `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts. -- cgit v1.2.3 From d5df62edd8297251fcbd7472ede27e76d6ded419 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 20:55:13 +0000 Subject: tests: add tests for mrf_vocabulary --- .../activity_pub/mrf/vocabulary_policy_test.exs | 123 +++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 test/web/activity_pub/mrf/vocabulary_policy_test.exs diff --git a/test/web/activity_pub/mrf/vocabulary_policy_test.exs b/test/web/activity_pub/mrf/vocabulary_policy_test.exs new file mode 100644 index 000000000..c3b11d7a1 --- /dev/null +++ b/test/web/activity_pub/mrf/vocabulary_policy_test.exs @@ -0,0 +1,123 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicyTest do + use Pleroma.DataCase + + alias Pleroma.Web.ActivityPub.MRF.VocabularyPolicy + + describe "accept" do + test "it accepts based on parent activity type" do + config = Pleroma.Config.get([:mrf_vocabulary, :accept]) + Pleroma.Config.put([:mrf_vocabulary, :accept], ["Like"]) + + message = %{ + "type" => "Like", + "object" => "whatever" + } + + {:ok, ^message} = VocabularyPolicy.filter(message) + + Pleroma.Config.put([:mrf_vocabulary, :accept], config) + end + + test "it accepts based on child object type" do + config = Pleroma.Config.get([:mrf_vocabulary, :accept]) + Pleroma.Config.put([:mrf_vocabulary, :accept], ["Create", "Note"]) + + message = %{ + "type" => "Create", + "object" => %{ + "type" => "Note", + "content" => "whatever" + } + } + + {:ok, ^message} = VocabularyPolicy.filter(message) + + Pleroma.Config.put([:mrf_vocabulary, :accept], config) + end + + test "it does not accept disallowed child objects" do + config = Pleroma.Config.get([:mrf_vocabulary, :accept]) + Pleroma.Config.put([:mrf_vocabulary, :accept], ["Create", "Note"]) + + message = %{ + "type" => "Create", + "object" => %{ + "type" => "Article", + "content" => "whatever" + } + } + + {:reject, nil} = VocabularyPolicy.filter(message) + + Pleroma.Config.put([:mrf_vocabulary, :accept], config) + end + + test "it does not accept disallowed parent types" do + config = Pleroma.Config.get([:mrf_vocabulary, :accept]) + Pleroma.Config.put([:mrf_vocabulary, :accept], ["Announce", "Note"]) + + message = %{ + "type" => "Create", + "object" => %{ + "type" => "Note", + "content" => "whatever" + } + } + + {:reject, nil} = VocabularyPolicy.filter(message) + + Pleroma.Config.put([:mrf_vocabulary, :accept], config) + end + end + + describe "reject" do + test "it rejects based on parent activity type" do + config = Pleroma.Config.get([:mrf_vocabulary, :reject]) + Pleroma.Config.put([:mrf_vocabulary, :reject], ["Like"]) + + message = %{ + "type" => "Like", + "object" => "whatever" + } + + {:reject, nil} = VocabularyPolicy.filter(message) + + Pleroma.Config.put([:mrf_vocabulary, :reject], config) + end + + test "it rejects based on child object type" do + config = Pleroma.Config.get([:mrf_vocabulary, :reject]) + Pleroma.Config.put([:mrf_vocabulary, :reject], ["Note"]) + + message = %{ + "type" => "Create", + "object" => %{ + "type" => "Note", + "content" => "whatever" + } + } + + {:reject, nil} = VocabularyPolicy.filter(message) + + Pleroma.Config.put([:mrf_vocabulary, :reject], config) + end + + test "it passes through objects that aren't disallowed" do + config = Pleroma.Config.get([:mrf_vocabulary, :reject]) + Pleroma.Config.put([:mrf_vocabulary, :reject], ["Like"]) + + message = %{ + "type" => "Announce", + "object" => "whatever" + } + + {:ok, ^message} = VocabularyPolicy.filter(message) + + Pleroma.Config.put([:mrf_vocabulary, :reject], config) + end + end +end -- cgit v1.2.3 From b3afc0bb326e3b4d95e73684c71c1bf8f695d3c6 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 21:00:23 +0000 Subject: update changelog for mrf_vocabulary --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35dcd3401..3c85bb90d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - Relays: Added a task to list relay subscriptions. +- MRF: Support for filtering posts based on ActivityStreams vocabulary (`Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`) - MRF (Simple Policy): Support for wildcard domains. - Support for wildcard domains in user domain blocks setting. - Configuration: `quarantined_instances` support wildcard domains. -- cgit v1.2.3 From c7a3a15c3de93212a131ea5886278c4030309941 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 22:39:26 +0000 Subject: mrf_vocabulary: add describe API support --- lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index de00b23da..74da8d57e 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -31,4 +31,6 @@ def filter(%{"type" => message_type} = message) do end def filter(message), do: {:ok, message} + + def describe, do: {:ok, %{mrf_vocabulary: Pleroma.Config.get(:mrf_vocabulary)}} end -- cgit v1.2.3 From 4681747ec1e889530b0df52fe599aa236078a0b6 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 22:40:18 +0000 Subject: docs tweak --- docs/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.md b/docs/config.md index a4461521b..4699baa80 100644 --- a/docs/config.md +++ b/docs/config.md @@ -268,7 +268,7 @@ config :pleroma, :mrf_subchain, * `replace`: A list of tuples containing `{pattern, replacement}`, `pattern` can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) ## :mrf_vocabulary -* `accept`: A list of ActivityStreams terms to accept. If empty, all messages are accepted. +* `accept`: A list of ActivityStreams terms to accept. If empty, all supported messages are accepted. * `reject`: A list of ActivityStreams terms to reject. If empty, no messages are rejected. ## :media_proxy -- cgit v1.2.3 From 606ccbab035dce7dfc543e1d6354ed8ae3c87c3c Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 14 Aug 2019 01:36:42 +0000 Subject: update changelog to cover MRF describe API. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c85bb90d..4effcf887 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - `federation_incoming_replies_max_depth` option being ignored in certain cases. ### Added +- **Breaking:** MRF describe API, which adds support for exposing configuration information about MRF policies to NodeInfo. + Custom modules will need to be updated by adding, at the very least, `def describe, do: {:ok, %{}}` to the MRF policy modules. - Relays: Added a task to list relay subscriptions. - MRF: Support for filtering posts based on ActivityStreams vocabulary (`Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`) - MRF (Simple Policy): Support for wildcard domains. -- cgit v1.2.3 From e652f83e5a99e7f8526952147eec39eee63914e1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 9 Jul 2019 10:39:36 +0000 Subject: Apply suggestion to docs/config.md --- docs/config.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/config.md b/docs/config.md index 4699baa80..37824ecac 100644 --- a/docs/config.md +++ b/docs/config.md @@ -87,6 +87,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`). * `account_activation_required`: Require users to confirm their emails before signing in. * `federating`: Enable federation with other instances +* `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while fetching very long threads. If set to `nil`, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes. * `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it. * `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance * `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default: -- cgit v1.2.3 From 1a46a13d1523347e8310dde9f5bddf396bd11342 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 29 Jun 2019 20:04:50 +0300 Subject: [#161] Limited replies depth on incoming federation in order to prevent memory leaks on recursive replies fetching. --- lib/pleroma/object.ex | 24 ++--- lib/pleroma/object/fetcher.ex | 8 +- lib/pleroma/web/activity_pub/transmogrifier.ex | 110 ++++++++++++++--------- lib/pleroma/web/federator/federator.ex | 6 ++ lib/pleroma/web/ostatus/handlers/note_handler.ex | 13 +-- lib/pleroma/web/ostatus/ostatus.ex | 22 ++--- test/web/activity_pub/transmogrifier_test.exs | 37 ++++++-- test/web/ostatus/ostatus_test.exs | 28 +++++- 8 files changed, 169 insertions(+), 79 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 4b181ec59..b8647dd26 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -44,20 +44,20 @@ def get_by_ap_id(ap_id) do Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id))) end - def normalize(_, fetch_remote \\ true) + def normalize(_, fetch_remote \\ true, options \\ []) # If we pass an Activity to Object.normalize(), we can try to use the preloaded object. # Use this whenever possible, especially when walking graphs in an O(N) loop! - def normalize(%Object{} = object, _), do: object - def normalize(%Activity{object: %Object{} = object}, _), do: object + def normalize(%Object{} = object, _, _), do: object + def normalize(%Activity{object: %Object{} = object}, _, _), do: object # A hack for fake activities - def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _) do + def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _, _) do %Object{id: "pleroma:fake_object_id", data: data} end # Catch and log Object.normalize() calls where the Activity's child object is not # preloaded. - def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote) do + def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote, _) do Logger.debug( "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" ) @@ -67,7 +67,7 @@ def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote) do normalize(ap_id, fetch_remote) end - def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote) do + def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote, _) do Logger.debug( "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" ) @@ -78,10 +78,14 @@ def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote) do end # Old way, try fetching the object through cache. - def normalize(%{"id" => ap_id}, fetch_remote), do: normalize(ap_id, fetch_remote) - def normalize(ap_id, false) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) - def normalize(ap_id, true) when is_binary(ap_id), do: Fetcher.fetch_object_from_id!(ap_id) - def normalize(_, _), do: nil + def normalize(%{"id" => ap_id}, fetch_remote, _), do: normalize(ap_id, fetch_remote) + def normalize(ap_id, false, _) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) + + def normalize(ap_id, true, options) when is_binary(ap_id) do + Fetcher.fetch_object_from_id!(ap_id, options) + end + + def normalize(_, _, _), do: nil # Owned objects can only be mutated by their owner def authorize_mutation(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}), diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 3c137836c..a1c668156 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -22,7 +22,7 @@ defp reinject_object(data) do # TODO: # This will create a Create activity, which we need internally at the moment. - def fetch_object_from_id(id) do + def fetch_object_from_id(id, options \\ []) do if object = Object.get_cached_by_ap_id(id) do {:ok, object} else @@ -39,7 +39,7 @@ def fetch_object_from_id(id) do "object" => data }, {:containment, :ok} <- {:containment, Containment.contain_origin(id, params)}, - {:ok, activity} <- Transmogrifier.handle_incoming(params), + {:ok, activity} <- Transmogrifier.handle_incoming(params, options), {:object, _data, %Object{} = object} <- {:object, data, Object.normalize(activity, false)} do {:ok, object} @@ -69,8 +69,8 @@ def fetch_object_from_id(id) do end end - def fetch_object_from_id!(id) do - with {:ok, object} <- fetch_object_from_id(id) do + def fetch_object_from_id!(id, options \\ []) do + with {:ok, object} <- fetch_object_from_id(id, options) do object else _e -> diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index ca6940026..035d9486b 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.Federator import Ecto.Query @@ -22,20 +23,20 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do @doc """ Modifies an incoming AP object (mastodon format) to our internal format. """ - def fix_object(object) do + def fix_object(object, options \\ []) do object |> strip_internal_fields |> fix_actor |> fix_url |> fix_attachments |> fix_context - |> fix_in_reply_to + |> fix_in_reply_to(options) |> fix_emoji |> fix_tag |> fix_content_map |> fix_addressing |> fix_summary - |> fix_type + |> fix_type(options) end def fix_summary(%{"summary" => nil} = object) do @@ -152,7 +153,9 @@ def fix_actor(%{"attributedTo" => actor} = object) do def fix_in_reply_to(object, options \\ []) - def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, _options) + def fix_in_reply_to(object, options \\ []) + + def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) when not is_nil(in_reply_to) do in_reply_to_id = cond do @@ -170,24 +173,30 @@ def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, _options) "" end - case get_obj_helper(in_reply_to_id) do - {:ok, replied_object} -> - with %Activity{} = _activity <- - Activity.get_create_by_object_ap_id(replied_object.data["id"]) do - object - |> Map.put("inReplyTo", replied_object.data["id"]) - |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id) - |> Map.put("conversation", replied_object.data["context"] || object["conversation"]) - |> Map.put("context", replied_object.data["context"] || object["conversation"]) - else - e -> - Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") + object = Map.put(object, "inReplyToAtomUri", in_reply_to_id) + + if (options[:depth] || 1) <= Federator.max_replies_depth() do + case get_obj_helper(in_reply_to_id, options) do + {:ok, replied_object} -> + with %Activity{} = _activity <- + Activity.get_create_by_object_ap_id(replied_object.data["id"]) do object - end + |> Map.put("inReplyTo", replied_object.data["id"]) + |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id) + |> Map.put("conversation", replied_object.data["context"] || object["conversation"]) + |> Map.put("context", replied_object.data["context"] || object["conversation"]) + else + e -> + Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") + object + end - e -> - Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") - object + e -> + Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") + object + end + else + object end end @@ -369,9 +378,11 @@ defp get_follow_activity(follow_object, followed) do end end + def handle_incoming(data, options \\ []) + # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them # with nil ID. - def handle_incoming(%{"type" => "Flag", "object" => objects, "actor" => actor} = data) do + def handle_incoming(%{"type" => "Flag", "object" => objects, "actor" => actor} = data, _options) do with context <- data["context"] || Utils.generate_context_id(), content <- data["content"] || "", %User{} = actor <- User.get_cached_by_ap_id(actor), @@ -404,15 +415,19 @@ def handle_incoming(%{"type" => "Flag", "object" => objects, "actor" => actor} = end # disallow objects with bogus IDs - def handle_incoming(%{"id" => nil}), do: :error - def handle_incoming(%{"id" => ""}), do: :error + def handle_incoming(%{"id" => nil}, _options), do: :error + def handle_incoming(%{"id" => ""}, _options), do: :error # length of https:// = 8, should validate better, but good enough for now. - def handle_incoming(%{"id" => id}) when not (is_binary(id) and length(id) > 8), do: :error + def handle_incoming(%{"id" => id}, _options) when not (is_binary(id) and length(id) > 8), + do: :error # TODO: validate those with a Ecto scheme # - tags # - emoji - def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data) + def handle_incoming( + %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, + options + ) when objtype in ["Article", "Note", "Video", "Page", "Question", "Answer"] do actor = Containment.get_actor(data) @@ -422,7 +437,8 @@ def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = obj with nil <- Activity.get_create_by_object_ap_id(object["id"]), {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do - object = fix_object(data["object"]) + options = Keyword.put(options, :depth, (options[:depth] || 0) + 1) + object = fix_object(data["object"], options) params = %{ to: data["to"], @@ -447,7 +463,8 @@ def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = obj end def handle_incoming( - %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data + %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data, + _options ) do with %User{local: true} = followed <- User.get_cached_by_ap_id(followed), {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower), @@ -498,7 +515,8 @@ def handle_incoming( end def handle_incoming( - %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data + %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor), @@ -519,7 +537,8 @@ def handle_incoming( end def handle_incoming( - %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data + %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor), @@ -543,7 +562,8 @@ def handle_incoming( end def handle_incoming( - %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data + %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -556,7 +576,8 @@ def handle_incoming( end def handle_incoming( - %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data + %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -571,7 +592,8 @@ def handle_incoming( def handle_incoming( %{"type" => "Update", "object" => %{"type" => object_type} = object, "actor" => actor_id} = - data + data, + _options ) when object_type in ["Person", "Application", "Service", "Organization"] do with %User{ap_id: ^actor_id} = actor <- User.get_cached_by_ap_id(object["id"]) do @@ -609,7 +631,8 @@ def handle_incoming( # an error or a tombstone. This would allow us to verify that a deletion actually took # place. def handle_incoming( - %{"type" => "Delete", "object" => object_id, "actor" => actor, "id" => _id} = data + %{"type" => "Delete", "object" => object_id, "actor" => actor, "id" => _id} = data, + _options ) do object_id = Utils.get_ap_id(object_id) @@ -640,7 +663,8 @@ def handle_incoming( "object" => %{"type" => "Announce", "object" => object_id}, "actor" => _actor, "id" => id - } = data + } = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -658,7 +682,8 @@ def handle_incoming( "object" => %{"type" => "Follow", "object" => followed}, "actor" => follower, "id" => id - } = _data + } = _data, + _options ) do with %User{local: true} = followed <- User.get_cached_by_ap_id(followed), {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower), @@ -676,7 +701,8 @@ def handle_incoming( "object" => %{"type" => "Block", "object" => blocked}, "actor" => blocker, "id" => id - } = _data + } = _data, + _options ) do with %User{local: true} = blocked <- User.get_cached_by_ap_id(blocked), {:ok, %User{} = blocker} <- User.get_or_fetch_by_ap_id(blocker), @@ -689,7 +715,8 @@ def handle_incoming( end def handle_incoming( - %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data + %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data, + _options ) do with %User{local: true} = blocked = User.get_cached_by_ap_id(blocked), {:ok, %User{} = blocker} = User.get_or_fetch_by_ap_id(blocker), @@ -708,7 +735,8 @@ def handle_incoming( "object" => %{"type" => "Like", "object" => object_id}, "actor" => _actor, "id" => id - } = data + } = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -720,10 +748,10 @@ def handle_incoming( end end - def handle_incoming(_), do: :error + def handle_incoming(_, _), do: :error - def get_obj_helper(id) do - if object = Object.normalize(id), do: {:ok, object}, else: nil + def get_obj_helper(id, options \\ []) do + if object = Object.normalize(id, true, options), do: {:ok, object}, else: nil end def set_reply_to_uri(%{"inReplyTo" => in_reply_to} = object) when is_binary(in_reply_to) do diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index f4c9fe284..7c13ff323 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -22,6 +22,12 @@ def init do refresh_subscriptions() end + @max_replies_depth 100 + + @doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)" + # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength + def max_replies_depth, do: @max_replies_depth + # Client API def incoming_doc(doc) do diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index 5b4befbb0..5dbf35155 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Federator alias Pleroma.Web.OStatus alias Pleroma.Web.XML @@ -88,14 +89,15 @@ def add_external_url(note, entry) do Map.put(note, "external_url", url) end - def fetch_replied_to_activity(entry, in_reply_to) do + def fetch_replied_to_activity(entry, in_reply_to, options \\ []) do with %Activity{} = activity <- Activity.get_create_by_object_ap_id(in_reply_to) do activity else _e -> - with in_reply_to_href when not is_nil(in_reply_to_href) <- + with true <- (options[:depth] || 1) <= Federator.max_replies_depth(), + in_reply_to_href when not is_nil(in_reply_to_href) <- XML.string_from_xpath("//thr:in-reply-to[1]/@href", entry), - {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href) do + {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href, options) do activity else _e -> nil @@ -104,7 +106,7 @@ def fetch_replied_to_activity(entry, in_reply_to) do end # TODO: Clean this up a bit. - def handle_note(entry, doc \\ nil) do + def handle_note(entry, doc \\ nil, options \\ []) do with id <- XML.string_from_xpath("//id", entry), activity when is_nil(activity) <- Activity.get_create_by_object_ap_id_with_object(id), [author] <- :xmerl_xpath.string('//author[1]', doc), @@ -112,7 +114,8 @@ def handle_note(entry, doc \\ nil) do content_html <- OStatus.get_content(entry), cw <- OStatus.get_cw(entry), in_reply_to <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry), - in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to), + options <- Keyword.put(options, :depth, (options[:depth] || 0) + 1), + in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to, options), in_reply_to_object <- (in_reply_to_activity && Object.normalize(in_reply_to_activity)) || nil, in_reply_to <- (in_reply_to_object && in_reply_to_object.data["id"]) || in_reply_to, diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 56975926f..331cbc0b7 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -54,7 +54,7 @@ def remote_follow_path do "#{Web.base_url()}/ostatus_subscribe?acct={uri}" end - def handle_incoming(xml_string) do + def handle_incoming(xml_string, options \\ []) do with doc when doc != :error <- parse_document(xml_string) do with {:ok, actor_user} <- find_make_or_update_actor(doc), do: Pleroma.Instances.set_reachable(actor_user.ap_id) @@ -91,10 +91,12 @@ def handle_incoming(xml_string) do _ -> case object_type do 'http://activitystrea.ms/schema/1.0/note' -> - with {:ok, activity} <- NoteHandler.handle_note(entry, doc), do: activity + with {:ok, activity} <- NoteHandler.handle_note(entry, doc, options), + do: activity 'http://activitystrea.ms/schema/1.0/comment' -> - with {:ok, activity} <- NoteHandler.handle_note(entry, doc), do: activity + with {:ok, activity} <- NoteHandler.handle_note(entry, doc, options), + do: activity _ -> Logger.error("Couldn't parse incoming document") @@ -366,7 +368,7 @@ def get_atom_url(body) do end end - def fetch_activity_from_atom_url(url) do + def fetch_activity_from_atom_url(url, options \\ []) do with true <- String.starts_with?(url, "http"), {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get( @@ -374,7 +376,7 @@ def fetch_activity_from_atom_url(url) do [{:Accept, "application/atom+xml"}] ) do Logger.debug("Got document from #{url}, handling...") - handle_incoming(body) + handle_incoming(body, options) else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}") @@ -382,13 +384,13 @@ def fetch_activity_from_atom_url(url) do end end - def fetch_activity_from_html_url(url) do + def fetch_activity_from_html_url(url, options \\ []) do Logger.debug("Trying to fetch #{url}") with true <- String.starts_with?(url, "http"), {:ok, %{body: body}} <- HTTP.get(url, []), {:ok, atom_url} <- get_atom_url(body) do - fetch_activity_from_atom_url(atom_url) + fetch_activity_from_atom_url(atom_url, options) else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}") @@ -396,11 +398,11 @@ def fetch_activity_from_html_url(url) do end end - def fetch_activity_from_url(url) do - with {:ok, [_ | _] = activities} <- fetch_activity_from_atom_url(url) do + def fetch_activity_from_url(url, options \\ []) do + with {:ok, [_ | _] = activities} <- fetch_activity_from_atom_url(url, options) do {:ok, activities} else - _e -> fetch_activity_from_html_url(url) + _e -> fetch_activity_from_html_url(url, options) end rescue e -> diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index b8b998acd..5d8a6a3b6 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -11,12 +11,13 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.CommonAPI alias Pleroma.Web.OStatus alias Pleroma.Web.Websub.WebsubClientSubscription + import Mock import Pleroma.Factory import ExUnit.CaptureLog - alias Pleroma.Web.CommonAPI setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -46,12 +47,10 @@ test "it fetches replied-to activities if we don't have them" do data["object"] |> Map.put("inReplyTo", "https://shitposter.club/notice/2827873") - data = - data - |> Map.put("object", object) - + data = Map.put(data, "object", object) {:ok, returned_activity} = Transmogrifier.handle_incoming(data) - returned_object = Object.normalize(returned_activity.data["object"]) + + returned_object = Object.normalize(returned_activity.data["object"], false) assert activity = Activity.get_create_by_object_ap_id( @@ -61,6 +60,32 @@ test "it fetches replied-to activities if we don't have them" do assert returned_object.data["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873" end + test "it does not fetch replied-to activities beyond max_replies_depth" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + + object = + data["object"] + |> Map.put("inReplyTo", "https://shitposter.club/notice/2827873") + + data = Map.put(data, "object", object) + + with_mock Pleroma.Web.Federator, + max_replies_depth: fn -> 0 end do + {:ok, returned_activity} = Transmogrifier.handle_incoming(data) + + returned_object = Object.normalize(returned_activity.data["object"], false) + + refute Activity.get_create_by_object_ap_id( + "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" + ) + + assert returned_object.data["inReplyToAtomUri"] == + "https://shitposter.club/notice/2827873" + end + end + test "it does not crash if the object in inReplyTo can't be fetched" do data = File.read!("test/fixtures/mastodon-post-activity.json") diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 41cf188ca..9318292a6 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -11,8 +11,10 @@ defmodule Pleroma.Web.OStatusTest do alias Pleroma.User alias Pleroma.Web.OStatus alias Pleroma.Web.XML - import Pleroma.Factory + import ExUnit.CaptureLog + import Mock + import Pleroma.Factory setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -266,10 +268,13 @@ test "handle incoming favorites with locally available object - GS, websub" do assert favorited_activity.local end - test "handle incoming replies" do + test_with_mock "handle incoming replies, fetching replied-to activities if we don't have them", + OStatus, + [:passthrough], + [] do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) - object = Object.normalize(activity.data["object"]) + object = Object.normalize(activity.data["object"], false) assert activity.data["type"] == "Create" assert object.data["type"] == "Note" @@ -282,6 +287,23 @@ test "handle incoming replies" do assert object.data["id"] == "tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] + + assert called(OStatus.fetch_activity_from_url(object.data["inReplyTo"], :_)) + end + + test_with_mock "handle incoming replies, not fetching replied-to activities beyond max_replies_depth", + OStatus, + [:passthrough], + [] do + incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") + + with_mock Pleroma.Web.Federator, + max_replies_depth: fn -> 0 end do + {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"], false) + + refute called(OStatus.fetch_activity_from_url(object.data["inReplyTo"], :_)) + end end test "handle incoming follows" do -- cgit v1.2.3 From bc6d1f9ed951f690544d80b411b35c07d9fd7793 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 30 Jun 2019 15:58:50 +0300 Subject: [#161] Refactoring, documentation. --- CHANGELOG.md | 2 +- config/config.exs | 1 + lib/pleroma/web/activity_pub/transmogrifier.ex | 6 ++---- lib/pleroma/web/federator/federator.ex | 12 +++++++++--- lib/pleroma/web/ostatus/handlers/note_handler.ex | 2 +- test/web/activity_pub/transmogrifier_test.exs | 2 +- test/web/ostatus/ostatus_test.exs | 2 +- 7 files changed, 16 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4effcf887..f5289ea6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Not being able to access the Mastodon FE login page on private instances - MRF: ensure that subdomain_match calls are case-insensitive - Fix internal server error when using the healthcheck API. -- `federation_incoming_replies_max_depth` option being ignored in certain cases. ### Added - **Breaking:** MRF describe API, which adds support for exposing configuration information about MRF policies to NodeInfo. @@ -24,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Support for wildcard domains in user domain blocks setting. - Configuration: `quarantined_instances` support wildcard domains. - Mix Tasks: `mix pleroma.database fix_likes_collections` +- Federation: Support for restricting max. reply-to depth on fetching ### Removed - Federation: Remove `likes` from objects. diff --git a/config/config.exs b/config/config.exs index 66a8c18ea..7e901a1f5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -220,6 +220,7 @@ }, registrations_open: true, federating: true, + federation_incoming_replies_max_depth: 100, federation_reachability_timeout_days: 7, federation_publisher_modules: [ Pleroma.Web.ActivityPub.Publisher, diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 035d9486b..2a66b2162 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -153,8 +153,6 @@ def fix_actor(%{"attributedTo" => actor} = object) do def fix_in_reply_to(object, options \\ []) - def fix_in_reply_to(object, options \\ []) - def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) when not is_nil(in_reply_to) do in_reply_to_id = @@ -175,7 +173,7 @@ def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) object = Map.put(object, "inReplyToAtomUri", in_reply_to_id) - if (options[:depth] || 1) <= Federator.max_replies_depth() do + if Federator.allowed_incoming_reply_depth?(options[:depth]) do case get_obj_helper(in_reply_to_id, options) do {:ok, replied_object} -> with %Activity{} = _activity <- @@ -339,7 +337,7 @@ def fix_type(%{"inReplyTo" => reply_id, "name" => _} = object, options) when is_binary(reply_id) do reply = with true <- Federator.allowed_incoming_reply_depth?(options[:depth]), - {:ok, object} <- get_obj_helper(reply_id) do + {:ok, object} <- get_obj_helper(reply_id, options) do object end diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 7c13ff323..f4f9e83e0 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -22,11 +22,17 @@ def init do refresh_subscriptions() end - @max_replies_depth 100 - @doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)" # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength - def max_replies_depth, do: @max_replies_depth + def allowed_incoming_reply_depth?(depth) do + max_replies_depth = Pleroma.Config.get([:instance, :federation_incoming_replies_max_depth]) + + if max_replies_depth do + (depth || 1) <= max_replies_depth + else + true + end + end # Client API diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index 5dbf35155..c8c1c905e 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -94,7 +94,7 @@ def fetch_replied_to_activity(entry, in_reply_to, options \\ []) do activity else _e -> - with true <- (options[:depth] || 1) <= Federator.max_replies_depth(), + with true <- Federator.allowed_incoming_reply_depth?(options[:depth]), in_reply_to_href when not is_nil(in_reply_to_href) <- XML.string_from_xpath("//thr:in-reply-to[1]/@href", entry), {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href, options) do diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 5d8a6a3b6..9c1584dbe 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -72,7 +72,7 @@ test "it does not fetch replied-to activities beyond max_replies_depth" do data = Map.put(data, "object", object) with_mock Pleroma.Web.Federator, - max_replies_depth: fn -> 0 end do + allowed_incoming_reply_depth?: fn _ -> false end do {:ok, returned_activity} = Transmogrifier.handle_incoming(data) returned_object = Object.normalize(returned_activity.data["object"], false) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 9318292a6..ff9bb6351 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -298,7 +298,7 @@ test "handle incoming favorites with locally available object - GS, websub" do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") with_mock Pleroma.Web.Federator, - max_replies_depth: fn -> 0 end do + allowed_incoming_reply_depth?: fn _ -> false end do {:ok, [activity]} = OStatus.handle_incoming(incoming) object = Object.normalize(activity.data["object"], false) -- cgit v1.2.3 From 9fa2011c17926ee7bdfecdcd7a35649898420055 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 6 Jul 2019 07:18:26 +0000 Subject: Apply suggestion to CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5289ea6d..843e4fc21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Support for wildcard domains in user domain blocks setting. - Configuration: `quarantined_instances` support wildcard domains. - Mix Tasks: `mix pleroma.database fix_likes_collections` -- Federation: Support for restricting max. reply-to depth on fetching +- Configuration: `federation_incoming_replies_max_depth` option ### Removed - Federation: Remove `likes` from objects. -- cgit v1.2.3 From dcb4711e92382b88326260d7e1951eb66941ed1b Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 14 Aug 2019 01:47:26 +0000 Subject: mix: update version to 1.0.5 --- CHANGELOG.md | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 843e4fc21..f84815afd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [1.0.5] - 2019-08-?? +## [1.0.5] - 2019-08-13 ### Fixed - Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set - Mastodon API: `muted` in the Status entity, using author's account to determine if the thread was muted diff --git a/mix.exs b/mix.exs index 0673e4581..58d7d3b67 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do def project do [ app: :pleroma, - version: version("1.0.4"), + version: version("1.0.5"), elixir: "~> 1.7", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), -- cgit v1.2.3