Merge remote-tracking branch 'remotes/origin/develop' into 1505-threads-federation

# Conflicts:
#	CHANGELOG.md
#	config/config.exs

17 files changed, 210 insertions, 59 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0ab69d8b..2ab098591 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -74,6 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - User settings: Add _This account is a_ option.
 - A new users admin digest email
 - OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`).
+- Add an option `authorized_fetch_mode` to require HTTP signatures for AP fetches.
 - ActivityPub: support for `replies` collection (output for outgoing federation & fetching on incoming federation).
 <details>
   <summary>API Changes</summary>
@@ -116,6 +117,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `feed.logo` option for tag feed.
 - Tag feed: `/tags/:tag.rss` - list public statuses by hashtag.
 - Mastodon API: Add `reacted` property to `emoji_reactions`
+- Pleroma API: Add reactions for a single emoji.
 - ActivityPub: `[:activitypub, :note_replies_output_limit]` setting sets the number of note self-replies to output on outgoing federation.
 </details>
 
diff --git a/config/config.exs b/config/config.exs
index c5a56b14d..a92130ec8 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -327,7 +327,8 @@ config :pleroma, :activitypub,
   outgoing_blocks: true,
   follow_handshake_timeout: 500,
   note_replies_output_limit: 5,
-  sign_object_fetches: true
+  sign_object_fetches: true,
+  authorized_fetch_mode: false
 
 config :pleroma, :streamer,
   workers: 3,
@@ -618,6 +619,8 @@ config :pleroma, :modules, runtime_dir: "instance/modules"
 
 config :pleroma, configurable_from_database: false
 
+config :pleroma, Pleroma.Repo, parameters: [gin_fuzzy_search_limit: "500"]
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"
diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md
index 07e0af5e5..761d5c69c 100644
--- a/docs/API/pleroma_api.md
+++ b/docs/API/pleroma_api.md
@@ -459,3 +459,16 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
   {"name": "☕", "count": 1, "me": false, "accounts": [{"id" => "abc..."}]}
 ]
 ```
+
+## `GET /api/v1/pleroma/statuses/:id/reactions/:emoji`
+### Get an object of emoji to account mappings with accounts that reacted to the post for a specific emoji`
+* Method: `GET`
+* Authentication: optional
+* Params: None
+* Response: JSON, a list of emoji/account list tuples
+* Example Response:
+```json
+[
+  {"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]}
+]
+```
diff --git a/docs/administration/backup.md b/docs/administration/backup.md
index 685c45128..692aa7368 100644
--- a/docs/administration/backup.md
+++ b/docs/administration/backup.md
@@ -18,7 +18,9 @@
 6. Run `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>`
 7. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
 8. Restart the Pleroma service.
-
+9. After you've restarted Pleroma, you will notice that postgres will take up more cpu resources than usual. A lot in fact. To fix this you must do a VACUUM ANLAYZE. This can also be done while the instance is still running like so: 
+   $ sudo -u postgres psql pleroma_database_name
+   pleroma=# VACUUM ANALYZE;
 [^1]: Prefix with `MIX_ENV=prod` to run it using the production config file.
 
 ## Remove
diff --git a/docs/administration/updating.md b/docs/administration/updating.md
index 84e6ef18d..2a08dac1f 100644
--- a/docs/administration/updating.md
+++ b/docs/administration/updating.md
@@ -1,4 +1,21 @@
 # Updating your instance
+
+You should **always check the release notes/changelog** in case there are config deprecations, special update special update steps, etc.
+
+Besides that, doing the following is generally enough:
+
+## For OTP installations
+
+```sh
+# Download the new release
+su pleroma -s $SHELL -lc "./bin/pleroma_ctl update"
+
+# Migrate the database, you are advised to stop the instance before doing that
+su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
+```
+
+## For from source installations (using git)
+
 1. Go to the working directory of Pleroma (default is `/opt/pleroma`)
 2. Run `git pull`. This pulls the latest changes from upstream.
 3. Run `mix deps.get`. This pulls in any new dependencies.
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index c9559fe85..ac55a0b32 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -143,10 +143,11 @@ config :pleroma, :mrf_user_allowlist,
   * `:reject` rejects the message entirely
 
 ### :activitypub
-* ``unfollow_blocked``: Whether blocks result in people getting unfollowed
-* ``outgoing_blocks``: Whether to federate blocks to other instances
-* ``deny_follow_blocked``: Whether to disallow following an account that has blocked the user in question
-* ``sign_object_fetches``: Sign object fetches with HTTP signatures
+* `unfollow_blocked`: Whether blocks result in people getting unfollowed
+* `outgoing_blocks`: Whether to federate blocks to other instances
+* `deny_follow_blocked`: Whether to disallow following an account that has blocked the user in question
+* `sign_object_fetches`: Sign object fetches with HTTP signatures
+* `authorized_fetch_mode`: Require HTTP signatures for AP fetches
 
 ### :fetch_initial_posts
 * `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts
diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md
index 93230806c..aab5197a2 100644
--- a/docs/installation/otp_en.md
+++ b/docs/installation/otp_en.md
@@ -259,19 +259,14 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl user new joeuser joeuser@sld.tld --a
 ```
 This will create an account withe the username of 'joeuser' with the email address of joeuser@sld.tld, and set that user's account as an admin. This will result in a link that you can paste into the browser, which logs you in and enables you to set the password.
 
-### Updating
-Generally, doing the following is enough:
-```sh
-# Download the new release
-su pleroma -s $SHELL -lc "./bin/pleroma_ctl update"
-
-# Migrate the database, you are advised to stop the instance before doing that
-su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
-```
-But you should **always check the release notes/changelog** in case there are config deprecations, special update steps, etc.
-
 ## Further reading
 
 * [Backup your instance](../administration/backup.md)
 * [Hardening your instance](../configuration/hardening.md)
 * [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
+* [Updating your instance](../administration/updating.md)
+
+## Questions
+
+Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
+
diff --git a/lib/pleroma/plugs/http_signature.ex b/lib/pleroma/plugs/http_signature.ex
index 23d22a712..477a5b578 100644
--- a/lib/pleroma/plugs/http_signature.ex
+++ b/lib/pleroma/plugs/http_signature.ex
@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
   import Plug.Conn
+  import Phoenix.Controller, only: [get_format: 1, text: 2]
   require Logger
 
   def init(options) do
@@ -15,25 +16,27 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
   end
 
   def call(conn, _opts) do
-    headers = get_req_header(conn, "signature")
-    signature = Enum.at(headers, 0)
+    if get_format(conn) == "activity+json" do
+      conn
+      |> maybe_assign_valid_signature()
+      |> maybe_require_signature()
+    else
+      conn
+    end
+  end
 
-    if signature do
+  defp maybe_assign_valid_signature(conn) do
+    if has_signature_header?(conn) do
       # set (request-target) header to the appropriate value
       # we also replace the digest header with the one we computed
-      conn =
-        conn
-        |> put_req_header(
-          "(request-target)",
-          String.downcase("#{conn.method}") <> " #{conn.request_path}"
-        )
+      request_target = String.downcase("#{conn.method}") <> " #{conn.request_path}"
 
       conn =
-        if conn.assigns[:digest] do
-          conn
-          |> put_req_header("digest", conn.assigns[:digest])
-        else
-          conn
+        conn
+        |> put_req_header("(request-target)", request_target)
+        |> case do
+          %{assigns: %{digest: digest}} = conn -> put_req_header(conn, "digest", digest)
+          conn -> conn
         end
 
       assign(conn, :valid_signature, HTTPSignatures.validate_conn(conn))
@@ -42,4 +45,21 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
       conn
     end
   end
+
+  defp has_signature_header?(conn) do
+    conn |> get_req_header("signature") |> Enum.at(0, false)
+  end
+
+  defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn
+
+  defp maybe_require_signature(conn) do
+    if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do
+      conn
+      |> put_status(:unauthorized)
+      |> text("Request not signed")
+      |> halt()
+    else
+      conn
+    end
+  end
 end
diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
index 108e48438..f86a068fb 100644
--- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
@@ -41,24 +41,29 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do
 
   plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug)
 
-  def emoji_reactions_by(%{assigns: %{user: user}} = conn, %{"id" => activity_id}) do
+  def emoji_reactions_by(%{assigns: %{user: user}} = conn, %{"id" => activity_id} = params) do
     with %Activity{} = activity <- Activity.get_by_id_with_object(activity_id),
          %Object{data: %{"reactions" => emoji_reactions}} when is_list(emoji_reactions) <-
            Object.normalize(activity) do
       reactions =
         emoji_reactions
         |> Enum.map(fn [emoji, user_ap_ids] ->
-          users =
-            Enum.map(user_ap_ids, &User.get_cached_by_ap_id/1)
-            |> Enum.filter(& &1)
-
-          %{
-            name: emoji,
-            count: length(users),
-            accounts: AccountView.render("index.json", %{users: users, for: user, as: :user}),
-            me: !!(user && user.ap_id in user_ap_ids)
-          }
+          if params["emoji"] && params["emoji"] != emoji do
+            nil
+          else
+            users =
+              Enum.map(user_ap_ids, &User.get_cached_by_ap_id/1)
+              |> Enum.filter(& &1)
+
+            %{
+              name: emoji,
+              count: length(users),
+              accounts: AccountView.render("index.json", %{users: users, for: user, as: :user}),
+              me: !!(user && user.ap_id in user_ap_ids)
+            }
+          end
         end)
+        |> Enum.filter(& &1)
 
       conn
       |> json(reactions)
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 897215698..9bfe86704 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -271,6 +271,7 @@ defmodule Pleroma.Web.Router do
   scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do
     pipe_through(:api)
 
+    get("/statuses/:id/reactions/:emoji", PleromaAPIController, :emoji_reactions_by)
     get("/statuses/:id/reactions", PleromaAPIController, :emoji_reactions_by)
   end
 
diff --git a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex
index e0d4d5632..fbf31c7eb 100644
--- a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex
+++ b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex
@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do
@@ -69,7 +69,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do
   def do_follow(%{assigns: %{user: %User{} = user}} = conn, %{"user" => %{"id" => id}}) do
     with {:fetch_user, %User{} = followee} <- {:fetch_user, User.get_cached_by_id(id)},
          {:ok, _, _, _} <- CommonAPI.follow(user, followee) do
-      render(conn, "followed.html", %{error: false})
+      redirect(conn, to: "/users/#{followee.id}")
     else
       error ->
         handle_follow_error(conn, error)
@@ -80,7 +80,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do
     with {:fetch_user, %User{} = followee} <- {:fetch_user, User.get_cached_by_id(id)},
          {_, {:ok, user}, _} <- {:auth, Authenticator.get_user(conn), followee},
          {:ok, _, _, _} <- CommonAPI.follow(user, followee) do
-      render(conn, "followed.html", %{error: false})
+      redirect(conn, to: "/users/#{followee.id}")
     else
       error ->
         handle_follow_error(conn, error)
diff --git a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs
index 99102117f..c618ea381 100644
--- a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs
+++ b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs
@@ -17,7 +17,11 @@ defmodule Pleroma.Repo.Migrations.MigrateOldBookmarks do
     Repo.stream(query)
     |> Enum.each(fn %{id: user_id, bookmarks: bookmarks} ->
       Enum.each(bookmarks, fn ap_id ->
-        activity = Activity.get_create_by_object_ap_id(ap_id)
+        activity =
+          ap_id
+          |> Activity.create_by_object_ap_id()
+          |> Repo.one()
+
         unless is_nil(activity), do: {:ok, _} = Bookmark.create(user_id, activity.id)
       end)
     end)
diff --git a/priv/repo/migrations/20190710125158_add_following_address_from_source_data.exs b/priv/repo/migrations/20190710125158_add_following_address_from_source_data.exs
index a5170d521..44f9891b1 100644
--- a/priv/repo/migrations/20190710125158_add_following_address_from_source_data.exs
+++ b/priv/repo/migrations/20190710125158_add_following_address_from_source_data.exs
@@ -1,7 +1,8 @@
 defmodule Pleroma.Repo.Migrations.AddFollowingAddressFromSourceData do
-  use Ecto.Migration
-  import Ecto.Query
   alias Pleroma.User
+  import Ecto.Query
+  require Logger
+  use Ecto.Migration
 
   def change do
     query =
@@ -19,6 +20,9 @@ defmodule Pleroma.Repo.Migrations.AddFollowingAddressFromSourceData do
           :following_address
         ])
         |> Pleroma.Repo.update()
+
+      user ->
+        Logger.warn("User #{user.id} / #{user.nickname} does not seem to have source_data")
     end)
   end
 end
diff --git a/priv/repo/migrations/20190711042024_copy_muted_to_muted_notifications.exs b/priv/repo/migrations/20190711042024_copy_muted_to_muted_notifications.exs
index fc9bf70ba..bbd502044 100644
--- a/priv/repo/migrations/20190711042024_copy_muted_to_muted_notifications.exs
+++ b/priv/repo/migrations/20190711042024_copy_muted_to_muted_notifications.exs
@@ -2,6 +2,8 @@ defmodule Pleroma.Repo.Migrations.CopyMutedToMutedNotifications do
   use Ecto.Migration
 
   def change do
+    execute("update users set info = '{}' where info is null")
+
     execute(
       "update users set info = safe_jsonb_set(info, '{muted_notifications}', info->'mutes', true) where local = true"
     )
diff --git a/test/plugs/http_signature_plug_test.exs b/test/plugs/http_signature_plug_test.exs
index d8ace36da..55e8bafc0 100644
--- a/test/plugs/http_signature_plug_test.exs
+++ b/test/plugs/http_signature_plug_test.exs
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do
   alias Pleroma.Web.Plugs.HTTPSignaturePlug
 
   import Plug.Conn
+  import Phoenix.Controller, only: [put_format: 2]
   import Mock
 
   test "it call HTTPSignatures to check validity if the actor sighed it" do
@@ -20,10 +21,69 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do
           "signature",
           "keyId=\"http://mastodon.example.org/users/admin#main-key"
         )
+        |> put_format("activity+json")
         |> HTTPSignaturePlug.call(%{})
 
       assert conn.assigns.valid_signature == true
+      assert conn.halted == false
       assert called(HTTPSignatures.validate_conn(:_))
     end
   end
+
+  describe "requires a signature when `authorized_fetch_mode` is enabled" do
+    setup do
+      Pleroma.Config.put([:activitypub, :authorized_fetch_mode], true)
+
+      on_exit(fn ->
+        Pleroma.Config.put([:activitypub, :authorized_fetch_mode], false)
+      end)
+
+      params = %{"actor" => "http://mastodon.example.org/users/admin"}
+      conn = build_conn(:get, "/doesntmattter", params) |> put_format("activity+json")
+
+      [conn: conn]
+    end
+
+    test "when signature header is present", %{conn: conn} do
+      with_mock HTTPSignatures, validate_conn: fn _ -> false end do
+        conn =
+          conn
+          |> put_req_header(
+            "signature",
+            "keyId=\"http://mastodon.example.org/users/admin#main-key"
+          )
+          |> HTTPSignaturePlug.call(%{})
+
+        assert conn.assigns.valid_signature == false
+        assert conn.halted == true
+        assert conn.status == 401
+        assert conn.state == :sent
+        assert conn.resp_body == "Request not signed"
+        assert called(HTTPSignatures.validate_conn(:_))
+      end
+
+      with_mock HTTPSignatures, validate_conn: fn _ -> true end do
+        conn =
+          conn
+          |> put_req_header(
+            "signature",
+            "keyId=\"http://mastodon.example.org/users/admin#main-key"
+          )
+          |> HTTPSignaturePlug.call(%{})
+
+        assert conn.assigns.valid_signature == true
+        assert conn.halted == false
+        assert called(HTTPSignatures.validate_conn(:_))
+      end
+    end
+
+    test "halts the connection when `signature` header is not present", %{conn: conn} do
+      conn = HTTPSignaturePlug.call(conn, %{})
+      assert conn.assigns[:valid_signature] == nil
+      assert conn.halted == true
+      assert conn.status == 401
+      assert conn.state == :sent
+      assert conn.resp_body == "Request not signed"
+    end
+  end
 end
diff --git a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs
index 36868db38..164cfa695 100644
--- a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs
+++ b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs
@@ -96,6 +96,32 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do
              result
   end
 
+  test "GET /api/v1/pleroma/statuses/:id/reactions/:emoji", %{conn: conn} do
+    user = insert(:user)
+    other_user = insert(:user)
+
+    {:ok, activity} = CommonAPI.post(user, %{"status" => "#cofe"})
+
+    result =
+      conn
+      |> get("/api/v1/pleroma/statuses/#{activity.id}/reactions/🎅")
+      |> json_response(200)
+
+    assert result == []
+
+    {:ok, _, _} = CommonAPI.react_with_emoji(activity.id, other_user, "🎅")
+    {:ok, _, _} = CommonAPI.react_with_emoji(activity.id, other_user, "☕")
+
+    result =
+      conn
+      |> get("/api/v1/pleroma/statuses/#{activity.id}/reactions/🎅")
+      |> json_response(200)
+
+    [%{"name" => "🎅", "count" => 1, "accounts" => [represented_user], "me" => false}] = result
+
+    assert represented_user["id"] == other_user.id
+  end
+
   test "/api/v1/pleroma/conversations/:id" do
     user = insert(:user)
     %{user: other_user, conn: conn} = oauth_access(["read:statuses"])
diff --git a/test/web/twitter_api/remote_follow_controller_test.exs b/test/web/twitter_api/remote_follow_controller_test.exs
index 444949375..80a42989d 100644
--- a/test/web/twitter_api/remote_follow_controller_test.exs
+++ b/test/web/twitter_api/remote_follow_controller_test.exs
@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do
@@ -92,15 +92,13 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do
       user = insert(:user)
       user2 = insert(:user)
 
-      response =
+      conn =
         conn
         |> assign(:user, user)
         |> assign(:token, insert(:oauth_token, user: user, scopes: ["write:follows"]))
         |> post(remote_follow_path(conn, :do_follow), %{"user" => %{"id" => user2.id}})
-        |> response(200)
 
-      assert response =~ "Account followed!"
-      assert user2.follower_address in User.following(user)
+      assert redirected_to(conn) == "/users/#{user2.id}"
     end
 
     test "returns error when user is deactivated", %{conn: conn} do
@@ -149,14 +147,13 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do
       user2 = insert(:user)
       {:ok, _, _, _} = CommonAPI.follow(user, user2)
 
-      response =
+      conn =
         conn
         |> assign(:user, refresh_record(user))
         |> assign(:token, insert(:oauth_token, user: user, scopes: ["write:follows"]))
         |> post(remote_follow_path(conn, :do_follow), %{"user" => %{"id" => user2.id}})
-        |> response(200)
 
-      assert response =~ "Account followed!"
+      assert redirected_to(conn) == "/users/#{user2.id}"
     end
   end
 
@@ -165,14 +162,13 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do
       user = insert(:user)
       user2 = insert(:user)
 
-      response =
+      conn =
         conn
         |> post(remote_follow_path(conn, :do_follow), %{
           "authorization" => %{"name" => user.nickname, "password" => "test", "id" => user2.id}
         })
-        |> response(200)
 
-      assert response =~ "Account followed!"
+      assert redirected_to(conn) == "/users/#{user2.id}"
       assert user2.follower_address in User.following(user)
     end