Merge branch 'develop' into 'feat/floki-fast-html-2'

# Conflicts:
#   config/config.exs

10 files changed, 138 insertions, 87 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 572f9e84b..a8e80eb3c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [unreleased]
 
 ### Changed
+- **Breaking:** Added the ObjectAgePolicy to the default set of MRFs. This will delist and strip the follower collection of any message received that is older than 7 days. This will stop users from seeing very old messages in the timelines. The messages can still be viewed on the user's page and in conversations. They also still trigger notifications.
 - **Breaking:** Elixir >=1.9 is now required (was >= 1.8)
 - **Breaking:** Configuration: `:auto_linker, :opts` moved to `:pleroma, Pleroma.Formatter`. Old config namespace is deprecated.
 - In Conversations, return only direct messages as `last_status`
@@ -15,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `:instance, rewrite_policy` moved to `:mrf, policies`, `:instance, :mrf_transparency` moved to `:mrf, :transparency`, `:instance, :mrf_transparency_exclusions` moved to `:mrf, :transparency_exclusions`. Old config namespace is deprecated.
 - Configuration: `:media_proxy, whitelist` format changed to host with scheme (e.g. `http://example.com` instead of `example.com`). Domain format is deprecated.
 - **Breaking:** Configuration: `:instance, welcome_user_nickname` moved to `:welcome, :direct_message, :sender_nickname`, `:instance, :welcome_message` moved to `:welcome, :direct_message, :message`. Old config namespace is deprecated.
+- **Breaking:** LDAP: Fallback to local database authentication has been removed for security reasons and lack of a mechanism to ensure the passwords are synchronized when LDAP passwords are updated.
 
 <details>
   <summary>API Changes</summary>
diff --git a/config/config.exs b/config/config.exs
index 78f3232e6..eb85a6ed4 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -515,7 +515,13 @@ config :pleroma, Pleroma.User,
     "user-search",
     "user_exists",
     "users",
-    "web"
+    "web",
+    "verify_credentials",
+    "update_credentials",
+    "relationships",
+    "search",
+    "confirmation_resend",
+    "mfa"
   ],
   email_blacklist: []
 
@@ -739,6 +745,8 @@ config :pleroma, :instances_favicons, enabled: false
 
 config :floki, :html_parser, Floki.HTMLParser.FastHtml
 
+config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"
diff --git a/docs/administration/CLI_tasks/database.md b/docs/administration/CLI_tasks/database.md
index 647f6f274..64dd66c0c 100644
--- a/docs/administration/CLI_tasks/database.md
+++ b/docs/administration/CLI_tasks/database.md
@@ -97,4 +97,14 @@ but should only be run if necessary. **It is safe to cancel this.**
 
 ```sh tab="From Source"
 mix pleroma.database vacuum full
-```
\ No newline at end of file
+```
+
+## Add expiration to all local statuses
+
+```sh tab="OTP"
+./bin/pleroma_ctl database ensure_expiration
+```
+
+```sh tab="From Source"
+mix pleroma.database ensure_expiration
+```
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index f23cf4fe4..ca587af8e 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -858,9 +858,6 @@ Warning: it's discouraged to use this feature because of the associated security
 
 ### :auth
 
-* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator.
-* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication.
-
 Authentication / authorization settings.
 
 * `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`.
@@ -890,6 +887,9 @@ Pleroma account will be created with the same name as the LDAP user name.
 * `base`: LDAP base, e.g. "dc=example,dc=com"
 * `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base"
 
+Note, if your LDAP server is an Active Directory server the correct value is commonly `uid: "cn"`, but if you use an
+OpenLDAP server the value may be `uid: "uid"`.
+
 ### OAuth consumer mode
 
 OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.).
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index 82e2abdcb..d57e59b11 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -10,6 +10,7 @@ defmodule Mix.Tasks.Pleroma.Database do
   alias Pleroma.User
   require Logger
   require Pleroma.Constants
+  import Ecto.Query
   import Mix.Pleroma
   use Mix.Task
 
@@ -53,8 +54,6 @@ defmodule Mix.Tasks.Pleroma.Database do
   end
 
   def run(["prune_objects" | args]) do
-    import Ecto.Query
-
     {options, [], []} =
       OptionParser.parse(
         args,
@@ -94,8 +93,6 @@ defmodule Mix.Tasks.Pleroma.Database do
   end
 
   def run(["fix_likes_collections"]) do
-    import Ecto.Query
-
     start_pleroma()
 
     from(object in Object,
@@ -130,4 +127,23 @@ defmodule Mix.Tasks.Pleroma.Database do
 
     Maintenance.vacuum(args)
   end
+
+  def run(["ensure_expiration"]) do
+    start_pleroma()
+    days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365)
+
+    Pleroma.Activity
+    |> join(:left, [a], u in assoc(a, :expiration))
+    |> where(local: true)
+    |> where([a, u], is_nil(u))
+    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Stream.each(fn activities ->
+      Enum.each(activities, fn activity ->
+        expires_at = Timex.shift(activity.inserted_at, days: days)
+
+        Pleroma.ActivityExpiration.create(activity, expires_at, false)
+      end)
+    end)
+    |> Stream.run()
+  end
 end
diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex
index db9c88d84..7cc9668b3 100644
--- a/lib/pleroma/activity_expiration.ex
+++ b/lib/pleroma/activity_expiration.ex
@@ -20,11 +20,11 @@ defmodule Pleroma.ActivityExpiration do
     field(:scheduled_at, :naive_datetime)
   end
 
-  def changeset(%ActivityExpiration{} = expiration, attrs) do
+  def changeset(%ActivityExpiration{} = expiration, attrs, validate_scheduled_at) do
     expiration
     |> cast(attrs, [:scheduled_at])
     |> validate_required([:scheduled_at])
-    |> validate_scheduled_at()
+    |> validate_scheduled_at(validate_scheduled_at)
   end
 
   def get_by_activity_id(activity_id) do
@@ -33,9 +33,9 @@ defmodule Pleroma.ActivityExpiration do
     |> Repo.one()
   end
 
-  def create(%Activity{} = activity, scheduled_at) do
+  def create(%Activity{} = activity, scheduled_at, validate_scheduled_at \\ true) do
     %ActivityExpiration{activity_id: activity.id}
-    |> changeset(%{scheduled_at: scheduled_at})
+    |> changeset(%{scheduled_at: scheduled_at}, validate_scheduled_at)
     |> Repo.insert()
   end
 
@@ -49,7 +49,9 @@ defmodule Pleroma.ActivityExpiration do
     |> Repo.all()
   end
 
-  def validate_scheduled_at(changeset) do
+  def validate_scheduled_at(changeset, false), do: changeset
+
+  def validate_scheduled_at(changeset, true) do
     validate_change(changeset, :scheduled_at, fn _, scheduled_at ->
       if not expires_late_enough?(scheduled_at) do
         [scheduled_at: "an ephemeral activity must live for at least one hour"]
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 09e606b37..d1436a688 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -638,6 +638,34 @@ defmodule Pleroma.User do
   @spec force_password_reset(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
   def force_password_reset(user), do: update_password_reset_pending(user, true)
 
+  # Used to auto-register LDAP accounts which won't have a password hash stored locally
+  def register_changeset_ldap(struct, params = %{password: password})
+      when is_nil(password) do
+    params = Map.put_new(params, :accepts_chat_messages, true)
+
+    params =
+      if Map.has_key?(params, :email) do
+        Map.put_new(params, :email, params[:email])
+      else
+        params
+      end
+
+    struct
+    |> cast(params, [
+      :name,
+      :nickname,
+      :email,
+      :accepts_chat_messages
+    ])
+    |> validate_required([:name, :nickname])
+    |> unique_constraint(:nickname)
+    |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames]))
+    |> validate_format(:nickname, local_nickname_regex())
+    |> put_ap_id()
+    |> unique_constraint(:ap_id)
+    |> put_following_and_follower_address()
+  end
+
   def register_changeset(struct, params \\ %{}, opts \\ []) do
     bio_limit = Config.get([:instance, :user_bio_length], 5000)
     name_limit = Config.get([:instance, :user_name_length], 100)
diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex
index f63a66c03..402ab428b 100644
--- a/lib/pleroma/web/auth/ldap_authenticator.ex
+++ b/lib/pleroma/web/auth/ldap_authenticator.ex
@@ -28,10 +28,6 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
          %User{} = user <- ldap_user(name, password) do
       {:ok, user}
     else
-      {:error, {:ldap_connection_error, _}} ->
-        # When LDAP is unavailable, try default authenticator
-        @base.get_user(conn)
-
       {:ldap, _} ->
         @base.get_user(conn)
 
@@ -92,7 +88,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
             user
 
           _ ->
-            register_user(connection, base, uid, name, password)
+            register_user(connection, base, uid, name)
         end
 
       error ->
@@ -100,34 +96,31 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
     end
   end
 
-  defp register_user(connection, base, uid, name, password) do
+  defp register_user(connection, base, uid, name) do
     case :eldap.search(connection, [
            {:base, to_charlist(base)},
            {:filter, :eldap.equalityMatch(to_charlist(uid), to_charlist(name))},
            {:scope, :eldap.wholeSubtree()},
-           {:attributes, ['mail', 'email']},
            {:timeout, @search_timeout}
          ]) do
       {:ok, {:eldap_search_result, [{:eldap_entry, _, attributes}], _}} ->
-        with {_, [mail]} <- List.keyfind(attributes, 'mail', 0) do
-          params = %{
-            email: :erlang.list_to_binary(mail),
-            name: name,
-            nickname: name,
-            password: password,
-            password_confirmation: password
-          }
-
-          changeset = User.register_changeset(%User{}, params)
-
-          case User.register(changeset) do
-            {:ok, user} -> user
-            error -> error
+        params = %{
+          name: name,
+          nickname: name,
+          password: nil
+        }
+
+        params =
+          case List.keyfind(attributes, 'mail', 0) do
+            {_, [mail]} -> Map.put_new(params, :email, :erlang.list_to_binary(mail))
+            _ -> params
           end
-        else
-          _ ->
-            Logger.error("Could not find LDAP attribute mail: #{inspect(attributes)}")
-            {:error, :ldap_registration_missing_attributes}
+
+        changeset = User.register_changeset_ldap(%User{}, params)
+
+        case User.register(changeset) do
+          {:ok, user} -> user
+          error -> error
         end
 
       error ->
diff --git a/test/tasks/database_test.exs b/test/tasks/database_test.exs
index 883828d77..3a28aa133 100644
--- a/test/tasks/database_test.exs
+++ b/test/tasks/database_test.exs
@@ -127,4 +127,43 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do
       assert Enum.empty?(Object.get_by_id(object2.id).data["likes"])
     end
   end
+
+  describe "ensure_expiration" do
+    test "it adds to expiration old statuses" do
+      %{id: activity_id1} = insert(:note_activity)
+
+      %{id: activity_id2} =
+        insert(:note_activity, %{inserted_at: NaiveDateTime.from_iso8601!("2015-01-23 23:50:07")})
+
+      %{id: activity_id3} = activity3 = insert(:note_activity)
+
+      expires_at =
+        NaiveDateTime.utc_now()
+        |> NaiveDateTime.add(60 * 61, :second)
+        |> NaiveDateTime.truncate(:second)
+
+      Pleroma.ActivityExpiration.create(activity3, expires_at)
+
+      Mix.Tasks.Pleroma.Database.run(["ensure_expiration"])
+
+      expirations =
+        Pleroma.ActivityExpiration
+        |> order_by(:activity_id)
+        |> Repo.all()
+
+      assert [
+               %Pleroma.ActivityExpiration{
+                 activity_id: ^activity_id1
+               },
+               %Pleroma.ActivityExpiration{
+                 activity_id: ^activity_id2,
+                 scheduled_at: ~N[2016-01-23 23:50:07]
+               },
+               %Pleroma.ActivityExpiration{
+                 activity_id: ^activity_id3,
+                 scheduled_at: ^expires_at
+               }
+             ] = expirations
+    end
+  end
 end
diff --git a/test/web/oauth/ldap_authorization_test.exs b/test/web/oauth/ldap_authorization_test.exs
index 011642c08..63b1c0eb8 100644
--- a/test/web/oauth/ldap_authorization_test.exs
+++ b/test/web/oauth/ldap_authorization_test.exs
@@ -7,7 +7,6 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do
   alias Pleroma.Repo
   alias Pleroma.Web.OAuth.Token
   import Pleroma.Factory
-  import ExUnit.CaptureLog
   import Mock
 
   @skip if !Code.ensure_loaded?(:eldap), do: :skip
@@ -72,9 +71,7 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do
          equalityMatch: fn _type, _value -> :ok end,
          wholeSubtree: fn -> :ok end,
          search: fn _connection, _options ->
-           {:ok,
-            {:eldap_search_result, [{:eldap_entry, '', [{'mail', [to_charlist(user.email)]}]}],
-             []}}
+           {:ok, {:eldap_search_result, [{:eldap_entry, '', []}], []}}
          end,
          close: fn _connection ->
            send(self(), :close_connection)
@@ -102,50 +99,6 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do
   end
 
   @tag @skip
-  test "falls back to the default authorization when LDAP is unavailable" do
-    password = "testpassword"
-    user = insert(:user, password_hash: Pbkdf2.hash_pwd_salt(password))
-    app = insert(:oauth_app, scopes: ["read", "write"])
-
-    host = Pleroma.Config.get([:ldap, :host]) |> to_charlist
-    port = Pleroma.Config.get([:ldap, :port])
-
-    with_mocks [
-      {:eldap, [],
-       [
-         open: fn [^host], [{:port, ^port}, {:ssl, false} | _] -> {:error, 'connect failed'} end,
-         simple_bind: fn _connection, _dn, ^password -> :ok end,
-         close: fn _connection ->
-           send(self(), :close_connection)
-           :ok
-         end
-       ]}
-    ] do
-      log =
-        capture_log(fn ->
-          conn =
-            build_conn()
-            |> post("/oauth/token", %{
-              "grant_type" => "password",
-              "username" => user.nickname,
-              "password" => password,
-              "client_id" => app.client_id,
-              "client_secret" => app.client_secret
-            })
-
-          assert %{"access_token" => token} = json_response(conn, 200)
-
-          token = Repo.get_by(Token, token: token)
-
-          assert token.user_id == user.id
-        end)
-
-      assert log =~ "Could not open LDAP connection: 'connect failed'"
-      refute_received :close_connection
-    end
-  end
-
-  @tag @skip
   test "disallow authorization for wrong LDAP credentials" do
     password = "testpassword"
     user = insert(:user, password_hash: Pbkdf2.hash_pwd_salt(password))