From eb07428ffb0009085fcd01dd1b79d9953af8e0ad Mon Sep 17 00:00:00 2001
From: Richard Braun <rbraun@sceen.net>
Date: Fri, 23 Dec 2016 04:08:53 +0100
Subject: VM: fix pageout of external objects backed by the default pager

Double paging on such objects causes deadlocks.

* vm/vm_page.c: Include <vm/memory_object.h>.
(vm_page_seg_evict): Rename laundry to double_paging to increase
clarity. Set the `external_laundry' bit when evicting a page
from an external object backed by the default pager.
* vm/vm_pageout.c (vm_pageout_setup): Wire page if the
`external_laundry' bit is set.
---
 vm/vm_page.c    | 37 ++++++++++++++++++++++++++++---------
 vm/vm_pageout.c |  9 +++++++++
 2 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/vm/vm_page.c b/vm/vm_page.c
index b7b76a54..a7dab114 100644
--- a/vm/vm_page.c
+++ b/vm/vm_page.c
@@ -44,6 +44,7 @@
 #include <mach/vm_param.h>
 #include <machine/pmap.h>
 #include <sys/types.h>
+#include <vm/memory_object.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
 
@@ -1088,13 +1089,13 @@ vm_page_seg_evict(struct vm_page_seg *seg, boolean_t external_only,
                   boolean_t alloc_paused)
 {
     struct vm_page *page;
-    boolean_t reclaim, laundry;
+    boolean_t reclaim, double_paging;
     vm_object_t object;
     boolean_t was_active;
 
     page = NULL;
     object = NULL;
-    laundry = FALSE;
+    double_paging = FALSE;
 
 restart:
     vm_page_lock_queues();
@@ -1148,16 +1149,34 @@ restart:
      * processing of this page since it's immediately going to be
      * double paged out to the default pager. The laundry bit is
      * reset and the page is inserted into an internal object by
-     * vm_pageout_setup before the double paging pass.
+     * vm_pageout_setup before the second double paging pass.
+     *
+     * There is one important special case: the default pager can
+     * back external memory objects. When receiving the first
+     * pageout request, where the page is no longer present, a
+     * fault could occur, during which the map would be locked.
+     * This fault would cause a new paging request to the default
+     * pager. Receiving that request would deadlock when trying to
+     * lock the map again. Instead, the page isn't double paged.
+     * The external_laundry bit is set to indicate this situation
+     * to vm_pageout_setup.
      */
 
-    assert(!page->laundry);
-    assert(!(laundry && page->external));
+    assert(!page->laundry && !page->external_laundry);
+    assert(!(double_paging && page->external));
 
-    if (object->internal || !alloc_paused) {
-        laundry = FALSE;
+    if (object->internal) {
+        double_paging = FALSE;
     } else {
-        laundry = page->laundry = TRUE;
+        if (memory_manager_default_port(object->pager)) {
+            double_paging = FALSE;
+            page->external_laundry = TRUE;
+        } else if (!alloc_paused) {
+            double_paging = FALSE;
+        } else {
+            double_paging = TRUE;
+            page->laundry = TRUE;
+        }
     }
 
 out:
@@ -1204,7 +1223,7 @@ out:
     vm_pageout_page(page, FALSE, TRUE); /* flush it */
     vm_object_unlock(object);
 
-    if (laundry) {
+    if (double_paging) {
         goto restart;
     }
 
diff --git a/vm/vm_pageout.c b/vm/vm_pageout.c
index da0f8077..62a27f1a 100644
--- a/vm/vm_pageout.c
+++ b/vm/vm_pageout.c
@@ -259,6 +259,15 @@ vm_pageout_setup(
 
 		vm_page_wire(m);
 	} else {
+		/*
+		 *	The caller is telling us that this page belongs
+		 *	to an external object managed by the default pager.
+		 *	Wire it to avoid a deadlock on the default pager map.
+		 */
+		if (m->external_laundry) {
+			vm_page_wire(m);
+		}
+
 		m->external_laundry = TRUE;
 
 		/*
-- 
cgit v1.2.3