VM: fix pageout throttling to external pagers

Since the VM system has been tracking whether pages belong to internal
or external objects, pageout throttling to external pagers has simply
not been working. The reason is that, on pageout, requests for external
pages are correctly tracked, but on page release (which is used to
acknowledge the request), external pages are not marked external
any more. This is because the external bit tracks whether a page
belongs to an external object, and all pages, including external
ones, are moved to an internal object during pageout.

To solve this issue, a new "external_laundry" bit is added. It has
the same purpose as the laundry bit, but for external pagers.

* vm/vm_page.c (vm_page_seg_min_page_available): Function unused, remove.
(vm_page_seg_evict): Use vm_page_external_laundry_count instead of
vm_page_external_pagedout. Add an assertion about double paging.
(vm_page_check_usable): Use vm_page_external_laundry_count instead of
vm_page_external_pagedout.
(vm_page_evict): Likewise.
* vm/vm_page.h (struct vm_page): New `external_laundry' member.
(vm_page_external_pagedout): Rename to ...
(vm_page_external_laundry_count): ... this.
* vm/vm_pageout.c: Include kern/printf.h.
(DEBUG): New macro.
(VM_PAGEOUT_TIMEOUT): Likewise.
(vm_pageout_setup): Use vm_page_external_laundry_count instead of
vm_page_external_pagedout. Set `external_laundry' where appropriate.
(vm_pageout): Use VM_PAGEOUT_TIMEOUT with thread_set_timeout.
Add debugging code, commented out by default.
* vm/vm_resident.c (vm_page_external_pagedout): Rename to ...
(vm_page_external_laundry_count): ... this.
(vm_page_init_template): Set `external_laundry' member to FALSE.
(vm_page_release): Rename external parameter to external_laundry.
Slightly change pageout resuming.
(vm_page_free): Rename external variable to external_laundry.

4 files changed, 44 insertions, 26 deletions

diff --git a/vm/vm_page.c b/vm/vm_page.c
index 94439b51..92e36a1a 100644
--- a/vm/vm_page.c
+++ b/vm/vm_page.c
@@ -925,12 +925,6 @@ vm_page_seg_pull_cache_page(struct vm_page_seg *seg,
 }
 
 static boolean_t
-vm_page_seg_min_page_available(const struct vm_page_seg *seg)
-{
-    return (seg->nr_free_pages > seg->min_free_pages);
-}
-
-static boolean_t
 vm_page_seg_page_available(const struct vm_page_seg *seg)
 {
     return (seg->nr_free_pages > seg->high_free_pages);
@@ -1099,6 +1093,7 @@ vm_page_seg_evict(struct vm_page_seg *seg, boolean_t external_only,
 
     page = NULL;
     object = NULL;
+    laundry = FALSE;
 
 restart:
     vm_page_lock_queues();
@@ -1156,6 +1151,7 @@ restart:
      */
 
     assert(!page->laundry);
+    assert(!(laundry && page->external));
 
     if (object->internal || !alloc_paused) {
         laundry = FALSE;
@@ -1878,7 +1874,7 @@ vm_page_check_usable(void)
         }
     }
 
-    vm_page_external_pagedout = -1;
+    vm_page_external_laundry_count = -1;
     vm_page_alloc_paused = FALSE;
     thread_wakeup(&vm_page_alloc_paused);
     return TRUE;
@@ -1980,7 +1976,7 @@ vm_page_evict(boolean_t *should_wait)
     external_only = TRUE;
 
     simple_lock(&vm_page_queue_free_lock);
-    vm_page_external_pagedout = 0;
+    vm_page_external_laundry_count = 0;
     alloc_paused = vm_page_alloc_paused;
     simple_unlock(&vm_page_queue_free_lock);
 
@@ -2008,7 +2004,7 @@ again:
      * Keep in mind eviction may not cause pageouts, since non-precious
      * clean pages are simply released.
      */
-    if ((vm_page_external_pagedout == 0) && (vm_page_laundry_count == 0)) {
+    if ((vm_page_laundry_count == 0) && (vm_page_external_laundry_count == 0)) {
         /*
          * No pageout, but some clean pages were freed. Start a complete
          * scan again without waiting.
diff --git a/vm/vm_page.h b/vm/vm_page.h
index eb684c1b..2a0ad2c2 100644
--- a/vm/vm_page.h
+++ b/vm/vm_page.h
@@ -105,6 +105,7 @@ struct vm_page {
 	/* boolean_t */	inactive:1,	/* page is in inactive list (P) */
 			active:1,	/* page is in active list (P) */
 			laundry:1,	/* page is being cleaned now (P)*/
+			external_laundry:1,	/* same as laundry for external pagers (P)*/
 			free:1,		/* page is on free list (P) */
 			reference:1,	/* page has been used (P) */
 			external:1,	/* page in external object (P) */
@@ -165,7 +166,7 @@ int	vm_page_wire_count;	/* How many pages are wired? */
 extern
 int	vm_page_laundry_count;	/* How many pages being laundered? */
 extern
-int	vm_page_external_pagedout;	/* How many external pages being paged out? */
+int	vm_page_external_laundry_count;	/* How many external pages being paged out? */
 
 decl_simple_lock_data(extern,vm_page_queue_lock)/* lock on active and inactive
 						   page queues */
diff --git a/vm/vm_pageout.c b/vm/vm_pageout.c
index 85db021e..7dc9c12f 100644
--- a/vm/vm_pageout.c
+++ b/vm/vm_pageout.c
@@ -46,6 +46,7 @@
 #include <kern/slab.h>
 #include <kern/task.h>
 #include <kern/thread.h>
+#include <kern/printf.h>
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 #include <vm/vm_object.h>
@@ -53,6 +54,13 @@
 #include <vm/vm_pageout.h>
 #include <machine/locore.h>
 
+#define DEBUG 0
+
+/*
+ * Maximum delay, in milliseconds, between two pageout scans.
+ */
+#define VM_PAGEOUT_TIMEOUT 50
+
 /*
  * Event placeholder for pageout requests, synchronized with
  * the free page queue lock.
@@ -251,17 +259,19 @@ vm_pageout_setup(
 
 		vm_page_wire(m);
 	} else {
-		vm_page_activate(m);
+		m->external_laundry = TRUE;
 
 		/*
-		 *	If vm_page_external_pagedout is negative,
+		 *	If vm_page_external_laundry_count is negative,
 		 *	the pageout daemon isn't expecting to be
 		 *	notified.
 		 */
 
-		if (vm_page_external_pagedout >= 0) {
-			vm_page_external_pagedout++;
+		if (vm_page_external_laundry_count >= 0) {
+			vm_page_external_laundry_count++;
 		}
+
+		vm_page_activate(m);
 	}
 	vm_page_unlock_queues();
 
@@ -460,9 +470,19 @@ void vm_pageout(void)
 				     FALSE);
 		} else if (should_wait) {
 			assert_wait(&vm_pageout_continue, FALSE);
-			thread_set_timeout(500);
+			thread_set_timeout(VM_PAGEOUT_TIMEOUT);
 			simple_unlock(&vm_page_queue_free_lock);
 			thread_block(NULL);
+
+#if DEBUG
+			if (current_thread()->wait_result != THREAD_AWAKENED) {
+				printf("vm_pageout: timeout,"
+				       " vm_page_laundry_count:%d"
+				       " vm_page_external_laundry_count:%d\n",
+				       vm_page_laundry_count,
+				       vm_page_external_laundry_count);
+			}
+#endif
 		} else {
 			simple_unlock(&vm_page_queue_free_lock);
 		}
diff --git a/vm/vm_resident.c b/vm/vm_resident.c
index e3e34dc3..b5096e00 100644
--- a/vm/vm_resident.c
+++ b/vm/vm_resident.c
@@ -140,7 +140,7 @@ int	vm_page_wire_count;
  *	pageout daemon.
  */
 int	vm_page_laundry_count = 0;
-int	vm_page_external_pagedout = 0;
+int	vm_page_external_laundry_count = 0;
 
 
 /*
@@ -611,6 +611,7 @@ static void vm_page_init_template(vm_page_t m)
 	m->inactive = FALSE;
 	m->active = FALSE;
 	m->laundry = FALSE;
+	m->external_laundry = FALSE;
 	m->free = FALSE;
 	m->external = FALSE;
 
@@ -811,7 +812,7 @@ phys_addr_t vm_page_grab_phys_addr(void)
 void vm_page_release(
 	vm_page_t	mem,
 	boolean_t 	laundry,
-	boolean_t 	external)
+	boolean_t 	external_laundry)
 {
 	simple_lock(&vm_page_queue_free_lock);
 	if (mem->free)
@@ -825,20 +826,20 @@ void vm_page_release(
 			vm_pageout_resume();
 		}
 	}
-	if (external) {
+	if (external_laundry) {
 
 		/*
-		 *	If vm_page_external_pagedout is negative,
+		 *	If vm_page_external_laundry_count is negative,
 		 *	the pageout daemon isn't expecting to be
 		 *	notified.
 		 */
 
-		if (vm_page_external_pagedout > 0) {
-			vm_page_external_pagedout--;
-		}
+		if (vm_page_external_laundry_count > 0) {
+			vm_page_external_laundry_count--;
 
-		if (vm_page_external_pagedout == 0) {
-			vm_pageout_resume();
+			if (vm_page_external_laundry_count == 0) {
+				vm_pageout_resume();
+			}
 		}
 	}
 
@@ -977,9 +978,9 @@ void vm_page_free(
 		vm_page_release_fictitious(mem);
 	} else {
 		boolean_t laundry = mem->laundry;
-		boolean_t external = mem->external;
+		boolean_t external_laundry = mem->external_laundry;
 		vm_page_init(mem);
-		vm_page_release(mem, laundry, external);
+		vm_page_release(mem, laundry, external_laundry);
 	}
 }