summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSergio Lopez <slp@sinrega.org>2011-12-29 22:30:12 +0100
committerSergio Lopez <slp@sinrega.org>2011-12-29 22:30:12 +0100
commit666299d037be6ffa83345d6d281fa955431f55fe (patch)
treef63ff542ac6bbe6d6d99fa17348a5ca958685227
parent6af53cb9d48013971ad5fa37de173c1bbf73292a (diff)
* include/mach/memory_object.h: Add MEMORY_OBJECT_COPY_VMPRIV to mark objects that can't be blocked even if dirty page limits has been exceeded.k0ro/advisory_pageout/master
* vm/memory_object.c (memory_object_data_supply): Mark pages provided by user as external. * (memory_object_lock_request): Lock page queues before cleaning holding pages. * (memory_object_set_attributes_common): Deal with MEMORY_OBJECT_COPY_VMPRIV. * vm/vm_fault.c (vm_fault_page): If the number of potentially dirty pages has reached a certain static number, either request a pageout or block the thread. * (vm_fault_page): Force an early request for a real page instead of a fictitious one. * vm/vm_object.h (struct vm_object): New flag vm_privileged that relates with MEMORY_OBJECT_COPY_PRIV. * vm/vm_object.c (vm_object_bootstrap): Initialize vm_privileged property as FALSE. * vm/vm_page.h: Add variables vm_page_too_dirty, vm_page_dirty_count and vm_page_external_target. * vm/vm_pageout.c: Adjust VM_PAGE_FREE_TARGET, VM_PAGE_FREE_MIN and VM_PAGE_EXTERNAL_LIMIT. Make vm_page_external_target signed. * (vm_pageout_setup): Write lock cleaned pages. Adjust dirty page counters. * (vm_pageout_scan): Try freeing clean external pages first. Disable broken old code. * vm/vm_resident.c: Add variables vm_page_too_dirty and vm_page_dirty_count. * (vm_page_convert.c): Propagate external argument to converted page. * (vm_page_grab): Don't use old external page limits. * (vm_page_grab_contiguous_pages): Likewise. * (vm_page_wait): Likewise. * (vm_page_release): Don't adjust external count. * (vm_page_free): Adjust external and external-dirty counters here. Wake up blocked threads.
-rw-r--r--include/mach/memory_object.h2
-rw-r--r--vm/memory_object.c17
-rw-r--r--vm/vm_fault.c26
-rw-r--r--vm/vm_object.c1
-rw-r--r--vm/vm_object.h4
-rw-r--r--vm/vm_page.h6
-rw-r--r--vm/vm_pageout.c149
-rw-r--r--vm/vm_resident.c31
8 files changed, 214 insertions, 22 deletions
diff --git a/include/mach/memory_object.h b/include/mach/memory_object.h
index f281f04d..c5e19cb0 100644
--- a/include/mach/memory_object.h
+++ b/include/mach/memory_object.h
@@ -70,6 +70,8 @@ typedef int memory_object_copy_strategy_t;
/* ... Memory manager doesn't ... */
/* change data externally, and */
/* doesn't need to see changes. */
+#define MEMORY_OBJECT_COPY_VMPRIV 4
+ /* ... XXX Just for testing ... */
typedef int memory_object_return_t;
/* Which pages to return to manager
diff --git a/vm/memory_object.c b/vm/memory_object.c
index e281c6a3..41bfd739 100644
--- a/vm/memory_object.c
+++ b/vm/memory_object.c
@@ -217,12 +217,22 @@ retry_lookup:
data_m->dirty = FALSE;
pmap_clear_modify(data_m->phys_addr);
+ if (!object->internal) {
+ data_m->external = TRUE;
+ if (lock_value == VM_PROT_NONE) {
+ /* Be sure we get notified if page
+ could be potentially dirty */
+ lock_value = VM_PROT_WRITE;
+ }
+ }
+
data_m->page_lock = lock_value;
data_m->unlock_request = VM_PROT_NONE;
data_m->precious = precious;
vm_page_lock_queues();
vm_page_insert(data_m, object, offset);
+ vm_page_external_count++;
if (was_absent)
vm_page_activate(data_m);
@@ -749,11 +759,13 @@ MACRO_BEGIN \
\
vm_object_lock(object); \
\
+ vm_page_lock_queues(); \
for (i = 0; i < atop(new_offset); i++) { \
hp = holding_pages[i]; \
if (hp != VM_PAGE_NULL) \
VM_PAGE_FREE(hp); \
} \
+ vm_page_unlock_queues(); \
\
new_object = VM_OBJECT_NULL; \
MACRO_END
@@ -912,6 +924,7 @@ memory_object_set_attributes_common(object, object_ready, may_cache,
case MEMORY_OBJECT_COPY_CALL:
case MEMORY_OBJECT_COPY_DELAY:
case MEMORY_OBJECT_COPY_TEMPORARY:
+ case MEMORY_OBJECT_COPY_VMPRIV:
break;
default:
vm_object_deallocate(object);
@@ -943,6 +956,10 @@ memory_object_set_attributes_common(object, object_ready, may_cache,
object->pager_ready = object_ready;
if (copy_strategy == MEMORY_OBJECT_COPY_TEMPORARY) {
object->temporary = TRUE;
+ } else if (copy_strategy == MEMORY_OBJECT_COPY_VMPRIV) {
+ object->vm_privileged = TRUE;
+ object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
+ printf("creating an vm_privileged object\n");
} else {
object->copy_strategy = copy_strategy;
}
diff --git a/vm/vm_fault.c b/vm/vm_fault.c
index cce043a1..1f911d84 100644
--- a/vm/vm_fault.c
+++ b/vm/vm_fault.c
@@ -509,6 +509,7 @@ vm_fault_return_t vm_fault_page(first_object, first_offset,
if (access_required & m->page_lock) {
if ((access_required & m->unlock_request) != access_required) {
vm_prot_t new_unlock_request;
+ vm_size_t ugly_hack = PAGE_SIZE;
kern_return_t rc;
if (!object->pager_ready) {
@@ -518,6 +519,25 @@ vm_fault_return_t vm_fault_page(first_object, first_offset,
goto block_and_backoff;
}
+ /* XXX - These limits should be dynamic */
+ if (vm_page_dirty_count > 3000) {
+ if (!object->vm_privileged) {
+ printf("blocking not vm_privileged object\n");
+ vm_page_too_dirty++;
+ assert_wait((event_t)&vm_page_too_dirty, FALSE);
+ goto block_and_backoff;
+ }
+ else {
+ printf("NOT blocking vm_privileged object\n");
+ }
+ }
+ else if (vm_page_dirty_count > 2700 && !object->vm_privileged) {
+ ugly_hack++;
+ }
+
+ m->overwriting = TRUE;
+ vm_page_dirty_count++;
+
new_unlock_request = m->unlock_request =
(access_required | m->unlock_request);
vm_object_unlock(object);
@@ -525,7 +545,7 @@ vm_fault_return_t vm_fault_page(first_object, first_offset,
object->pager,
object->pager_request,
offset + object->paging_offset,
- PAGE_SIZE,
+ ugly_hack,
new_unlock_request))
!= KERN_SUCCESS) {
printf("vm_fault: memory_object_data_unlock failed\n");
@@ -587,6 +607,10 @@ vm_fault_return_t vm_fault_page(first_object, first_offset,
vm_fault_cleanup(object, first_m);
return(VM_FAULT_FICTITIOUS_SHORTAGE);
}
+ else if (m->fictitious && !vm_page_convert(m, TRUE)) {
+ VM_PAGE_FREE(m);
+ vm_fault_cleanup(object, first_m);
+ }
vm_page_lock_queues();
vm_page_insert(m, object, offset);
diff --git a/vm/vm_object.c b/vm/vm_object.c
index 9057973d..75d7a397 100644
--- a/vm/vm_object.c
+++ b/vm/vm_object.c
@@ -293,6 +293,7 @@ void vm_object_bootstrap(void)
vm_object_template->lock_restart = FALSE;
vm_object_template->use_old_pageout = TRUE; /* XXX change later */
vm_object_template->last_alloc = (vm_offset_t) 0;
+ vm_object_template->vm_privileged = FALSE;
#if MACH_PAGEMAP
vm_object_template->existence_info = VM_EXTERNAL_NULL;
diff --git a/vm/vm_object.h b/vm/vm_object.h
index c9925709..d912b70e 100644
--- a/vm/vm_object.h
+++ b/vm/vm_object.h
@@ -148,7 +148,9 @@ struct vm_object {
*/
/* boolean_t */ use_shared_copy : 1,/* Use shared (i.e.,
* delayed) copy on write */
- /* boolean_t */ shadowed: 1; /* Shadow may exist */
+ /* boolean_t */ shadowed: 1, /* Shadow may exist */
+ /* boolean_t */ vm_privileged: 1; /* Object is allowed to exceed
+ dirty pages limit */
queue_chain_t cached_list; /* Attachment point for the list
* of objects cached as a result
diff --git a/vm/vm_page.h b/vm/vm_page.h
index f13b0af9..e50b8b62 100644
--- a/vm/vm_page.h
+++ b/vm/vm_page.h
@@ -183,6 +183,10 @@ extern
int vm_page_laundry_count; /* How many pages being laundered? */
extern
int vm_page_external_limit; /* Max number of pages for external objects */
+extern
+int vm_page_too_dirty; /* Pages on hold due to dirty limit */
+extern
+int vm_page_dirty_count; /* How many pages are (potentially) dirty? */
/* Only objects marked with the extcounted bit are included in this total.
Pages which we scan for possible pageout, but which are not actually
@@ -190,6 +194,8 @@ int vm_page_external_limit; /* Max number of pages for external objects */
in this way. */
extern
int vm_page_external_count; /* How many pages for external objects? */
+extern
+int vm_page_external_target;/* How many external pages do we want? */
diff --git a/vm/vm_pageout.c b/vm/vm_pageout.c
index 7a755bf4..7ac62876 100644
--- a/vm/vm_pageout.c
+++ b/vm/vm_pageout.c
@@ -97,7 +97,7 @@
*/
#ifndef VM_PAGE_FREE_TARGET
-#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80)
+#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 20)
#endif /* VM_PAGE_FREE_TARGET */
/*
@@ -106,7 +106,7 @@
*/
#ifndef VM_PAGE_FREE_MIN
-#define VM_PAGE_FREE_MIN(free) (10 + (free) / 100)
+#define VM_PAGE_FREE_MIN(free) (10 + (free) / 40)
#endif /* VM_PAGE_FREE_MIN */
/* When vm_page_external_count exceeds vm_page_external_limit,
@@ -114,7 +114,7 @@
*/
#ifndef VM_PAGE_EXTERNAL_LIMIT
-#define VM_PAGE_EXTERNAL_LIMIT(free) ((free) / 2)
+#define VM_PAGE_EXTERNAL_LIMIT(free) ((free) / 10)
#endif /* VM_PAGE_EXTERNAL_LIMIT */
/* Attempt to keep the number of externally paged pages less
@@ -165,7 +165,7 @@ extern void vm_pageout_scan_continue();
unsigned int vm_pageout_reserved_internal = 0;
unsigned int vm_pageout_reserved_really = 0;
-unsigned int vm_page_external_target = 0;
+int vm_page_external_target = 0;
unsigned int vm_pageout_burst_max = 0;
unsigned int vm_pageout_burst_min = 0;
@@ -313,10 +313,35 @@ vm_pageout_setup(m, paging_offset, new_object, new_offset, flush)
pmap_clear_modify(m->phys_addr);
/*
+ * If this is an external object,
+ * write lock the page so a new
+ * unlock request is generated if
+ * it's going to become dirty.
+ */
+ if (!old_object->internal)
+ m->page_lock = VM_PROT_WRITE;
+
+ /*
* Deactivate old page.
*/
vm_page_lock_queues();
vm_page_deactivate(m);
+
+ /*
+ * If this is an external page,
+ * and we're really keeping track
+ * of it, adjust dirty count.
+ */
+ if (m->external && m->overwriting) {
+ m->overwriting = FALSE;
+ vm_page_dirty_count--;
+ if (vm_page_too_dirty) {
+ vm_page_too_dirty--;
+ if (vm_page_too_dirty == 0)
+ thread_wakeup((event_t)&vm_page_too_dirty);
+ }
+ }
+
vm_page_unlock_queues();
PAGE_WAKEUP_DONE(m);
@@ -340,6 +365,7 @@ vm_pageout_setup(m, paging_offset, new_object, new_offset, flush)
*/
m = new_m;
m->dirty = TRUE;
+ m->external = FALSE;
assert(!m->precious);
PAGE_WAKEUP_DONE(m);
}
@@ -514,6 +540,10 @@ void vm_pageout_scan()
{
unsigned int burst_count;
unsigned int want_pages;
+ unsigned int external_available;
+ int current_target;
+ boolean_t found = FALSE;
+ boolean_t first_run = TRUE;
/*
* We want to gradually dribble pages from the active queue
@@ -640,8 +670,92 @@ void vm_pageout_scan()
}
want_pages = ((free_count < vm_page_free_target) ||
vm_page_free_wanted);
+ current_target = vm_page_free_target - free_count;
+ external_available = vm_page_external_count - vm_page_external_limit;
simple_unlock(&vm_page_queue_free_lock);
+ if (first_run &&
+ current_target > 0 &&
+ external_available > (current_target / 2)) {
+ int target = external_available > current_target ?
+ current_target : external_available;
+ int iters = 0;
+
+ first_run = FALSE;
+ printf("ec: t=%u, ct=%u, ea=%u, td=%d\n", target, current_target,
+ external_available, vm_page_too_dirty);
+ m = (vm_page_t) queue_first(&vm_page_queue_inactive);
+
+ while (target &&
+ (!queue_end(&vm_page_queue_inactive, (queue_entry_t) m))) {
+ vm_page_t old_page;
+
+ iters++;
+
+ if (!m->external) {
+ m = (vm_page_t) queue_next(&m->pageq);
+ continue;
+ }
+
+ object = m->object;
+ if (!vm_object_lock_try(object)) {
+ m = (vm_page_t) queue_next(&m->pageq);
+ continue;
+ }
+
+ if (!m->dirty)
+ m->dirty = pmap_is_modified(m->phys_addr);
+
+ if (!m->dirty && !m->precious &&
+ !m->absent && !m->busy) {
+ /* This page can be safely freed */
+
+ old_page = m;
+ m = (vm_page_t) queue_next(&old_page->pageq);
+ old_page->busy = TRUE;
+ pmap_page_protect(old_page->phys_addr, VM_PROT_NONE);
+ VM_PAGE_FREE(old_page);
+ target--;
+ current_target--;
+ }
+ else {
+ /* Page is not ready to be freed. Try next */
+ m = (vm_page_t) queue_next(&m->pageq);
+ }
+ vm_object_unlock(object);
+ }
+
+ printf("external_count: end target=%d, iters=%d, ic=%d\n",
+ target, iters, vm_page_inactive_count);
+
+ if (target || current_target) {
+ printf("Can't free enough external pages. Using old method\n");
+
+ simple_lock(&vm_page_queue_free_lock);
+ free_count = vm_page_free_count;
+ if ((free_count >= vm_page_free_target) &&
+ (vm_page_external_count <= vm_page_external_target) &&
+ (vm_page_free_wanted == 0)) {
+ vm_page_unlock_queues();
+ break;
+ }
+
+ want_pages = ((free_count < vm_page_free_target) || vm_page_free_wanted);
+
+ simple_unlock(&vm_page_queue_free_lock);
+ }
+ else {
+ vm_page_unlock_queues();
+ break;
+ }
+ }
+ else {
+ int target = external_available > current_target ?
+ current_target : external_available;
+ printf("INTERNAL ec: t=%u ct=%u ea=%u td=%d\n", target, current_target,
+ external_available, vm_page_too_dirty);
+ }
+
/*
* Sometimes we have to pause:
* 1) No inactive pages - nothing to do.
@@ -651,13 +765,19 @@ void vm_pageout_scan()
* this if the default pager already has work to do.
*/
pause:
- if (queue_empty(&vm_page_queue_inactive) ||
+ if (found == 2 ||
+ queue_empty(&vm_page_queue_inactive) ||
(burst_count >= vm_pageout_burst_max) ||
(vm_page_laundry_count >= vm_pageout_burst_max) ||
((free_count < vm_pageout_reserved_really) &&
(vm_page_laundry_count > 0))) {
unsigned int pages, msecs;
+ if (!want_pages) {
+ vm_page_unlock_queues();
+ break;
+ }
+
/*
* vm_pageout_burst_wait is msecs/page.
* If there is nothing for us to do, we wait
@@ -687,17 +807,24 @@ void vm_pageout_scan()
/* Find a page we are interested in paging out. If we
need pages, then we'll page anything out; otherwise
we only page out external pages. */
+ found = 0;
m = (vm_page_t) queue_first (&vm_page_queue_inactive);
- while (1)
+ while (!queue_end(&vm_page_queue_inactive, (queue_entry_t) m))
{
assert (!m->active && m->inactive);
- if (want_pages || m->external)
+ if (!m->external) {
+ found = 1;
break;
+ }
- m = (vm_page_t) queue_next (m);
- if (!m)
- goto pause;
+ m = (vm_page_t) queue_next(&m->pageq);
}
+
+ if (!found) {
+ printf("Can't find an internal page to flush\n");
+ found = 2;
+ goto pause;
+ }
object = m->object;
@@ -777,6 +904,7 @@ void vm_pageout_scan()
if (!m->dirty)
m->dirty = pmap_is_modified(m->phys_addr);
+#if 0
if (m->external) {
/* Figure out if we still care about this
page in the limit of externally managed pages.
@@ -804,6 +932,7 @@ void vm_pageout_scan()
vm_pageout_inactive_cleaned_external++;
continue;
}
+#endif
/*
* If it's clean and not precious, we can free the page.
diff --git a/vm/vm_resident.c b/vm/vm_resident.c
index 96354a45..d3db6869 100644
--- a/vm/vm_resident.c
+++ b/vm/vm_resident.c
@@ -162,6 +162,8 @@ int vm_page_inactive_target = 0;
int vm_page_free_reserved = 0;
int vm_page_laundry_count = 0;
int vm_page_external_limit = 0;
+int vm_page_too_dirty = 0;
+int vm_page_dirty_count = 0;
/*
@@ -207,6 +209,7 @@ void vm_page_bootstrap(
m->laundry = FALSE;
m->free = FALSE;
m->external = FALSE;
+ m->overwriting = FALSE;
m->busy = TRUE;
m->wanted = FALSE;
@@ -838,6 +841,7 @@ boolean_t vm_page_convert(
m->phys_addr = real_m->phys_addr;
m->fictitious = FALSE;
+ m->external = external;
real_m->phys_addr = vm_page_fictitious_addr;
real_m->fictitious = TRUE;
@@ -866,10 +870,8 @@ vm_page_t vm_page_grab(
* for externally-managed pages.
*/
- if (((vm_page_free_count < vm_page_free_reserved)
- || (external
- && (vm_page_external_count > vm_page_external_limit)))
- && !current_thread()->vm_privilege) {
+ if ((vm_page_free_count < vm_page_free_reserved) &&
+ !current_thread()->vm_privilege) {
simple_unlock(&vm_page_queue_free_lock);
return VM_PAGE_NULL;
}
@@ -979,8 +981,7 @@ vm_page_grab_contiguous_pages(
* Do not dip into the reserved pool.
*/
- if ((vm_page_free_count < vm_page_free_reserved)
- || (vm_page_external_count >= vm_page_external_limit)) {
+ if (vm_page_free_count < vm_page_free_reserved) {
printf_once("no more room for vm_page_grab_contiguous_pages");
simple_unlock(&vm_page_queue_free_lock);
return KERN_RESOURCE_SHORTAGE;
@@ -1163,8 +1164,6 @@ void vm_page_release(
mem->pageq.next = (queue_entry_t) vm_page_queue_free;
vm_page_queue_free = mem;
vm_page_free_count++;
- if (external)
- vm_page_external_count--;
/*
* Check if we should wake up someone waiting for page.
@@ -1215,8 +1214,7 @@ void vm_page_wait(
*/
simple_lock(&vm_page_queue_free_lock);
- if ((vm_page_free_count < vm_page_free_target)
- || (vm_page_external_count > vm_page_external_limit)) {
+ if (vm_page_free_count < vm_page_free_target) {
if (vm_page_free_wanted++ == 0)
thread_wakeup((event_t)&vm_page_free_wanted);
assert_wait((event_t)&vm_page_free_count, FALSE);
@@ -1292,6 +1290,19 @@ void vm_page_free(
if (mem->absent)
vm_object_absent_release(mem->object);
+ if (mem->external) {
+ vm_page_external_count--;
+ mem->external = FALSE;
+ if (mem->overwriting) {
+ vm_page_dirty_count--;
+ if (vm_page_too_dirty) {
+ vm_page_too_dirty--;
+ if (vm_page_too_dirty == 0)
+ thread_wakeup((event_t)&vm_page_too_dirty);
+ }
+ }
+ }
+
/*
* XXX The calls to vm_page_init here are
* really overkill.