diff options
-rw-r--r-- | include/mach/memory_object.h | 2 | ||||
-rw-r--r-- | vm/memory_object.c | 17 | ||||
-rw-r--r-- | vm/vm_fault.c | 26 | ||||
-rw-r--r-- | vm/vm_object.c | 1 | ||||
-rw-r--r-- | vm/vm_object.h | 4 | ||||
-rw-r--r-- | vm/vm_page.h | 6 | ||||
-rw-r--r-- | vm/vm_pageout.c | 149 | ||||
-rw-r--r-- | vm/vm_resident.c | 31 |
8 files changed, 214 insertions, 22 deletions
diff --git a/include/mach/memory_object.h b/include/mach/memory_object.h index f281f04d..c5e19cb0 100644 --- a/include/mach/memory_object.h +++ b/include/mach/memory_object.h @@ -70,6 +70,8 @@ typedef int memory_object_copy_strategy_t; /* ... Memory manager doesn't ... */ /* change data externally, and */ /* doesn't need to see changes. */ +#define MEMORY_OBJECT_COPY_VMPRIV 4 + /* ... XXX Just for testing ... */ typedef int memory_object_return_t; /* Which pages to return to manager diff --git a/vm/memory_object.c b/vm/memory_object.c index e281c6a3..41bfd739 100644 --- a/vm/memory_object.c +++ b/vm/memory_object.c @@ -217,12 +217,22 @@ retry_lookup: data_m->dirty = FALSE; pmap_clear_modify(data_m->phys_addr); + if (!object->internal) { + data_m->external = TRUE; + if (lock_value == VM_PROT_NONE) { + /* Be sure we get notified if page + could be potentially dirty */ + lock_value = VM_PROT_WRITE; + } + } + data_m->page_lock = lock_value; data_m->unlock_request = VM_PROT_NONE; data_m->precious = precious; vm_page_lock_queues(); vm_page_insert(data_m, object, offset); + vm_page_external_count++; if (was_absent) vm_page_activate(data_m); @@ -749,11 +759,13 @@ MACRO_BEGIN \ \ vm_object_lock(object); \ \ + vm_page_lock_queues(); \ for (i = 0; i < atop(new_offset); i++) { \ hp = holding_pages[i]; \ if (hp != VM_PAGE_NULL) \ VM_PAGE_FREE(hp); \ } \ + vm_page_unlock_queues(); \ \ new_object = VM_OBJECT_NULL; \ MACRO_END @@ -912,6 +924,7 @@ memory_object_set_attributes_common(object, object_ready, may_cache, case MEMORY_OBJECT_COPY_CALL: case MEMORY_OBJECT_COPY_DELAY: case MEMORY_OBJECT_COPY_TEMPORARY: + case MEMORY_OBJECT_COPY_VMPRIV: break; default: vm_object_deallocate(object); @@ -943,6 +956,10 @@ memory_object_set_attributes_common(object, object_ready, may_cache, object->pager_ready = object_ready; if (copy_strategy == MEMORY_OBJECT_COPY_TEMPORARY) { object->temporary = TRUE; + } else if (copy_strategy == MEMORY_OBJECT_COPY_VMPRIV) { + object->vm_privileged = TRUE; + object->copy_strategy = MEMORY_OBJECT_COPY_NONE; + printf("creating an vm_privileged object\n"); } else { object->copy_strategy = copy_strategy; } diff --git a/vm/vm_fault.c b/vm/vm_fault.c index cce043a1..1f911d84 100644 --- a/vm/vm_fault.c +++ b/vm/vm_fault.c @@ -509,6 +509,7 @@ vm_fault_return_t vm_fault_page(first_object, first_offset, if (access_required & m->page_lock) { if ((access_required & m->unlock_request) != access_required) { vm_prot_t new_unlock_request; + vm_size_t ugly_hack = PAGE_SIZE; kern_return_t rc; if (!object->pager_ready) { @@ -518,6 +519,25 @@ vm_fault_return_t vm_fault_page(first_object, first_offset, goto block_and_backoff; } + /* XXX - These limits should be dynamic */ + if (vm_page_dirty_count > 3000) { + if (!object->vm_privileged) { + printf("blocking not vm_privileged object\n"); + vm_page_too_dirty++; + assert_wait((event_t)&vm_page_too_dirty, FALSE); + goto block_and_backoff; + } + else { + printf("NOT blocking vm_privileged object\n"); + } + } + else if (vm_page_dirty_count > 2700 && !object->vm_privileged) { + ugly_hack++; + } + + m->overwriting = TRUE; + vm_page_dirty_count++; + new_unlock_request = m->unlock_request = (access_required | m->unlock_request); vm_object_unlock(object); @@ -525,7 +545,7 @@ vm_fault_return_t vm_fault_page(first_object, first_offset, object->pager, object->pager_request, offset + object->paging_offset, - PAGE_SIZE, + ugly_hack, new_unlock_request)) != KERN_SUCCESS) { printf("vm_fault: memory_object_data_unlock failed\n"); @@ -587,6 +607,10 @@ vm_fault_return_t vm_fault_page(first_object, first_offset, vm_fault_cleanup(object, first_m); return(VM_FAULT_FICTITIOUS_SHORTAGE); } + else if (m->fictitious && !vm_page_convert(m, TRUE)) { + VM_PAGE_FREE(m); + vm_fault_cleanup(object, first_m); + } vm_page_lock_queues(); vm_page_insert(m, object, offset); diff --git a/vm/vm_object.c b/vm/vm_object.c index 9057973d..75d7a397 100644 --- a/vm/vm_object.c +++ b/vm/vm_object.c @@ -293,6 +293,7 @@ void vm_object_bootstrap(void) vm_object_template->lock_restart = FALSE; vm_object_template->use_old_pageout = TRUE; /* XXX change later */ vm_object_template->last_alloc = (vm_offset_t) 0; + vm_object_template->vm_privileged = FALSE; #if MACH_PAGEMAP vm_object_template->existence_info = VM_EXTERNAL_NULL; diff --git a/vm/vm_object.h b/vm/vm_object.h index c9925709..d912b70e 100644 --- a/vm/vm_object.h +++ b/vm/vm_object.h @@ -148,7 +148,9 @@ struct vm_object { */ /* boolean_t */ use_shared_copy : 1,/* Use shared (i.e., * delayed) copy on write */ - /* boolean_t */ shadowed: 1; /* Shadow may exist */ + /* boolean_t */ shadowed: 1, /* Shadow may exist */ + /* boolean_t */ vm_privileged: 1; /* Object is allowed to exceed + dirty pages limit */ queue_chain_t cached_list; /* Attachment point for the list * of objects cached as a result diff --git a/vm/vm_page.h b/vm/vm_page.h index f13b0af9..e50b8b62 100644 --- a/vm/vm_page.h +++ b/vm/vm_page.h @@ -183,6 +183,10 @@ extern int vm_page_laundry_count; /* How many pages being laundered? */ extern int vm_page_external_limit; /* Max number of pages for external objects */ +extern +int vm_page_too_dirty; /* Pages on hold due to dirty limit */ +extern +int vm_page_dirty_count; /* How many pages are (potentially) dirty? */ /* Only objects marked with the extcounted bit are included in this total. Pages which we scan for possible pageout, but which are not actually @@ -190,6 +194,8 @@ int vm_page_external_limit; /* Max number of pages for external objects */ in this way. */ extern int vm_page_external_count; /* How many pages for external objects? */ +extern +int vm_page_external_target;/* How many external pages do we want? */ diff --git a/vm/vm_pageout.c b/vm/vm_pageout.c index 7a755bf4..7ac62876 100644 --- a/vm/vm_pageout.c +++ b/vm/vm_pageout.c @@ -97,7 +97,7 @@ */ #ifndef VM_PAGE_FREE_TARGET -#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80) +#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 20) #endif /* VM_PAGE_FREE_TARGET */ /* @@ -106,7 +106,7 @@ */ #ifndef VM_PAGE_FREE_MIN -#define VM_PAGE_FREE_MIN(free) (10 + (free) / 100) +#define VM_PAGE_FREE_MIN(free) (10 + (free) / 40) #endif /* VM_PAGE_FREE_MIN */ /* When vm_page_external_count exceeds vm_page_external_limit, @@ -114,7 +114,7 @@ */ #ifndef VM_PAGE_EXTERNAL_LIMIT -#define VM_PAGE_EXTERNAL_LIMIT(free) ((free) / 2) +#define VM_PAGE_EXTERNAL_LIMIT(free) ((free) / 10) #endif /* VM_PAGE_EXTERNAL_LIMIT */ /* Attempt to keep the number of externally paged pages less @@ -165,7 +165,7 @@ extern void vm_pageout_scan_continue(); unsigned int vm_pageout_reserved_internal = 0; unsigned int vm_pageout_reserved_really = 0; -unsigned int vm_page_external_target = 0; +int vm_page_external_target = 0; unsigned int vm_pageout_burst_max = 0; unsigned int vm_pageout_burst_min = 0; @@ -313,10 +313,35 @@ vm_pageout_setup(m, paging_offset, new_object, new_offset, flush) pmap_clear_modify(m->phys_addr); /* + * If this is an external object, + * write lock the page so a new + * unlock request is generated if + * it's going to become dirty. + */ + if (!old_object->internal) + m->page_lock = VM_PROT_WRITE; + + /* * Deactivate old page. */ vm_page_lock_queues(); vm_page_deactivate(m); + + /* + * If this is an external page, + * and we're really keeping track + * of it, adjust dirty count. + */ + if (m->external && m->overwriting) { + m->overwriting = FALSE; + vm_page_dirty_count--; + if (vm_page_too_dirty) { + vm_page_too_dirty--; + if (vm_page_too_dirty == 0) + thread_wakeup((event_t)&vm_page_too_dirty); + } + } + vm_page_unlock_queues(); PAGE_WAKEUP_DONE(m); @@ -340,6 +365,7 @@ vm_pageout_setup(m, paging_offset, new_object, new_offset, flush) */ m = new_m; m->dirty = TRUE; + m->external = FALSE; assert(!m->precious); PAGE_WAKEUP_DONE(m); } @@ -514,6 +540,10 @@ void vm_pageout_scan() { unsigned int burst_count; unsigned int want_pages; + unsigned int external_available; + int current_target; + boolean_t found = FALSE; + boolean_t first_run = TRUE; /* * We want to gradually dribble pages from the active queue @@ -640,8 +670,92 @@ void vm_pageout_scan() } want_pages = ((free_count < vm_page_free_target) || vm_page_free_wanted); + current_target = vm_page_free_target - free_count; + external_available = vm_page_external_count - vm_page_external_limit; simple_unlock(&vm_page_queue_free_lock); + if (first_run && + current_target > 0 && + external_available > (current_target / 2)) { + int target = external_available > current_target ? + current_target : external_available; + int iters = 0; + + first_run = FALSE; + printf("ec: t=%u, ct=%u, ea=%u, td=%d\n", target, current_target, + external_available, vm_page_too_dirty); + m = (vm_page_t) queue_first(&vm_page_queue_inactive); + + while (target && + (!queue_end(&vm_page_queue_inactive, (queue_entry_t) m))) { + vm_page_t old_page; + + iters++; + + if (!m->external) { + m = (vm_page_t) queue_next(&m->pageq); + continue; + } + + object = m->object; + if (!vm_object_lock_try(object)) { + m = (vm_page_t) queue_next(&m->pageq); + continue; + } + + if (!m->dirty) + m->dirty = pmap_is_modified(m->phys_addr); + + if (!m->dirty && !m->precious && + !m->absent && !m->busy) { + /* This page can be safely freed */ + + old_page = m; + m = (vm_page_t) queue_next(&old_page->pageq); + old_page->busy = TRUE; + pmap_page_protect(old_page->phys_addr, VM_PROT_NONE); + VM_PAGE_FREE(old_page); + target--; + current_target--; + } + else { + /* Page is not ready to be freed. Try next */ + m = (vm_page_t) queue_next(&m->pageq); + } + vm_object_unlock(object); + } + + printf("external_count: end target=%d, iters=%d, ic=%d\n", + target, iters, vm_page_inactive_count); + + if (target || current_target) { + printf("Can't free enough external pages. Using old method\n"); + + simple_lock(&vm_page_queue_free_lock); + free_count = vm_page_free_count; + if ((free_count >= vm_page_free_target) && + (vm_page_external_count <= vm_page_external_target) && + (vm_page_free_wanted == 0)) { + vm_page_unlock_queues(); + break; + } + + want_pages = ((free_count < vm_page_free_target) || vm_page_free_wanted); + + simple_unlock(&vm_page_queue_free_lock); + } + else { + vm_page_unlock_queues(); + break; + } + } + else { + int target = external_available > current_target ? + current_target : external_available; + printf("INTERNAL ec: t=%u ct=%u ea=%u td=%d\n", target, current_target, + external_available, vm_page_too_dirty); + } + /* * Sometimes we have to pause: * 1) No inactive pages - nothing to do. @@ -651,13 +765,19 @@ void vm_pageout_scan() * this if the default pager already has work to do. */ pause: - if (queue_empty(&vm_page_queue_inactive) || + if (found == 2 || + queue_empty(&vm_page_queue_inactive) || (burst_count >= vm_pageout_burst_max) || (vm_page_laundry_count >= vm_pageout_burst_max) || ((free_count < vm_pageout_reserved_really) && (vm_page_laundry_count > 0))) { unsigned int pages, msecs; + if (!want_pages) { + vm_page_unlock_queues(); + break; + } + /* * vm_pageout_burst_wait is msecs/page. * If there is nothing for us to do, we wait @@ -687,17 +807,24 @@ void vm_pageout_scan() /* Find a page we are interested in paging out. If we need pages, then we'll page anything out; otherwise we only page out external pages. */ + found = 0; m = (vm_page_t) queue_first (&vm_page_queue_inactive); - while (1) + while (!queue_end(&vm_page_queue_inactive, (queue_entry_t) m)) { assert (!m->active && m->inactive); - if (want_pages || m->external) + if (!m->external) { + found = 1; break; + } - m = (vm_page_t) queue_next (m); - if (!m) - goto pause; + m = (vm_page_t) queue_next(&m->pageq); } + + if (!found) { + printf("Can't find an internal page to flush\n"); + found = 2; + goto pause; + } object = m->object; @@ -777,6 +904,7 @@ void vm_pageout_scan() if (!m->dirty) m->dirty = pmap_is_modified(m->phys_addr); +#if 0 if (m->external) { /* Figure out if we still care about this page in the limit of externally managed pages. @@ -804,6 +932,7 @@ void vm_pageout_scan() vm_pageout_inactive_cleaned_external++; continue; } +#endif /* * If it's clean and not precious, we can free the page. diff --git a/vm/vm_resident.c b/vm/vm_resident.c index 96354a45..d3db6869 100644 --- a/vm/vm_resident.c +++ b/vm/vm_resident.c @@ -162,6 +162,8 @@ int vm_page_inactive_target = 0; int vm_page_free_reserved = 0; int vm_page_laundry_count = 0; int vm_page_external_limit = 0; +int vm_page_too_dirty = 0; +int vm_page_dirty_count = 0; /* @@ -207,6 +209,7 @@ void vm_page_bootstrap( m->laundry = FALSE; m->free = FALSE; m->external = FALSE; + m->overwriting = FALSE; m->busy = TRUE; m->wanted = FALSE; @@ -838,6 +841,7 @@ boolean_t vm_page_convert( m->phys_addr = real_m->phys_addr; m->fictitious = FALSE; + m->external = external; real_m->phys_addr = vm_page_fictitious_addr; real_m->fictitious = TRUE; @@ -866,10 +870,8 @@ vm_page_t vm_page_grab( * for externally-managed pages. */ - if (((vm_page_free_count < vm_page_free_reserved) - || (external - && (vm_page_external_count > vm_page_external_limit))) - && !current_thread()->vm_privilege) { + if ((vm_page_free_count < vm_page_free_reserved) && + !current_thread()->vm_privilege) { simple_unlock(&vm_page_queue_free_lock); return VM_PAGE_NULL; } @@ -979,8 +981,7 @@ vm_page_grab_contiguous_pages( * Do not dip into the reserved pool. */ - if ((vm_page_free_count < vm_page_free_reserved) - || (vm_page_external_count >= vm_page_external_limit)) { + if (vm_page_free_count < vm_page_free_reserved) { printf_once("no more room for vm_page_grab_contiguous_pages"); simple_unlock(&vm_page_queue_free_lock); return KERN_RESOURCE_SHORTAGE; @@ -1163,8 +1164,6 @@ void vm_page_release( mem->pageq.next = (queue_entry_t) vm_page_queue_free; vm_page_queue_free = mem; vm_page_free_count++; - if (external) - vm_page_external_count--; /* * Check if we should wake up someone waiting for page. @@ -1215,8 +1214,7 @@ void vm_page_wait( */ simple_lock(&vm_page_queue_free_lock); - if ((vm_page_free_count < vm_page_free_target) - || (vm_page_external_count > vm_page_external_limit)) { + if (vm_page_free_count < vm_page_free_target) { if (vm_page_free_wanted++ == 0) thread_wakeup((event_t)&vm_page_free_wanted); assert_wait((event_t)&vm_page_free_count, FALSE); @@ -1292,6 +1290,19 @@ void vm_page_free( if (mem->absent) vm_object_absent_release(mem->object); + if (mem->external) { + vm_page_external_count--; + mem->external = FALSE; + if (mem->overwriting) { + vm_page_dirty_count--; + if (vm_page_too_dirty) { + vm_page_too_dirty--; + if (vm_page_too_dirty == 0) + thread_wakeup((event_t)&vm_page_too_dirty); + } + } + } + /* * XXX The calls to vm_page_init here are * really overkill. |