diff options
Diffstat (limited to 'vm/vm_map.c')
-rw-r--r-- | vm/vm_map.c | 353 |
1 files changed, 228 insertions, 125 deletions
diff --git a/vm/vm_map.c b/vm/vm_map.c index 7fe3e141..7db76b7b 100644 --- a/vm/vm_map.c +++ b/vm/vm_map.c @@ -43,6 +43,7 @@ #include <kern/assert.h> #include <kern/debug.h> #include <kern/kalloc.h> +#include <kern/mach.server.h> #include <kern/list.h> #include <kern/rbtree.h> #include <kern/slab.h> @@ -112,8 +113,7 @@ MACRO_END * start or end value.] Note that these clippings may not * always be necessary (as the two resulting entries are then * not changed); however, the clipping is done for convenience. - * No attempt is currently made to "glue back together" two - * abutting entries. + * The entries can later be "glued back together" (coalesced). * * The symmetric (shadow) copy strategy implements virtual copy * by copying VM object references from one map to @@ -280,8 +280,8 @@ void vm_map_unlock(struct vm_map *map) #define vm_map_copy_entry_create(copy) \ _vm_map_entry_create(&(copy)->cpy_hdr) -vm_map_entry_t _vm_map_entry_create(map_header) - const struct vm_map_header *map_header; +static vm_map_entry_t +_vm_map_entry_create(const struct vm_map_header *map_header) { vm_map_entry_t entry; @@ -303,9 +303,9 @@ vm_map_entry_t _vm_map_entry_create(map_header) #define vm_map_copy_entry_dispose(map, entry) \ _vm_map_entry_dispose(&(copy)->cpy_hdr, (entry)) -void _vm_map_entry_dispose(map_header, entry) - const struct vm_map_header *map_header; - vm_map_entry_t entry; +static void +_vm_map_entry_dispose(const struct vm_map_header *map_header, + vm_map_entry_t entry) { (void)map_header; @@ -551,10 +551,12 @@ void vm_map_deallocate(vm_map_t map) c = --map->ref_count; simple_unlock(&map->ref_lock); + /* Check the refcount */ if (c > 0) { return; } + /* If no more references, call vm_map_delete without locking the map */ projected_buffer_collect(map); (void) vm_map_delete(map, map->min_offset, map->max_offset); @@ -635,27 +637,6 @@ boolean_t vm_map_lookup_entry( } /* - * Routine: invalid_user_access - * - * Verifies whether user access is valid. - */ - -boolean_t -invalid_user_access( - vm_map_t map, - vm_offset_t start, - vm_offset_t end, - vm_prot_t prot) -{ - vm_map_entry_t entry; - - return (map == VM_MAP_NULL || map == kernel_map || - !vm_map_lookup_entry(map, start, &entry) || - entry->vme_end < end || - (prot & ~(entry->protection))); -} - -/* * Find a range of available space from the specified map. * * If successful, this function returns the map entry immediately preceding @@ -685,16 +666,16 @@ vm_map_find_entry_anywhere(struct vm_map *map, if (((mask + 1) & mask) != 0) { /* We have high bits in addition to the low bits */ - int first0 = ffs(~mask); /* First zero after low bits */ + int first0 = __builtin_ffs(~mask); /* First zero after low bits */ vm_offset_t lowmask = (1UL << (first0-1)) - 1; /* low bits */ vm_offset_t himask = mask - lowmask; /* high bits */ - int second1 = ffs(himask); /* First one after low bits */ + int second1 = __builtin_ffs(himask); /* First one after low bits */ max = 1UL << (second1-1); if (himask + max != 0) { /* high bits do not continue up to the end */ - printf("invalid mask %lx\n", mask); + printf("invalid mask %zx\n", mask); return NULL; } @@ -737,7 +718,7 @@ restart: max_size = size + mask; if (max_size < size) { - printf("max_size %x got smaller than size %x with mask %lx\n", + printf("max_size %zd got smaller than size %zd with mask %zd\n", max_size, size, mask); goto error; } @@ -773,7 +754,7 @@ restart: assert(end <= (entry->vme_end + entry->gap_size)); if (end > max) { /* Does not respect the allowed maximum */ - printf("%lx does not respect %lx\n", end, max); + printf("%zx does not respect %zx\n", end, max); return NULL; } *startp = start; @@ -915,7 +896,7 @@ boolean_t vm_map_pmap_enter_enable = FALSE; * In/out conditions: * The source map should not be locked on entry. */ -void +static void vm_map_pmap_enter( vm_map_t map, vm_offset_t addr, @@ -939,7 +920,7 @@ vm_map_pmap_enter( if (vm_map_pmap_enter_print) { printf("vm_map_pmap_enter:"); - printf("map: %p, addr: %lx, object: %p, offset: %lx\n", + printf("map: %p, addr: %zx, object: %p, offset: %zx\n", map, addr, object, offset); } @@ -987,6 +968,7 @@ kern_return_t vm_map_enter( vm_inherit_t inheritance) { vm_map_entry_t entry; + vm_map_entry_t next_entry; vm_offset_t start; vm_offset_t end; kern_return_t result = KERN_SUCCESS; @@ -1007,6 +989,7 @@ kern_return_t vm_map_enter( end = start + size; *address = start; + next_entry = entry->vme_next; } else { vm_map_entry_t temp_entry; @@ -1041,14 +1024,15 @@ kern_return_t vm_map_enter( RETURN(KERN_NO_SPACE); entry = temp_entry; + next_entry = entry->vme_next; /* * ... the next region doesn't overlap the * end point. */ - if ((entry->vme_next != vm_map_to_entry(map)) && - (entry->vme_next->vme_start < end)) + if ((next_entry != vm_map_to_entry(map)) && + (next_entry->vme_start < end)) RETURN(KERN_NO_SPACE); } @@ -1064,12 +1048,10 @@ kern_return_t vm_map_enter( /* * See whether we can avoid creating a new entry (and object) by - * extending one of our neighbors. [So far, we only attempt to - * extend from below.] + * extending one of our neighbors. */ - if ((object == VM_OBJECT_NULL) && - (entry != vm_map_to_entry(map)) && + if ((entry != vm_map_to_entry(map)) && (entry->vme_end == start) && (!entry->is_shared) && (!entry->is_sub_map) && @@ -1079,20 +1061,63 @@ kern_return_t vm_map_enter( (entry->wired_count == 0) && (entry->projected_on == 0)) { if (vm_object_coalesce(entry->object.vm_object, - VM_OBJECT_NULL, + object, entry->offset, - (vm_offset_t) 0, + offset, (vm_size_t)(entry->vme_end - entry->vme_start), - (vm_size_t)(end - entry->vme_end))) { + size, + &entry->object.vm_object, + &entry->offset)) { /* * Coalesced the two objects - can extend * the previous map entry to include the * new range. */ - map->size += (end - entry->vme_end); + map->size += size; entry->vme_end = end; vm_map_gap_update(&map->hdr, entry); + /* + * Now that we did, perhaps we could simplify + * things even further by coalescing the next + * entry into the one we just extended. + */ + vm_map_coalesce_entry(map, next_entry); + RETURN(KERN_SUCCESS); + } + } + if ((next_entry != vm_map_to_entry(map)) && + (next_entry->vme_start == end) && + (!next_entry->is_shared) && + (!next_entry->is_sub_map) && + (next_entry->inheritance == inheritance) && + (next_entry->protection == cur_protection) && + (next_entry->max_protection == max_protection) && + (next_entry->wired_count == 0) && + (next_entry->projected_on == 0)) { + if (vm_object_coalesce(object, + next_entry->object.vm_object, + offset, + next_entry->offset, + size, + (vm_size_t)(next_entry->vme_end - next_entry->vme_start), + &next_entry->object.vm_object, + &next_entry->offset)) { + + /* + * Coalesced the two objects - can extend + * the next map entry to include the + * new range. + */ + map->size += size; + next_entry->vme_start = start; + vm_map_gap_update(&map->hdr, entry); + /* + * Now that we did, perhaps we could simplify + * things even further by coalescing the + * entry into the previous one. + */ + vm_map_coalesce_entry(map, next_entry); RETURN(KERN_SUCCESS); } } @@ -1571,6 +1596,7 @@ kern_return_t vm_map_protect( { vm_map_entry_t current; vm_map_entry_t entry; + vm_map_entry_t next; vm_map_lock(map); @@ -1632,7 +1658,8 @@ kern_return_t vm_map_protect( */ if ((current->protection != VM_PROT_NONE) && - (current->wired_access != VM_PROT_NONE)) { + (current->wired_access != VM_PROT_NONE || + map->wiring_required)) { current->wired_access = current->protection; } @@ -1645,9 +1672,16 @@ kern_return_t vm_map_protect( current->vme_end, current->protection); } - current = current->vme_next; + + next = current->vme_next; + vm_map_coalesce_entry(map, current); + current = next; } + next = current->vme_next; + if (vm_map_coalesce_entry(map, current)) + current = next; + /* Returns with the map read-locked if successful */ vm_map_pageable_scan(map, entry, current); @@ -1671,6 +1705,7 @@ kern_return_t vm_map_inherit( { vm_map_entry_t entry; vm_map_entry_t temp_entry; + vm_map_entry_t next; vm_map_lock(map); @@ -1688,9 +1723,13 @@ kern_return_t vm_map_inherit( entry->inheritance = new_inheritance; - entry = entry->vme_next; + next = entry->vme_next; + vm_map_coalesce_entry(map, entry); + entry = next; } + vm_map_coalesce_entry(map, entry); + vm_map_unlock(map); return(KERN_SUCCESS); } @@ -1792,6 +1831,30 @@ kern_return_t vm_map_pageable( return(KERN_SUCCESS); } +/* Update pageability of all the memory currently in the map. + * The map must be locked, and protection mismatch will not be checked, see + * vm_map_pageable(). + */ +static kern_return_t +vm_map_pageable_current(vm_map_t map, vm_prot_t access_type) +{ + struct rbtree_node *node; + vm_offset_t min_address, max_address; + + node = rbtree_first(&map->hdr.tree); + min_address = rbtree_entry(node, struct vm_map_entry, + tree_node)->vme_start; + + node = rbtree_last(&map->hdr.tree); + max_address = rbtree_entry(node, struct vm_map_entry, + tree_node)->vme_end; + + /* Returns with the map read-locked if successful */ + return vm_map_pageable(map, min_address, max_address,access_type, + FALSE, FALSE); +} + + /* * vm_map_pageable_all: * @@ -1822,8 +1885,7 @@ vm_map_pageable_all(struct vm_map *map, vm_wire_t flags) map->wiring_required = FALSE; /* Returns with the map read-locked if successful */ - kr = vm_map_pageable(map, map->min_offset, map->max_offset, - VM_PROT_NONE, FALSE, FALSE); + kr = vm_map_pageable_current(map, VM_PROT_NONE); vm_map_unlock(map); return kr; } @@ -1836,9 +1898,7 @@ vm_map_pageable_all(struct vm_map *map, vm_wire_t flags) if (flags & VM_WIRE_CURRENT) { /* Returns with the map read-locked if successful */ - kr = vm_map_pageable(map, map->min_offset, map->max_offset, - VM_PROT_READ | VM_PROT_WRITE, - FALSE, FALSE); + kr = vm_map_pageable_current(map, VM_PROT_READ | VM_PROT_WRITE); if (kr != KERN_SUCCESS) { if (flags & VM_WIRE_FUTURE) { @@ -1865,11 +1925,13 @@ void vm_map_entry_delete( vm_map_entry_t entry) { vm_offset_t s, e; + vm_size_t size; vm_object_t object; extern vm_object_t kernel_object; s = entry->vme_start; e = entry->vme_end; + size = e - s; /*Check if projected buffer*/ if (map != kernel_map && entry->projected_on != 0) { @@ -1908,15 +1970,29 @@ void vm_map_entry_delete( if (object == kernel_object) { vm_object_lock(object); vm_object_page_remove(object, entry->offset, - entry->offset + (e - s)); + entry->offset + size); vm_object_unlock(object); } else if (entry->is_shared) { vm_object_pmap_remove(object, entry->offset, - entry->offset + (e - s)); - } - else { + entry->offset + size); + } else { pmap_remove(map->pmap, s, e); + /* + * If this object has no pager and our + * reference to the object is the only + * one, we can release the deleted pages + * now. + */ + vm_object_lock(object); + if ((!object->pager_created) && + (object->ref_count == 1) && + (object->paging_in_progress == 0)) { + vm_object_page_remove(object, + entry->offset, + entry->offset + size); + } + vm_object_unlock(object); } } @@ -1931,7 +2007,7 @@ void vm_map_entry_delete( vm_object_deallocate(entry->object.vm_object); vm_map_entry_unlink(map, entry); - map->size -= e - s; + map->size -= size; vm_map_entry_dispose(map, entry); } @@ -1951,6 +2027,14 @@ kern_return_t vm_map_delete( vm_map_entry_t entry; vm_map_entry_t first_entry; + if (map->pmap == kernel_pmap && (start < kernel_virtual_start || end > kernel_virtual_end)) + panic("vm_map_delete(%lx-%lx) falls in physical memory area!\n", (unsigned long) start, (unsigned long) end); + + /* + * Must be called with map lock taken unless refcount is zero + */ + assert((map->ref_count > 0 && have_lock(map->lock)) || (map->ref_count == 0)); + /* * Find the start of the region, and clip it */ @@ -2048,7 +2132,7 @@ kern_return_t vm_map_remove( * Steal all the pages from a vm_map_copy page_list by copying ones * that have not already been stolen. */ -void +static void vm_map_copy_steal_pages(vm_map_copy_t copy) { vm_page_t m, new_m; @@ -3674,7 +3758,7 @@ kern_return_t vm_map_copyin_object( * the scheduler. */ -kern_return_t vm_map_copyin_page_list_cont( +static kern_return_t vm_map_copyin_page_list_cont( vm_map_copyin_args_t cont_args, vm_map_copy_t *copy_result) /* OUT */ { @@ -3809,7 +3893,7 @@ kern_return_t vm_map_copyin_page_list( copy->offset = src_addr; copy->size = len; copy->cpy_cont = ((kern_return_t (*)()) 0); - copy->cpy_cont_args = (char *) VM_MAP_COPYIN_ARGS_NULL; + copy->cpy_cont_args = VM_MAP_COPYIN_ARGS_NULL; /* * Find the beginning of the region. @@ -3899,7 +3983,7 @@ make_continuation: } cont_args->steal_pages = steal_pages; - copy->cpy_cont_args = (char *) cont_args; + copy->cpy_cont_args = cont_args; copy->cpy_cont = vm_map_copyin_page_list_cont; src_end = src_start; @@ -4238,7 +4322,7 @@ retry: cont_args->destroy_len = src_end - src_start; cont_args->steal_pages = FALSE; - copy->cpy_cont_args = (char *) cont_args; + copy->cpy_cont_args = cont_args; copy->cpy_cont = vm_map_copyin_page_list_cont; } @@ -4530,8 +4614,9 @@ vm_map_t vm_map_fork(vm_map_t old_map) * In order to later verify this lookup, a "version" * is returned. * - * The map should not be locked; it will not be - * locked on exit. In order to guarantee the + * The map should not be locked; it will be + * unlocked on exit unless keep_map_locked is set and + * the lookup succeeds. In order to guarantee the * existence of the returned object, it is returned * locked. * @@ -4544,6 +4629,7 @@ kern_return_t vm_map_lookup( vm_map_t *var_map, /* IN/OUT */ vm_offset_t vaddr, vm_prot_t fault_type, + boolean_t keep_map_locked, vm_map_version_t *out_version, /* OUT */ vm_object_t *object, /* OUT */ @@ -4565,7 +4651,8 @@ kern_return_t vm_map_lookup( #define RETURN(why) \ { \ - vm_map_unlock_read(map); \ + if (!(keep_map_locked && (why == KERN_SUCCESS))) \ + vm_map_unlock_read(map); \ return(why); \ } @@ -4823,7 +4910,8 @@ vm_region_create_proxy (task_t task, vm_address_t address, kern_return_t ret; vm_map_entry_t entry, tmp_entry; vm_object_t object; - vm_offset_t offset, start; + rpc_vm_offset_t rpc_offset, rpc_start; + rpc_vm_size_t rpc_len = (rpc_vm_size_t) len; ipc_port_t pager; if (task == TASK_NULL) @@ -4859,16 +4947,16 @@ vm_region_create_proxy (task_t task, vm_address_t address, pager = ipc_port_copy_send(object->pager); vm_object_unlock(object); - start = (address - entry->vme_start) + entry->offset; - offset = 0; + rpc_start = (address - entry->vme_start) + entry->offset; + rpc_offset = 0; vm_map_unlock_read(task->map); ret = memory_object_create_proxy(task->itk_space, max_protection, &pager, 1, - &offset, 1, - &start, 1, - &len, 1, port); + &rpc_offset, 1, + &rpc_start, 1, + &rpc_len, 1, port); if (ret) ipc_port_release_send(pager); @@ -4876,64 +4964,80 @@ vm_region_create_proxy (task_t task, vm_address_t address, } /* - * Routine: vm_map_simplify - * - * Description: - * Attempt to simplify the map representation in - * the vicinity of the given starting address. - * Note: - * This routine is intended primarily to keep the - * kernel maps more compact -- they generally don't - * benefit from the "expand a map entry" technology - * at allocation time because the adjacent entry - * is often wired down. + * Routine: vm_map_coalesce_entry + * Purpose: + * Try to coalesce an entry with the preceeding entry in the map. + * Conditions: + * The map is locked. If coalesced, the entry is destroyed + * by the call. + * Returns: + * Whether the entry was coalesced. */ -void vm_map_simplify( +boolean_t +vm_map_coalesce_entry( vm_map_t map, - vm_offset_t start) + vm_map_entry_t entry) { - vm_map_entry_t this_entry; - vm_map_entry_t prev_entry; + vm_map_entry_t prev = entry->vme_prev; + vm_size_t prev_size; + vm_size_t entry_size; - vm_map_lock(map); - if ( - (vm_map_lookup_entry(map, start, &this_entry)) && - ((prev_entry = this_entry->vme_prev) != vm_map_to_entry(map)) && - - (prev_entry->vme_end == start) && - - (prev_entry->is_shared == FALSE) && - (prev_entry->is_sub_map == FALSE) && - - (this_entry->is_shared == FALSE) && - (this_entry->is_sub_map == FALSE) && - - (prev_entry->inheritance == this_entry->inheritance) && - (prev_entry->protection == this_entry->protection) && - (prev_entry->max_protection == this_entry->max_protection) && - (prev_entry->wired_count == this_entry->wired_count) && - - (prev_entry->needs_copy == this_entry->needs_copy) && - - (prev_entry->object.vm_object == this_entry->object.vm_object) && - ((prev_entry->offset + (prev_entry->vme_end - prev_entry->vme_start)) - == this_entry->offset) && - (prev_entry->projected_on == 0) && - (this_entry->projected_on == 0) - ) { - if (map->first_free == this_entry) - map->first_free = prev_entry; - - SAVE_HINT(map, prev_entry); - prev_entry->vme_end = this_entry->vme_end; - vm_map_entry_unlink(map, this_entry); - vm_object_deallocate(this_entry->object.vm_object); - vm_map_entry_dispose(map, this_entry); - } - vm_map_unlock(map); + /* + * Check the basic conditions for coalescing the two entries. + */ + if ((entry == vm_map_to_entry(map)) || + (prev == vm_map_to_entry(map)) || + (prev->vme_end != entry->vme_start) || + (prev->is_shared || entry->is_shared) || + (prev->is_sub_map || entry->is_sub_map) || + (prev->inheritance != entry->inheritance) || + (prev->protection != entry->protection) || + (prev->max_protection != entry->max_protection) || + (prev->needs_copy != entry->needs_copy) || + (prev->in_transition || entry->in_transition) || + (prev->wired_count != entry->wired_count) || + (prev->projected_on != 0) || + (entry->projected_on != 0)) + return FALSE; + + prev_size = prev->vme_end - prev->vme_start; + entry_size = entry->vme_end - entry->vme_start; + assert(prev->gap_size == 0); + + /* + * See if we can coalesce the two objects. + */ + if (!vm_object_coalesce(prev->object.vm_object, + entry->object.vm_object, + prev->offset, + entry->offset, + prev_size, + entry_size, + &prev->object.vm_object, + &prev->offset)) + return FALSE; + + /* + * Update the hints. + */ + if (map->hint == entry) + SAVE_HINT(map, prev); + if (map->first_free == entry) + map->first_free = prev; + + /* + * Get rid of the entry without changing any wirings or the pmap, + * and without altering map->size. + */ + prev->vme_end = entry->vme_end; + vm_map_entry_unlink(map, entry); + vm_map_entry_dispose(map, entry); + + return TRUE; } + /* * Routine: vm_map_machine_attribute * Purpose: @@ -5078,8 +5182,7 @@ void vm_map_print(db_expr_t addr, boolean_t have_addr, db_expr_t count, const ch * Pretty-print a copy object for ddb. */ -void vm_map_copy_print(copy) - const vm_map_copy_t copy; +void vm_map_copy_print(const vm_map_copy_t copy) { int i, npages; |