summaryrefslogtreecommitdiff
path: root/vm/vm_map.c
diff options
context:
space:
mode:
Diffstat (limited to 'vm/vm_map.c')
-rw-r--r--vm/vm_map.c353
1 files changed, 228 insertions, 125 deletions
diff --git a/vm/vm_map.c b/vm/vm_map.c
index 7fe3e141..7db76b7b 100644
--- a/vm/vm_map.c
+++ b/vm/vm_map.c
@@ -43,6 +43,7 @@
#include <kern/assert.h>
#include <kern/debug.h>
#include <kern/kalloc.h>
+#include <kern/mach.server.h>
#include <kern/list.h>
#include <kern/rbtree.h>
#include <kern/slab.h>
@@ -112,8 +113,7 @@ MACRO_END
* start or end value.] Note that these clippings may not
* always be necessary (as the two resulting entries are then
* not changed); however, the clipping is done for convenience.
- * No attempt is currently made to "glue back together" two
- * abutting entries.
+ * The entries can later be "glued back together" (coalesced).
*
* The symmetric (shadow) copy strategy implements virtual copy
* by copying VM object references from one map to
@@ -280,8 +280,8 @@ void vm_map_unlock(struct vm_map *map)
#define vm_map_copy_entry_create(copy) \
_vm_map_entry_create(&(copy)->cpy_hdr)
-vm_map_entry_t _vm_map_entry_create(map_header)
- const struct vm_map_header *map_header;
+static vm_map_entry_t
+_vm_map_entry_create(const struct vm_map_header *map_header)
{
vm_map_entry_t entry;
@@ -303,9 +303,9 @@ vm_map_entry_t _vm_map_entry_create(map_header)
#define vm_map_copy_entry_dispose(map, entry) \
_vm_map_entry_dispose(&(copy)->cpy_hdr, (entry))
-void _vm_map_entry_dispose(map_header, entry)
- const struct vm_map_header *map_header;
- vm_map_entry_t entry;
+static void
+_vm_map_entry_dispose(const struct vm_map_header *map_header,
+ vm_map_entry_t entry)
{
(void)map_header;
@@ -551,10 +551,12 @@ void vm_map_deallocate(vm_map_t map)
c = --map->ref_count;
simple_unlock(&map->ref_lock);
+ /* Check the refcount */
if (c > 0) {
return;
}
+ /* If no more references, call vm_map_delete without locking the map */
projected_buffer_collect(map);
(void) vm_map_delete(map, map->min_offset, map->max_offset);
@@ -635,27 +637,6 @@ boolean_t vm_map_lookup_entry(
}
/*
- * Routine: invalid_user_access
- *
- * Verifies whether user access is valid.
- */
-
-boolean_t
-invalid_user_access(
- vm_map_t map,
- vm_offset_t start,
- vm_offset_t end,
- vm_prot_t prot)
-{
- vm_map_entry_t entry;
-
- return (map == VM_MAP_NULL || map == kernel_map ||
- !vm_map_lookup_entry(map, start, &entry) ||
- entry->vme_end < end ||
- (prot & ~(entry->protection)));
-}
-
-/*
* Find a range of available space from the specified map.
*
* If successful, this function returns the map entry immediately preceding
@@ -685,16 +666,16 @@ vm_map_find_entry_anywhere(struct vm_map *map,
if (((mask + 1) & mask) != 0) {
/* We have high bits in addition to the low bits */
- int first0 = ffs(~mask); /* First zero after low bits */
+ int first0 = __builtin_ffs(~mask); /* First zero after low bits */
vm_offset_t lowmask = (1UL << (first0-1)) - 1; /* low bits */
vm_offset_t himask = mask - lowmask; /* high bits */
- int second1 = ffs(himask); /* First one after low bits */
+ int second1 = __builtin_ffs(himask); /* First one after low bits */
max = 1UL << (second1-1);
if (himask + max != 0) {
/* high bits do not continue up to the end */
- printf("invalid mask %lx\n", mask);
+ printf("invalid mask %zx\n", mask);
return NULL;
}
@@ -737,7 +718,7 @@ restart:
max_size = size + mask;
if (max_size < size) {
- printf("max_size %x got smaller than size %x with mask %lx\n",
+ printf("max_size %zd got smaller than size %zd with mask %zd\n",
max_size, size, mask);
goto error;
}
@@ -773,7 +754,7 @@ restart:
assert(end <= (entry->vme_end + entry->gap_size));
if (end > max) {
/* Does not respect the allowed maximum */
- printf("%lx does not respect %lx\n", end, max);
+ printf("%zx does not respect %zx\n", end, max);
return NULL;
}
*startp = start;
@@ -915,7 +896,7 @@ boolean_t vm_map_pmap_enter_enable = FALSE;
* In/out conditions:
* The source map should not be locked on entry.
*/
-void
+static void
vm_map_pmap_enter(
vm_map_t map,
vm_offset_t addr,
@@ -939,7 +920,7 @@ vm_map_pmap_enter(
if (vm_map_pmap_enter_print) {
printf("vm_map_pmap_enter:");
- printf("map: %p, addr: %lx, object: %p, offset: %lx\n",
+ printf("map: %p, addr: %zx, object: %p, offset: %zx\n",
map, addr, object, offset);
}
@@ -987,6 +968,7 @@ kern_return_t vm_map_enter(
vm_inherit_t inheritance)
{
vm_map_entry_t entry;
+ vm_map_entry_t next_entry;
vm_offset_t start;
vm_offset_t end;
kern_return_t result = KERN_SUCCESS;
@@ -1007,6 +989,7 @@ kern_return_t vm_map_enter(
end = start + size;
*address = start;
+ next_entry = entry->vme_next;
} else {
vm_map_entry_t temp_entry;
@@ -1041,14 +1024,15 @@ kern_return_t vm_map_enter(
RETURN(KERN_NO_SPACE);
entry = temp_entry;
+ next_entry = entry->vme_next;
/*
* ... the next region doesn't overlap the
* end point.
*/
- if ((entry->vme_next != vm_map_to_entry(map)) &&
- (entry->vme_next->vme_start < end))
+ if ((next_entry != vm_map_to_entry(map)) &&
+ (next_entry->vme_start < end))
RETURN(KERN_NO_SPACE);
}
@@ -1064,12 +1048,10 @@ kern_return_t vm_map_enter(
/*
* See whether we can avoid creating a new entry (and object) by
- * extending one of our neighbors. [So far, we only attempt to
- * extend from below.]
+ * extending one of our neighbors.
*/
- if ((object == VM_OBJECT_NULL) &&
- (entry != vm_map_to_entry(map)) &&
+ if ((entry != vm_map_to_entry(map)) &&
(entry->vme_end == start) &&
(!entry->is_shared) &&
(!entry->is_sub_map) &&
@@ -1079,20 +1061,63 @@ kern_return_t vm_map_enter(
(entry->wired_count == 0) &&
(entry->projected_on == 0)) {
if (vm_object_coalesce(entry->object.vm_object,
- VM_OBJECT_NULL,
+ object,
entry->offset,
- (vm_offset_t) 0,
+ offset,
(vm_size_t)(entry->vme_end - entry->vme_start),
- (vm_size_t)(end - entry->vme_end))) {
+ size,
+ &entry->object.vm_object,
+ &entry->offset)) {
/*
* Coalesced the two objects - can extend
* the previous map entry to include the
* new range.
*/
- map->size += (end - entry->vme_end);
+ map->size += size;
entry->vme_end = end;
vm_map_gap_update(&map->hdr, entry);
+ /*
+ * Now that we did, perhaps we could simplify
+ * things even further by coalescing the next
+ * entry into the one we just extended.
+ */
+ vm_map_coalesce_entry(map, next_entry);
+ RETURN(KERN_SUCCESS);
+ }
+ }
+ if ((next_entry != vm_map_to_entry(map)) &&
+ (next_entry->vme_start == end) &&
+ (!next_entry->is_shared) &&
+ (!next_entry->is_sub_map) &&
+ (next_entry->inheritance == inheritance) &&
+ (next_entry->protection == cur_protection) &&
+ (next_entry->max_protection == max_protection) &&
+ (next_entry->wired_count == 0) &&
+ (next_entry->projected_on == 0)) {
+ if (vm_object_coalesce(object,
+ next_entry->object.vm_object,
+ offset,
+ next_entry->offset,
+ size,
+ (vm_size_t)(next_entry->vme_end - next_entry->vme_start),
+ &next_entry->object.vm_object,
+ &next_entry->offset)) {
+
+ /*
+ * Coalesced the two objects - can extend
+ * the next map entry to include the
+ * new range.
+ */
+ map->size += size;
+ next_entry->vme_start = start;
+ vm_map_gap_update(&map->hdr, entry);
+ /*
+ * Now that we did, perhaps we could simplify
+ * things even further by coalescing the
+ * entry into the previous one.
+ */
+ vm_map_coalesce_entry(map, next_entry);
RETURN(KERN_SUCCESS);
}
}
@@ -1571,6 +1596,7 @@ kern_return_t vm_map_protect(
{
vm_map_entry_t current;
vm_map_entry_t entry;
+ vm_map_entry_t next;
vm_map_lock(map);
@@ -1632,7 +1658,8 @@ kern_return_t vm_map_protect(
*/
if ((current->protection != VM_PROT_NONE) &&
- (current->wired_access != VM_PROT_NONE)) {
+ (current->wired_access != VM_PROT_NONE ||
+ map->wiring_required)) {
current->wired_access = current->protection;
}
@@ -1645,9 +1672,16 @@ kern_return_t vm_map_protect(
current->vme_end,
current->protection);
}
- current = current->vme_next;
+
+ next = current->vme_next;
+ vm_map_coalesce_entry(map, current);
+ current = next;
}
+ next = current->vme_next;
+ if (vm_map_coalesce_entry(map, current))
+ current = next;
+
/* Returns with the map read-locked if successful */
vm_map_pageable_scan(map, entry, current);
@@ -1671,6 +1705,7 @@ kern_return_t vm_map_inherit(
{
vm_map_entry_t entry;
vm_map_entry_t temp_entry;
+ vm_map_entry_t next;
vm_map_lock(map);
@@ -1688,9 +1723,13 @@ kern_return_t vm_map_inherit(
entry->inheritance = new_inheritance;
- entry = entry->vme_next;
+ next = entry->vme_next;
+ vm_map_coalesce_entry(map, entry);
+ entry = next;
}
+ vm_map_coalesce_entry(map, entry);
+
vm_map_unlock(map);
return(KERN_SUCCESS);
}
@@ -1792,6 +1831,30 @@ kern_return_t vm_map_pageable(
return(KERN_SUCCESS);
}
+/* Update pageability of all the memory currently in the map.
+ * The map must be locked, and protection mismatch will not be checked, see
+ * vm_map_pageable().
+ */
+static kern_return_t
+vm_map_pageable_current(vm_map_t map, vm_prot_t access_type)
+{
+ struct rbtree_node *node;
+ vm_offset_t min_address, max_address;
+
+ node = rbtree_first(&map->hdr.tree);
+ min_address = rbtree_entry(node, struct vm_map_entry,
+ tree_node)->vme_start;
+
+ node = rbtree_last(&map->hdr.tree);
+ max_address = rbtree_entry(node, struct vm_map_entry,
+ tree_node)->vme_end;
+
+ /* Returns with the map read-locked if successful */
+ return vm_map_pageable(map, min_address, max_address,access_type,
+ FALSE, FALSE);
+}
+
+
/*
* vm_map_pageable_all:
*
@@ -1822,8 +1885,7 @@ vm_map_pageable_all(struct vm_map *map, vm_wire_t flags)
map->wiring_required = FALSE;
/* Returns with the map read-locked if successful */
- kr = vm_map_pageable(map, map->min_offset, map->max_offset,
- VM_PROT_NONE, FALSE, FALSE);
+ kr = vm_map_pageable_current(map, VM_PROT_NONE);
vm_map_unlock(map);
return kr;
}
@@ -1836,9 +1898,7 @@ vm_map_pageable_all(struct vm_map *map, vm_wire_t flags)
if (flags & VM_WIRE_CURRENT) {
/* Returns with the map read-locked if successful */
- kr = vm_map_pageable(map, map->min_offset, map->max_offset,
- VM_PROT_READ | VM_PROT_WRITE,
- FALSE, FALSE);
+ kr = vm_map_pageable_current(map, VM_PROT_READ | VM_PROT_WRITE);
if (kr != KERN_SUCCESS) {
if (flags & VM_WIRE_FUTURE) {
@@ -1865,11 +1925,13 @@ void vm_map_entry_delete(
vm_map_entry_t entry)
{
vm_offset_t s, e;
+ vm_size_t size;
vm_object_t object;
extern vm_object_t kernel_object;
s = entry->vme_start;
e = entry->vme_end;
+ size = e - s;
/*Check if projected buffer*/
if (map != kernel_map && entry->projected_on != 0) {
@@ -1908,15 +1970,29 @@ void vm_map_entry_delete(
if (object == kernel_object) {
vm_object_lock(object);
vm_object_page_remove(object, entry->offset,
- entry->offset + (e - s));
+ entry->offset + size);
vm_object_unlock(object);
} else if (entry->is_shared) {
vm_object_pmap_remove(object,
entry->offset,
- entry->offset + (e - s));
- }
- else {
+ entry->offset + size);
+ } else {
pmap_remove(map->pmap, s, e);
+ /*
+ * If this object has no pager and our
+ * reference to the object is the only
+ * one, we can release the deleted pages
+ * now.
+ */
+ vm_object_lock(object);
+ if ((!object->pager_created) &&
+ (object->ref_count == 1) &&
+ (object->paging_in_progress == 0)) {
+ vm_object_page_remove(object,
+ entry->offset,
+ entry->offset + size);
+ }
+ vm_object_unlock(object);
}
}
@@ -1931,7 +2007,7 @@ void vm_map_entry_delete(
vm_object_deallocate(entry->object.vm_object);
vm_map_entry_unlink(map, entry);
- map->size -= e - s;
+ map->size -= size;
vm_map_entry_dispose(map, entry);
}
@@ -1951,6 +2027,14 @@ kern_return_t vm_map_delete(
vm_map_entry_t entry;
vm_map_entry_t first_entry;
+ if (map->pmap == kernel_pmap && (start < kernel_virtual_start || end > kernel_virtual_end))
+ panic("vm_map_delete(%lx-%lx) falls in physical memory area!\n", (unsigned long) start, (unsigned long) end);
+
+ /*
+ * Must be called with map lock taken unless refcount is zero
+ */
+ assert((map->ref_count > 0 && have_lock(map->lock)) || (map->ref_count == 0));
+
/*
* Find the start of the region, and clip it
*/
@@ -2048,7 +2132,7 @@ kern_return_t vm_map_remove(
* Steal all the pages from a vm_map_copy page_list by copying ones
* that have not already been stolen.
*/
-void
+static void
vm_map_copy_steal_pages(vm_map_copy_t copy)
{
vm_page_t m, new_m;
@@ -3674,7 +3758,7 @@ kern_return_t vm_map_copyin_object(
* the scheduler.
*/
-kern_return_t vm_map_copyin_page_list_cont(
+static kern_return_t vm_map_copyin_page_list_cont(
vm_map_copyin_args_t cont_args,
vm_map_copy_t *copy_result) /* OUT */
{
@@ -3809,7 +3893,7 @@ kern_return_t vm_map_copyin_page_list(
copy->offset = src_addr;
copy->size = len;
copy->cpy_cont = ((kern_return_t (*)()) 0);
- copy->cpy_cont_args = (char *) VM_MAP_COPYIN_ARGS_NULL;
+ copy->cpy_cont_args = VM_MAP_COPYIN_ARGS_NULL;
/*
* Find the beginning of the region.
@@ -3899,7 +3983,7 @@ make_continuation:
}
cont_args->steal_pages = steal_pages;
- copy->cpy_cont_args = (char *) cont_args;
+ copy->cpy_cont_args = cont_args;
copy->cpy_cont = vm_map_copyin_page_list_cont;
src_end = src_start;
@@ -4238,7 +4322,7 @@ retry:
cont_args->destroy_len = src_end - src_start;
cont_args->steal_pages = FALSE;
- copy->cpy_cont_args = (char *) cont_args;
+ copy->cpy_cont_args = cont_args;
copy->cpy_cont = vm_map_copyin_page_list_cont;
}
@@ -4530,8 +4614,9 @@ vm_map_t vm_map_fork(vm_map_t old_map)
* In order to later verify this lookup, a "version"
* is returned.
*
- * The map should not be locked; it will not be
- * locked on exit. In order to guarantee the
+ * The map should not be locked; it will be
+ * unlocked on exit unless keep_map_locked is set and
+ * the lookup succeeds. In order to guarantee the
* existence of the returned object, it is returned
* locked.
*
@@ -4544,6 +4629,7 @@ kern_return_t vm_map_lookup(
vm_map_t *var_map, /* IN/OUT */
vm_offset_t vaddr,
vm_prot_t fault_type,
+ boolean_t keep_map_locked,
vm_map_version_t *out_version, /* OUT */
vm_object_t *object, /* OUT */
@@ -4565,7 +4651,8 @@ kern_return_t vm_map_lookup(
#define RETURN(why) \
{ \
- vm_map_unlock_read(map); \
+ if (!(keep_map_locked && (why == KERN_SUCCESS))) \
+ vm_map_unlock_read(map); \
return(why); \
}
@@ -4823,7 +4910,8 @@ vm_region_create_proxy (task_t task, vm_address_t address,
kern_return_t ret;
vm_map_entry_t entry, tmp_entry;
vm_object_t object;
- vm_offset_t offset, start;
+ rpc_vm_offset_t rpc_offset, rpc_start;
+ rpc_vm_size_t rpc_len = (rpc_vm_size_t) len;
ipc_port_t pager;
if (task == TASK_NULL)
@@ -4859,16 +4947,16 @@ vm_region_create_proxy (task_t task, vm_address_t address,
pager = ipc_port_copy_send(object->pager);
vm_object_unlock(object);
- start = (address - entry->vme_start) + entry->offset;
- offset = 0;
+ rpc_start = (address - entry->vme_start) + entry->offset;
+ rpc_offset = 0;
vm_map_unlock_read(task->map);
ret = memory_object_create_proxy(task->itk_space, max_protection,
&pager, 1,
- &offset, 1,
- &start, 1,
- &len, 1, port);
+ &rpc_offset, 1,
+ &rpc_start, 1,
+ &rpc_len, 1, port);
if (ret)
ipc_port_release_send(pager);
@@ -4876,64 +4964,80 @@ vm_region_create_proxy (task_t task, vm_address_t address,
}
/*
- * Routine: vm_map_simplify
- *
- * Description:
- * Attempt to simplify the map representation in
- * the vicinity of the given starting address.
- * Note:
- * This routine is intended primarily to keep the
- * kernel maps more compact -- they generally don't
- * benefit from the "expand a map entry" technology
- * at allocation time because the adjacent entry
- * is often wired down.
+ * Routine: vm_map_coalesce_entry
+ * Purpose:
+ * Try to coalesce an entry with the preceeding entry in the map.
+ * Conditions:
+ * The map is locked. If coalesced, the entry is destroyed
+ * by the call.
+ * Returns:
+ * Whether the entry was coalesced.
*/
-void vm_map_simplify(
+boolean_t
+vm_map_coalesce_entry(
vm_map_t map,
- vm_offset_t start)
+ vm_map_entry_t entry)
{
- vm_map_entry_t this_entry;
- vm_map_entry_t prev_entry;
+ vm_map_entry_t prev = entry->vme_prev;
+ vm_size_t prev_size;
+ vm_size_t entry_size;
- vm_map_lock(map);
- if (
- (vm_map_lookup_entry(map, start, &this_entry)) &&
- ((prev_entry = this_entry->vme_prev) != vm_map_to_entry(map)) &&
-
- (prev_entry->vme_end == start) &&
-
- (prev_entry->is_shared == FALSE) &&
- (prev_entry->is_sub_map == FALSE) &&
-
- (this_entry->is_shared == FALSE) &&
- (this_entry->is_sub_map == FALSE) &&
-
- (prev_entry->inheritance == this_entry->inheritance) &&
- (prev_entry->protection == this_entry->protection) &&
- (prev_entry->max_protection == this_entry->max_protection) &&
- (prev_entry->wired_count == this_entry->wired_count) &&
-
- (prev_entry->needs_copy == this_entry->needs_copy) &&
-
- (prev_entry->object.vm_object == this_entry->object.vm_object) &&
- ((prev_entry->offset + (prev_entry->vme_end - prev_entry->vme_start))
- == this_entry->offset) &&
- (prev_entry->projected_on == 0) &&
- (this_entry->projected_on == 0)
- ) {
- if (map->first_free == this_entry)
- map->first_free = prev_entry;
-
- SAVE_HINT(map, prev_entry);
- prev_entry->vme_end = this_entry->vme_end;
- vm_map_entry_unlink(map, this_entry);
- vm_object_deallocate(this_entry->object.vm_object);
- vm_map_entry_dispose(map, this_entry);
- }
- vm_map_unlock(map);
+ /*
+ * Check the basic conditions for coalescing the two entries.
+ */
+ if ((entry == vm_map_to_entry(map)) ||
+ (prev == vm_map_to_entry(map)) ||
+ (prev->vme_end != entry->vme_start) ||
+ (prev->is_shared || entry->is_shared) ||
+ (prev->is_sub_map || entry->is_sub_map) ||
+ (prev->inheritance != entry->inheritance) ||
+ (prev->protection != entry->protection) ||
+ (prev->max_protection != entry->max_protection) ||
+ (prev->needs_copy != entry->needs_copy) ||
+ (prev->in_transition || entry->in_transition) ||
+ (prev->wired_count != entry->wired_count) ||
+ (prev->projected_on != 0) ||
+ (entry->projected_on != 0))
+ return FALSE;
+
+ prev_size = prev->vme_end - prev->vme_start;
+ entry_size = entry->vme_end - entry->vme_start;
+ assert(prev->gap_size == 0);
+
+ /*
+ * See if we can coalesce the two objects.
+ */
+ if (!vm_object_coalesce(prev->object.vm_object,
+ entry->object.vm_object,
+ prev->offset,
+ entry->offset,
+ prev_size,
+ entry_size,
+ &prev->object.vm_object,
+ &prev->offset))
+ return FALSE;
+
+ /*
+ * Update the hints.
+ */
+ if (map->hint == entry)
+ SAVE_HINT(map, prev);
+ if (map->first_free == entry)
+ map->first_free = prev;
+
+ /*
+ * Get rid of the entry without changing any wirings or the pmap,
+ * and without altering map->size.
+ */
+ prev->vme_end = entry->vme_end;
+ vm_map_entry_unlink(map, entry);
+ vm_map_entry_dispose(map, entry);
+
+ return TRUE;
}
+
/*
* Routine: vm_map_machine_attribute
* Purpose:
@@ -5078,8 +5182,7 @@ void vm_map_print(db_expr_t addr, boolean_t have_addr, db_expr_t count, const ch
* Pretty-print a copy object for ddb.
*/
-void vm_map_copy_print(copy)
- const vm_map_copy_t copy;
+void vm_map_copy_print(const vm_map_copy_t copy)
{
int i, npages;