Fix early physical page allocation

Import upstream biosmem and vm_page changes, and adjust for local
modifications.

Specifically, the biosmem module was mistakenly loading physical
segments that did not clip with the heap as completely available.
This change makes it load them as completely unavailable during
startup, and once the VM system is ready, additional pages are
loaded.

* i386/i386at/biosmem.c (DEBUG): New macro.
(struct biosmem_segment): Remove members `avail_start' and `avail_end'.
(biosmem_heap_cur): Remove variable.
(biosmem_heap_bottom, biosmem_heap_top, biosmem_heap_topdown): New variables.
(biosmem_find_boot_data_update, biosmem_find_boot_data): Remove functions.
(biosmem_find_heap_clip, biosmem_find_heap): New functions.
(biosmem_setup_allocator): Rewritten to use the new biosmem_find_heap
function.
(biosmem_bootalloc): Support both bottom-up and top-down allocations.
(biosmem_directmap_size): Renamed to ...
(biosmem_directmap_end): ... this function.
(biosmem_load_segment): Fix segment loading.
(biosmem_setup): Restrict usable memory to the directmap segment.
(biosmem_free_usable_range): Add checks on input parameters.
(biosmem_free_usable_update_start, biosmem_free_usable_start,
biosmem_free_usable_reserved, biosmem_free_usable_end): Remove functions.
(biosmem_free_usable_entry): Rewritten to use the new biosmem_find_heap
function.
(biosmem_free_usable): Restrict usable memory to the directmap segment.
* i386/i386at/biosmem.h (biosmem_bootalloc): Update description.
(biosmem_directmap_size): Renamed to ...
(biosmem_directmap_end): ... this function.
(biosmem_free_usable): Update declaration.
* i386/i386at/model_dep.c (machine_init): Call biosmem_free_usable.
* vm/vm_page.c (DEBUG): New macro.
(struct vm_page_seg): New member `heap_present'.
(vm_page_load): Remove heap related parameters.
(vm_page_load_heap): New function.
* vm/vm_page.h (vm_page_load): Remove heap related parameters. Update
description.
(vm_page_load_heap): New function.

5 files changed, 280 insertions, 184 deletions

diff --git a/i386/i386at/biosmem.c b/i386/i386at/biosmem.c
index a7a440ef..567cc745 100644
--- a/i386/i386at/biosmem.c
+++ b/i386/i386at/biosmem.c
@@ -29,6 +29,8 @@
 #include <sys/types.h>
 #include <vm/vm_page.h>
 
+#define DEBUG 0
+
 #define __boot
 #define __bootdata
 #define __init
@@ -72,15 +74,10 @@ struct biosmem_map_entry {
 
 /*
  * Contiguous block of physical memory.
- *
- * Tha "available" range records what has been passed to the VM system as
- * available inside the segment.
  */
 struct biosmem_segment {
     phys_addr_t start;
     phys_addr_t end;
-    phys_addr_t avail_start;
-    phys_addr_t avail_end;
 };
 
 /*
@@ -104,9 +101,18 @@ static struct biosmem_segment biosmem_segments[VM_PAGE_MAX_SEGS] __bootdata;
  * This heap is located above BIOS memory.
  */
 static uint32_t biosmem_heap_start __bootdata;
-static uint32_t biosmem_heap_cur __bootdata;
+static uint32_t biosmem_heap_bottom __bootdata;
+static uint32_t biosmem_heap_top __bootdata;
 static uint32_t biosmem_heap_end __bootdata;
 
+/*
+ * Boot allocation policy.
+ *
+ * Top-down allocations are normally preferred to avoid unnecessarily
+ * filling the DMA segment.
+ */
+static boolean_t biosmem_heap_topdown __bootdata;
+
 static char biosmem_panic_toobig_msg[] __bootdata
     = "biosmem: too many memory map entries";
 #ifndef MACH_HYP
@@ -408,20 +414,37 @@ biosmem_save_cmdline_sizes(struct multiboot_raw_info *mbi)
     }
 }
 
-static void __boot
-biosmem_find_boot_data_update(uint32_t min, uint32_t *start, uint32_t *end,
-                              uint32_t data_start, uint32_t data_end)
+static int __boot
+biosmem_find_heap_clip(phys_addr_t *heap_start, phys_addr_t *heap_end,
+                       phys_addr_t data_start, phys_addr_t data_end)
 {
-    if ((min <= data_start) && (data_start < *start)) {
-        *start = data_start;
-        *end = data_end;
+    assert(data_start < data_end);
+
+    if ((data_end <= *heap_start) || (data_start >= *heap_end)) {
+        return 0;
+    }
+
+    if (data_start > *heap_start) {
+        *heap_end = data_start;
+    } else {
+        if (data_end >= *heap_end) {
+            return -1;
+        }
+
+        *heap_start = data_end;
     }
+
+    return 0;
 }
 
 /*
- * Find the first boot data in the given range, and return their containing
- * area (start address is returned directly, end address is returned in end).
- * The following are considered boot data :
+ * Find available memory for an allocation heap.
+ *
+ * The search starts at the given start address, up to the given end address.
+ * If a range is found, it is stored through the heap_startp and heap_endp
+ * pointers.
+ *
+ * The search skips boot data, that is :
  *  - the kernel
  *  - the kernel command line
  *  - the module table
@@ -429,48 +452,83 @@ biosmem_find_boot_data_update(uint32_t min, uint32_t *start, uint32_t *end,
  *  - the modules command lines
  *  - the ELF section header table
  *  - the ELF .shstrtab, .symtab and .strtab sections
- *
- * If no boot data was found, 0 is returned, and the end address isn't set.
  */
-static uint32_t __boot
-biosmem_find_boot_data(const struct multiboot_raw_info *mbi, uint32_t min,
-                       uint32_t max, uint32_t *endp)
+static int __boot
+biosmem_find_heap(const struct multiboot_raw_info *mbi,
+                  phys_addr_t start, phys_addr_t end,
+                  phys_addr_t *heap_start, phys_addr_t *heap_end)
 {
     struct multiboot_raw_module *mod;
     struct elf_shdr *shdr;
-    uint32_t i, start, end = end;
     unsigned long tmp;
+    unsigned int i;
+    int error;
 
-    start = max;
+    if (start >= end) {
+        return -1;
+    }
 
-    biosmem_find_boot_data_update(min, &start, &end, _kvtophys(&_start),
-                                  _kvtophys(&_end));
+    *heap_start = start;
+    *heap_end = end;
 
-    if ((mbi->flags & MULTIBOOT_LOADER_CMDLINE) && (mbi->cmdline != 0))
-        biosmem_find_boot_data_update(min, &start, &end, mbi->cmdline,
-                                      mbi->cmdline + mbi->unused0);
+    error = biosmem_find_heap_clip(heap_start, heap_end,
+                                   _kvtophys(&_start), _kvtophys(&_end));
+
+    if (error) {
+        return error;
+    }
+
+    if ((mbi->flags & MULTIBOOT_LOADER_CMDLINE) && (mbi->cmdline != 0)) {
+        error = biosmem_find_heap_clip(heap_start, heap_end,
+                                       mbi->cmdline,
+                                       mbi->cmdline + mbi->unused0);
+
+        if (error) {
+            return error;
+        }
+    }
 
     if (mbi->flags & MULTIBOOT_LOADER_MODULES) {
         i = mbi->mods_count * sizeof(struct multiboot_raw_module);
-        biosmem_find_boot_data_update(min, &start, &end, mbi->mods_addr,
-                                      mbi->mods_addr + i);
+        error = biosmem_find_heap_clip(heap_start, heap_end,
+                                       mbi->mods_addr, mbi->mods_addr + i);
+
+        if (error) {
+            return error;
+        }
+
         tmp = phystokv(mbi->mods_addr);
 
         for (i = 0; i < mbi->mods_count; i++) {
             mod = (struct multiboot_raw_module *)tmp + i;
-            biosmem_find_boot_data_update(min, &start, &end, mod->mod_start,
-                                          mod->mod_end);
+            error = biosmem_find_heap_clip(heap_start, heap_end,
+                                           mod->mod_start, mod->mod_end);
+
+            if (error) {
+                return error;
+            }
 
-            if (mod->string != 0)
-                biosmem_find_boot_data_update(min, &start, &end, mod->string,
-                                              mod->string + mod->reserved);
+            if (mod->string != 0) {
+                error = biosmem_find_heap_clip(heap_start, heap_end,
+                                               mod->string,
+                                               mod->string + mod->reserved);
+
+                if (error) {
+                    return error;
+                }
+            }
         }
     }
 
     if (mbi->flags & MULTIBOOT_LOADER_SHDR) {
         tmp = mbi->shdr_num * mbi->shdr_size;
-        biosmem_find_boot_data_update(min, &start, &end, mbi->shdr_addr,
-                                      mbi->shdr_addr + tmp);
+        error = biosmem_find_heap_clip(heap_start, heap_end,
+                                       mbi->shdr_addr, mbi->shdr_addr + tmp);
+
+        if (error) {
+            return error;
+        }
+
         tmp = phystokv(mbi->shdr_addr);
 
         for (i = 0; i < mbi->shdr_num; i++) {
@@ -480,53 +538,53 @@ biosmem_find_boot_data(const struct multiboot_raw_info *mbi, uint32_t min,
                 && (shdr->type != ELF_SHT_STRTAB))
                 continue;
 
-            biosmem_find_boot_data_update(min, &start, &end, shdr->addr,
-                                          shdr->addr + shdr->size);
+            error = biosmem_find_heap_clip(heap_start, heap_end,
+                                           shdr->addr, shdr->addr + shdr->size);
         }
     }
 
-    if (start == max)
-        return 0;
-
-    *endp = end;
-    return start;
+    return 0;
 }
 
 static void __boot
 biosmem_setup_allocator(struct multiboot_raw_info *mbi)
 {
-    uint32_t heap_start, heap_end, max_heap_start, max_heap_end;
-    uint32_t mem_end, next;
+    phys_addr_t heap_start, heap_end, max_heap_start, max_heap_end;
+    phys_addr_t start, end;
+    int error;
 
     /*
      * Find some memory for the heap. Look for the largest unused area in
      * upper memory, carefully avoiding all boot data.
      */
-    mem_end = vm_page_trunc((mbi->mem_upper + 1024) << 10);
+    end = vm_page_trunc((mbi->mem_upper + 1024) << 10);
 
 #ifndef __LP64__
-    if (mem_end > VM_PAGE_DIRECTMAP_LIMIT)
-        mem_end = VM_PAGE_DIRECTMAP_LIMIT;
+    if (end > VM_PAGE_DIRECTMAP_LIMIT)
+        end = VM_PAGE_DIRECTMAP_LIMIT;
 #endif /* __LP64__ */
 
     max_heap_start = 0;
     max_heap_end = 0;
-    next = BIOSMEM_END;
+    start = BIOSMEM_END;
 
-    do {
-        heap_start = next;
-        heap_end = biosmem_find_boot_data(mbi, heap_start, mem_end, &next);
+    for (;;) {
+        error = biosmem_find_heap(mbi, start, end, &heap_start, &heap_end);
 
-        if (heap_end == 0) {
-            heap_end = mem_end;
-            next = 0;
+        if (error) {
+            break;
         }
 
         if ((heap_end - heap_start) > (max_heap_end - max_heap_start)) {
             max_heap_start = heap_start;
             max_heap_end = heap_end;
         }
-    } while (next != 0);
+
+        start = heap_end;
+    }
+
+    if (max_heap_start >= max_heap_end)
+        boot_panic(biosmem_panic_setup_msg);
 
     max_heap_start = vm_page_round(max_heap_start);
     max_heap_end = vm_page_trunc(max_heap_end);
@@ -536,7 +594,9 @@ biosmem_setup_allocator(struct multiboot_raw_info *mbi)
 
     biosmem_heap_start = max_heap_start;
     biosmem_heap_end = max_heap_end;
-    biosmem_heap_cur = biosmem_heap_end;
+    biosmem_heap_bottom = biosmem_heap_start;
+    biosmem_heap_top = biosmem_heap_end;
+    biosmem_heap_topdown = TRUE;
 }
 
 #endif /* MACH_HYP */
@@ -621,15 +681,18 @@ biosmem_xen_bootstrap(void)
         biosmem_heap_end = VM_PAGE_DIRECTMAP_LIMIT;
 #endif /* __LP64__ */
 
+    biosmem_heap_bottom = biosmem_heap_start;
+    biosmem_heap_top = biosmem_heap_end;
+
     /*
-     * XXX Allocation on Xen must be bottom-up :
+     * XXX Allocation on Xen are initially bottom-up :
      * At the "start of day", only 512k are available after the boot
      * data. The pmap module then creates a 4g mapping so all physical
      * memory is available, but it uses this allocator to do so.
      * Therefore, it must return pages from this small 512k regions
      * first.
      */
-    biosmem_heap_cur = biosmem_heap_start;
+    biosmem_heap_topdown = FALSE;
 }
 
 #else /* MACH_HYP */
@@ -666,27 +729,32 @@ biosmem_bootalloc(unsigned int nr_pages)
     if (size == 0)
         boot_panic(biosmem_panic_inval_msg);
 
-#ifdef MACH_HYP
-    addr = biosmem_heap_cur;
-#else /* MACH_HYP */
-    /* Top-down allocation to avoid unnecessarily filling DMA segments */
-    addr = biosmem_heap_cur - size;
-#endif /* MACH_HYP */
+    if (biosmem_heap_topdown) {
+        addr = biosmem_heap_top - size;
 
-    if ((addr < biosmem_heap_start) || (addr > biosmem_heap_cur))
-        boot_panic(biosmem_panic_nomem_msg);
+        if ((addr < biosmem_heap_start) || (addr > biosmem_heap_top)) {
+            boot_panic(biosmem_panic_nomem_msg);
+        }
 
-#ifdef MACH_HYP
-    biosmem_heap_cur += size;
-#else /* MACH_HYP */
-    biosmem_heap_cur = addr;
-#endif /* MACH_HYP */
+        biosmem_heap_top = addr;
+    } else {
+        unsigned long end;
+
+        addr = biosmem_heap_bottom;
+        end = addr + size;
+
+        if ((end > biosmem_heap_end) || (end < biosmem_heap_bottom)) {
+            boot_panic(biosmem_panic_nomem_msg);
+        }
+
+        biosmem_heap_bottom = end;
+    }
 
     return addr;
 }
 
 phys_addr_t __boot
-biosmem_directmap_size(void)
+biosmem_directmap_end(void)
 {
     if (biosmem_segment_size(VM_PAGE_SEG_DIRECTMAP) != 0)
         return biosmem_segment_end(VM_PAGE_SEG_DIRECTMAP);
@@ -733,12 +801,13 @@ biosmem_map_show(void)
 }
 
 static void __init
-biosmem_load_segment(struct biosmem_segment *seg, uint64_t max_phys_end,
-                     phys_addr_t phys_start, phys_addr_t phys_end,
-                     phys_addr_t avail_start, phys_addr_t avail_end)
+biosmem_load_segment(struct biosmem_segment *seg, uint64_t max_phys_end)
 {
+    phys_addr_t phys_start, phys_end, avail_start, avail_end;
     unsigned int seg_index;
 
+    phys_start = seg->start;
+    phys_end = seg->end;
     seg_index = seg - biosmem_segments;
 
     if (phys_end > max_phys_end) {
@@ -753,15 +822,28 @@ biosmem_load_segment(struct biosmem_segment *seg, uint64_t max_phys_end,
         phys_end = max_phys_end;
     }
 
-    if ((avail_start < phys_start) || (avail_start >= phys_end))
-        avail_start = phys_start;
+    vm_page_load(seg_index, phys_start, phys_end);
+
+    /*
+     * Clip the remaining available heap to fit it into the loaded
+     * segment if possible.
+     */
+
+    if ((biosmem_heap_top > phys_start) && (biosmem_heap_bottom < phys_end)) {
+        if (biosmem_heap_bottom >= phys_start) {
+            avail_start = biosmem_heap_bottom;
+        } else {
+            avail_start = phys_start;
+        }
 
-    if ((avail_end <= phys_start) || (avail_end > phys_end))
-        avail_end = phys_end;
+        if (biosmem_heap_top <= phys_end) {
+            avail_end = biosmem_heap_top;
+        } else {
+            avail_end = phys_end;
+        }
 
-    seg->avail_start = avail_start;
-    seg->avail_end = avail_end;
-    vm_page_load(seg_index, phys_start, phys_end, avail_start, avail_end);
+        vm_page_load_heap(seg_index, avail_start, avail_end);
+    }
 }
 
 void __init
@@ -777,19 +859,33 @@ biosmem_setup(void)
             break;
 
         seg = &biosmem_segments[i];
-        biosmem_load_segment(seg, VM_PAGE_HIGHMEM_LIMIT, seg->start, seg->end,
-                             biosmem_heap_start, biosmem_heap_cur);
+
+        /* XXX Limit to directmap until highmem is supported */
+        biosmem_load_segment(seg, VM_PAGE_DIRECTMAP_LIMIT);
     }
 }
 
+#ifndef MACH_HYP
+
 static void __init
 biosmem_free_usable_range(phys_addr_t start, phys_addr_t end)
 {
     struct vm_page *page;
 
+    assert(start < end);
+
+    start = vm_page_round(start);
+    end = vm_page_round(end);
+
+    if (start >= end) {
+        return;
+    }
+
+#if DEBUG
     printf("biosmem: release to vm_page: %llx-%llx (%lluk)\n",
            (unsigned long long)start, (unsigned long long)end,
            (unsigned long long)((end - start) >> 10));
+#endif
 
     while (start < end) {
         page = vm_page_lookup_pa(start);
@@ -800,90 +896,35 @@ biosmem_free_usable_range(phys_addr_t start, phys_addr_t end)
 }
 
 static void __init
-biosmem_free_usable_update_start(phys_addr_t *start, phys_addr_t res_start,
-                                 phys_addr_t res_end)
+biosmem_free_usable_entry(struct multiboot_raw_info *mbi, phys_addr_t start,
+                          phys_addr_t end)
 {
-    if ((*start >= res_start) && (*start < res_end))
-        *start = res_end;
-}
-
-static phys_addr_t __init
-biosmem_free_usable_start(phys_addr_t start)
-{
-    const struct biosmem_segment *seg;
-    unsigned int i;
-
-    biosmem_free_usable_update_start(&start, _kvtophys(&_start),
-                                     _kvtophys(&_end));
-    biosmem_free_usable_update_start(&start, biosmem_heap_start,
-                                     biosmem_heap_end);
-
-    for (i = 0; i < ARRAY_SIZE(biosmem_segments); i++) {
-        seg = &biosmem_segments[i];
-        biosmem_free_usable_update_start(&start, seg->avail_start,
-                                         seg->avail_end);
-    }
-
-    return start;
-}
-
-static int __init
-biosmem_free_usable_reserved(phys_addr_t addr)
-{
-    const struct biosmem_segment *seg;
-    unsigned int i;
-
-    if ((addr >= _kvtophys(&_start))
-        && (addr < _kvtophys(&_end)))
-        return 1;
-
-    if ((addr >= biosmem_heap_start) && (addr < biosmem_heap_end))
-        return 1;
-
-    for (i = 0; i < ARRAY_SIZE(biosmem_segments); i++) {
-        seg = &biosmem_segments[i];
+    phys_addr_t heap_start, heap_end;
+    int error;
 
-        if ((addr >= seg->avail_start) && (addr < seg->avail_end))
-            return 1;
-    }
+    /* XXX Abuse biosmem_find_heap to locate usable areas */
 
-    return 0;
-}
+    for (;;) {
+        error = biosmem_find_heap(mbi, start, end, &heap_start, &heap_end);
 
-static phys_addr_t __init
-biosmem_free_usable_end(phys_addr_t start, phys_addr_t entry_end)
-{
-    while (start < entry_end) {
-        if (biosmem_free_usable_reserved(start))
+        if (error) {
             break;
+        }
 
-        start += PAGE_SIZE;
-    }
-
-    return start;
-}
-
-static void __init
-biosmem_free_usable_entry(phys_addr_t start, phys_addr_t end)
-{
-    phys_addr_t entry_end;
-
-    entry_end = end;
-
-    for (;;) {
-        start = biosmem_free_usable_start(start);
+        error = biosmem_find_heap_clip(&heap_start, &heap_end,
+                                       biosmem_heap_start, biosmem_heap_end);
 
-        if (start >= entry_end)
-            return;
+        if (error) {
+            break;
+        }
 
-        end = biosmem_free_usable_end(start, entry_end);
-        biosmem_free_usable_range(start, end);
-        start = end;
+        biosmem_free_usable_range(heap_start, heap_end);
+        start = heap_end;
     }
 }
 
 void __init
-biosmem_free_usable(void)
+biosmem_free_usable(struct multiboot_raw_info *mbi)
 {
     struct biosmem_map_entry *entry;
     uint64_t start, end;
@@ -902,9 +943,20 @@ biosmem_free_usable(void)
 
         end = vm_page_trunc(entry->base_addr + entry->length);
 
+        /* XXX Limit to directmap until highmem is supported */
+        if (end > VM_PAGE_DIRECTMAP_LIMIT) {
+            end = VM_PAGE_DIRECTMAP_LIMIT;
+        }
+
         if (start < BIOSMEM_BASE)
             start = BIOSMEM_BASE;
 
-        biosmem_free_usable_entry(start, end);
+        if (start >= end) {
+            continue;
+        }
+
+        biosmem_free_usable_entry(mbi, start, end);
     }
 }
+
+#endif /* MACH_HYP */
diff --git a/i386/i386at/biosmem.h b/i386/i386at/biosmem.h
index 1db63f9f..f48cfc30 100644
--- a/i386/i386at/biosmem.h
+++ b/i386/i386at/biosmem.h
@@ -56,20 +56,21 @@ void biosmem_bootstrap(struct multiboot_raw_info *mbi);
 /*
  * Allocate contiguous physical pages during bootstrap.
  *
- * This function is called before paging is enabled. It should only be used
- * to allocate initial page table pages. Those pages are later loaded into
- * the VM system (as reserved pages) which means they can be freed like other
- * regular pages. Users should fix up the type of those pages once the VM
- * system is initialized.
+ * This function is called before paging is enabled. The pages returned
+ * are guaranteed to be part of the direct physical mapping when paging
+ * is enabled.
+ *
+ * This function should only be used to allocate initial page table pages.
+ * Those pages are later loaded into the VM system (as reserved pages)
+ * which means they can be freed like other regular pages. Users should
+ * fix up the type of those pages once the VM system is initialized.
  */
 unsigned long biosmem_bootalloc(unsigned int nr_pages);
 
 /*
- * Return the amount of physical memory that can be directly mapped.
- *
- * This includes the size of both the DMA/DMA32 and DIRECTMAP segments.
+ * Return the limit of physical memory that can be directly mapped.
  */
-phys_addr_t biosmem_directmap_size(void);
+phys_addr_t biosmem_directmap_end(void);
 
 /*
  * Set up physical memory based on the information obtained during bootstrap
@@ -79,10 +80,9 @@ void biosmem_setup(void);
 
 /*
  * Free all usable memory.
- *
- * This includes ranges that weren't part of the bootstrap allocator initial
- * heap, e.g. because they contained boot data.
  */
-void biosmem_free_usable(void);
+#ifndef MACH_HYP
+void biosmem_free_usable(struct multiboot_raw_info *mbi);
+#endif /* MACH_HYP */
 
 #endif /* _X86_BIOSMEM_H */
diff --git a/i386/i386at/model_dep.c b/i386/i386at/model_dep.c
index 679d524a..8e98bd9e 100644
--- a/i386/i386at/model_dep.c
+++ b/i386/i386at/model_dep.c
@@ -153,6 +153,16 @@ void machine_init(void)
 	cninit();
 
 	/*
+	 * Make more free memory.
+	 *
+	 * This is particularly important for the Linux drivers which
+	 * require available DMA memory.
+	 */
+#ifndef MACH_HYP
+	biosmem_free_usable((struct multiboot_raw_info *) &boot_info);
+#endif /* MACH_HYP */
+
+	/*
 	 * Set up to use floating point.
 	 */
 	init_fpu();
diff --git a/vm/vm_page.c b/vm/vm_page.c
index a868fce8..12e6a5ea 100644
--- a/vm/vm_page.c
+++ b/vm/vm_page.c
@@ -43,6 +43,8 @@
 #include <sys/types.h>
 #include <vm/vm_page.h>
 
+#define DEBUG 0
+
 #define __init
 #define __initdata
 #define __read_mostly
@@ -123,6 +125,7 @@ struct vm_page_seg {
 struct vm_page_boot_seg {
     phys_addr_t start;
     phys_addr_t end;
+    boolean_t heap_present;
     phys_addr_t avail_start;
     phys_addr_t avail_end;
 };
@@ -483,29 +486,55 @@ vm_page_seg_free(struct vm_page_seg *seg, struct vm_page *page,
 }
 
 void __init
-vm_page_load(unsigned int seg_index, phys_addr_t start, phys_addr_t end,
-             phys_addr_t avail_start, phys_addr_t avail_end)
+vm_page_load(unsigned int seg_index, phys_addr_t start, phys_addr_t end)
 {
     struct vm_page_boot_seg *seg;
 
     assert(seg_index < ARRAY_SIZE(vm_page_boot_segs));
     assert(vm_page_aligned(start));
     assert(vm_page_aligned(end));
-    assert(vm_page_aligned(avail_start));
-    assert(vm_page_aligned(avail_end));
     assert(start < end);
-    assert(start <= avail_start);
-    assert(avail_end <= end);
     assert(vm_page_segs_size < ARRAY_SIZE(vm_page_boot_segs));
 
     seg = &vm_page_boot_segs[seg_index];
     seg->start = start;
     seg->end = end;
-    seg->avail_start = avail_start;
-    seg->avail_end = avail_end;
+    seg->heap_present = FALSE;
+
+#if DEBUG
+    printf("vm_page: load: %s: %llx:%llx\n",
+           vm_page_seg_name(seg_index),
+           (unsigned long long)start, (unsigned long long)end);
+#endif
+
     vm_page_segs_size++;
 }
 
+void
+vm_page_load_heap(unsigned int seg_index, phys_addr_t start, phys_addr_t end)
+{
+    struct vm_page_boot_seg *seg;
+
+    assert(seg_index < ARRAY_SIZE(vm_page_boot_segs));
+    assert(vm_page_aligned(start));
+    assert(vm_page_aligned(end));
+
+    seg = &vm_page_boot_segs[seg_index];
+
+    assert(seg->start <= start);
+    assert(end <= seg-> end);
+
+    seg->avail_start = start;
+    seg->avail_end = end;
+    seg->heap_present = TRUE;
+
+#if DEBUG
+    printf("vm_page: heap: %s: %llx:%llx\n",
+           vm_page_seg_name(seg_index),
+           (unsigned long long)start, (unsigned long long)end);
+#endif
+}
+
 int
 vm_page_ready(void)
 {
diff --git a/vm/vm_page.h b/vm/vm_page.h
index f2e20a78..dc5d5c3a 100644
--- a/vm/vm_page.h
+++ b/vm/vm_page.h
@@ -442,13 +442,18 @@ vm_page_get_priv(const struct vm_page *page)
 /*
  * Load physical memory into the vm_page module at boot time.
  *
- * The avail_start and avail_end parameters are used to maintain a simple
- * heap for bootstrap allocations.
- *
  * All addresses must be page-aligned. Segments can be loaded in any order.
  */
-void vm_page_load(unsigned int seg_index, phys_addr_t start, phys_addr_t end,
-                  phys_addr_t avail_start, phys_addr_t avail_end);
+void vm_page_load(unsigned int seg_index, phys_addr_t start, phys_addr_t end);
+
+/*
+ * Load available physical memory into the vm_page module at boot time.
+ *
+ * The segment referred to must have been loaded with vm_page_load
+ * before loading its heap.
+ */
+void vm_page_load_heap(unsigned int seg_index, phys_addr_t start,
+                       phys_addr_t end);
 
 /*
  * Return true if the vm_page module is completely initialized, false