diff options
Diffstat (limited to 'i386/intel')
-rw-r--r-- | i386/intel/pmap.c | 186 | ||||
-rw-r--r-- | i386/intel/pmap.h | 43 |
2 files changed, 217 insertions, 12 deletions
diff --git a/i386/intel/pmap.c b/i386/intel/pmap.c index ffbd2ae4..3feb745d 100644 --- a/i386/intel/pmap.c +++ b/i386/intel/pmap.c @@ -92,6 +92,21 @@ #define WRITE_PTE(pte_p, pte_entry) *(pte_p) = (pte_entry); #endif /* MACH_PSEUDO_PHYS */ +#ifdef KPTI +static inline void write_pde(pt_entry_t *pde_p, pt_entry_t *pdeu_p, pt_entry_t pde_entry) +{ + if (pde_entry & INTEL_PTE_USER) + /* User page, show it in shadow page table too */ + *pdeu_p = pde_entry; + *pde_p = pde_entry; +} +#else +static inline void write_pde(pt_entry_t *pde_p, pt_entry_t *pdeu_p, pt_entry_t pde_entry) +{ + *pde_p = pde_entry; +} +#endif + /* * Private data structures. */ @@ -413,6 +428,19 @@ unsigned int inuse_ptepages_count = 0; /* debugging */ pt_entry_t *kernel_page_dir; /* + * Pointer to the template for user page table. + * Initialized by pmap_bootstrap(). + */ +pt_entry_t *user_page_dir; + + +/* + * Current kernel and user page table, used by the user/kernel switch assembly + * routine + */ +vm_offset_t kernel_pt __section(".data.shared"), user_pt __section(".data.shared"); + +/* * Two slots for temporary physical page mapping, to allow for * physical-to-physical transfers. */ @@ -432,6 +460,20 @@ pmap_pde(const pmap_t pmap, vm_offset_t addr) return &page_dir[lin2pdenum(addr)]; } +static inline pt_entry_t * +pmap_user_pde(const pmap_t pmap, vm_offset_t addr) +{ + pt_entry_t *page_dir; + if (pmap == kernel_pmap) + addr = kvtolin(addr); +#if PAE + page_dir = (pt_entry_t *) ptetokv(pmap->pdpbase_user[lin2pdpnum(addr)]); +#else + page_dir = pmap->dirbase_user; +#endif + return &page_dir[lin2pdenum(addr)]; +} + /* * Given an offset and a map, compute the address of the * pte. If the address is invalid with respect to the map @@ -576,6 +618,40 @@ vm_offset_t pmap_map_bd( return(virt); } +#ifdef KPTI +/* Set this virtual address in the user page table, thus shared between the + * kernel and the user page tables. */ +static void pmap_bootstrap_shared_kernel_page(vm_offset_t va, int pteflags) +{ + pt_entry_t *pdp = user_page_dir + lin2pdenum_cont(kvtolin(va)); + pt_entry_t pde = *pdp; + pt_entry_t *ptp; + + if (pde & INTEL_PTE_VALID) { + ptp = (pt_entry_t *) ptetokv(pde); + } else { + unsigned i; + + ptp = (pt_entry_t *) phystokv(pmap_grab_page()); + for (i = 0; i < NPTES; i++) + ptp[i] = 0; + + WRITE_PTE(pdp, pa_to_pte((vm_offset_t)_kvtophys(ptp)) + | (pteflags & ~INTEL_PTE_GLOBAL)); + } + + WRITE_PTE(ptp + ptenum(kvtolin(va)), + pa_to_pte(_kvtophys(va)) | pteflags); +} + + +void pmap_share_kernel_page(pmap_t pmap, vm_offset_t va, int pteflags) +{ + pt_entry_t *pgp = pmap_user_pde(pmap, va); +} + +#endif + /* * Bootstrap the system enough to run with virtual memory. * Allocate the kernel page directory and page tables, @@ -635,7 +711,14 @@ void pmap_bootstrap(void) vm_offset_t addr; init_alloc_aligned(PDPNUM * INTEL_PGBYTES, &addr); kernel_page_dir = (pt_entry_t*)phystokv(addr); +#ifdef KPTI + init_alloc_aligned(PDPNUM * INTEL_PGBYTES, &addr); + user_page_dir = (pt_entry_t*)phystokv(addr); +#endif } +#ifdef KPTI + kernel_pmap->pdpbase_user = +#endif kernel_pmap->pdpbase = (pt_entry_t*)phystokv(pmap_grab_page()); { int i; @@ -646,12 +729,22 @@ void pmap_bootstrap(void) | INTEL_PTE_VALID); } #else /* PAE */ +#ifdef KPTI + kernel_pmap->dirbase_user = +#endif kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(pmap_grab_page()); +#ifdef KPTI + user_page_dir = (pt_entry_t*)phystokv(pmap_grab_page()); +#endif #endif /* PAE */ { unsigned i; for (i = 0; i < NPDES; i++) kernel_page_dir[i] = 0; +#ifdef KPTI + for (i = 0; i < NPDES; i++) + user_page_dir[i] = 0; +#endif } #ifdef MACH_PV_PAGETABLES @@ -800,6 +893,17 @@ void pmap_bootstrap(void) panic("couldn't pin page %p(%p)\n", ptable, (vm_offset_t) kv_to_ma (ptable)); #endif /* MACH_PV_PAGETABLES */ } + +#ifdef KPTI + for (va = (vm_offset_t) _sharedtext_start; + va < (vm_offset_t) _sharedtext_end; + va += INTEL_PGBYTES) + pmap_bootstrap_shared_kernel_page(va, INTEL_PTE_VALID | global); + for (va = (vm_offset_t) _shareddata_start; + va < (vm_offset_t) _shareddata_end; + va += INTEL_PGBYTES) + pmap_bootstrap_shared_kernel_page(va, INTEL_PTE_VALID | INTEL_PTE_WRITE | global); +#endif } /* Architecture-specific code will turn on paging @@ -1001,8 +1105,7 @@ void pmap_init(void) KMEM_CACHE_PHYSMEM); #if PAE kmem_cache_init(&pdpt_cache, "pdpt", - PDPNUM * sizeof(pt_entry_t), - PDPNUM * sizeof(pt_entry_t), NULL, + INTEL_PGBYTES, INTEL_PGBYTES, NULL, KMEM_CACHE_PHYSMEM); #endif s = (vm_size_t) sizeof(struct pv_entry); @@ -1164,6 +1267,9 @@ pmap_page_table_page_dealloc(vm_offset_t pa) pmap_t pmap_create(vm_size_t size) { pt_entry_t *page_dir[PDPNUM]; +#ifdef KPTI + pt_entry_t *user_page_dir[PDPNUM]; +#endif int i; pmap_t p; pmap_statistics_t stats; @@ -1192,14 +1298,40 @@ pmap_t pmap_create(vm_size_t size) while (i >= 0) { kmem_cache_free(&pd_cache, (vm_address_t) page_dir[i]); +#ifdef KPTI + kmem_cache_free(&pd_cache, + (vm_address_t) user_page_dir[i]); +#endif + i -= 1; + } + kmem_cache_free(&pmap_cache, (vm_address_t) p); + return PMAP_NULL; + } +#ifdef KPTI + user_page_dir[i] = (pt_entry_t *) kmem_cache_alloc(&pd_cache); + if (user_page_dir[i] == NULL) { + kmem_cache_free(&pd_cache, + (vm_address_t) page_dir[i]); + i -= 1; + while (i >= 0) { + kmem_cache_free(&pd_cache, + (vm_address_t) page_dir[i]); + kmem_cache_free(&pd_cache, + (vm_address_t) user_page_dir[i]); i -= 1; } kmem_cache_free(&pmap_cache, (vm_address_t) p); return PMAP_NULL; } +#endif memcpy(page_dir[i], (void *) kernel_page_dir + i * INTEL_PGBYTES, INTEL_PGBYTES); +#ifdef KPTI + memcpy(user_page_dir[i], + (void *) user_page_dir + i * INTEL_PGBYTES, + INTEL_PGBYTES); +#endif } #ifdef LINUX_DEV @@ -1226,22 +1358,51 @@ pmap_t pmap_create(vm_size_t size) p->pdpbase = (pt_entry_t *) kmem_cache_alloc(&pdpt_cache); if (p->pdpbase == NULL) { for (i = 0; i < PDPNUM; i++) + { kmem_cache_free(&pd_cache, (vm_address_t) page_dir[i]); +#ifdef KPTI + kmem_cache_free(&pd_cache, (vm_address_t) user_page_dir[i]); +#endif + } + kmem_cache_free(&pmap_cache, (vm_address_t) p); + return PMAP_NULL; + } + +#ifdef KPTI + p->pdpbase_user = (pt_entry_t *) kmem_cache_alloc(&pdpt_cache); + if (p->pdpbase_user == NULL) { + kmem_cache_free(&pdpt_cache, (vm_address_t) p->pdpbase); + for (i = 0; i < PDPNUM; i++) + { + kmem_cache_free(&pd_cache, (vm_address_t) page_dir[i]); + kmem_cache_free(&pd_cache, (vm_address_t) user_page_dir[i]); + } kmem_cache_free(&pmap_cache, (vm_address_t) p); return PMAP_NULL; } +#endif { for (i = 0; i < PDPNUM; i++) + { WRITE_PTE(&p->pdpbase[i], pa_to_pte(kvtophys((vm_offset_t) page_dir[i])) | INTEL_PTE_VALID); +#ifdef KPTI + WRITE_PTE(&p->pdpbase_user[i], + pa_to_pte(kvtophys((vm_offset_t) user_page_dir[i])) + | INTEL_PTE_VALID); +#endif + } } #ifdef MACH_PV_PAGETABLES pmap_set_page_readonly(p->pdpbase); #endif /* MACH_PV_PAGETABLES */ #else /* PAE */ p->dirbase = page_dir[0]; +#ifdef KPTI + p->dirbase_user = user_page_dir[0]; +#endif #endif /* PAE */ p->ref_count = 1; @@ -1273,6 +1434,9 @@ void pmap_destroy(pmap_t p) #endif boolean_t free_all; pt_entry_t *page_dir; +#ifdef KPTI + pt_entry_t *user_page_dir; +#endif pt_entry_t *pdep; phys_addr_t pa; int c, s; @@ -1295,9 +1459,15 @@ void pmap_destroy(pmap_t p) for (i = 0; i <= lin2pdpnum(LINEAR_MIN_KERNEL_ADDRESS); i++) { free_all = i < lin2pdpnum(LINEAR_MIN_KERNEL_ADDRESS); page_dir = (pt_entry_t *) ptetokv(p->pdpbase[i]); +#ifdef KPTI + user_page_dir = (pt_entry_t *) ptetokv(p->pdpbase_user[i]); +#endif #else free_all = FALSE; page_dir = p->dirbase; +#ifdef KPTI + user_page_dir = p->dirbase_user; +#endif #endif /* @@ -1331,6 +1501,9 @@ void pmap_destroy(pmap_t p) pmap_set_page_readwrite((void*) page_dir); #endif /* MACH_PV_PAGETABLES */ kmem_cache_free(&pd_cache, (vm_offset_t) page_dir); +#ifdef KPTI + kmem_cache_free(&pd_cache, (vm_offset_t) user_page_dir); +#endif #if PAE } @@ -1912,7 +2085,7 @@ Retry: * Need to allocate a new page-table page. */ vm_offset_t ptp; - pt_entry_t *pdp; + pt_entry_t *pdp, *pdpu; int i; if (pmap == kernel_pmap) { @@ -1953,6 +2126,7 @@ Retry: */ i = ptes_per_vm_page; pdp = pmap_pde(pmap, v); + pdpu = pmap_user_pde(pmap, v); do { #ifdef MACH_PV_PAGETABLES pmap_set_page_readonly((void *) ptp); @@ -1964,9 +2138,9 @@ Retry: | INTEL_PTE_WRITE)) panic("%s:%d could not set pde %p(%p,%p) to %p(%p,%p) %p\n",__FILE__,__LINE__, pdp, kvtophys((vm_offset_t)pdp), (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)pdp)), ptp, kvtophys(ptp), (vm_offset_t) pa_to_ma(kvtophys(ptp)), (vm_offset_t) pa_to_pte(kv_to_ma(ptp))); #else /* MACH_PV_PAGETABLES */ - *pdp = pa_to_pte(kvtophys(ptp)) | INTEL_PTE_VALID - | INTEL_PTE_USER - | INTEL_PTE_WRITE; + write_pde(pdp, pdpu, pa_to_pte(kvtophys(ptp)) | INTEL_PTE_VALID + | INTEL_PTE_USER + | INTEL_PTE_WRITE); #endif /* MACH_PV_PAGETABLES */ pdp++; /* Note: This is safe b/c we stay in one page. */ ptp += INTEL_PGBYTES; diff --git a/i386/intel/pmap.h b/i386/intel/pmap.h index 5fa2a0c4..45a88ea1 100644 --- a/i386/intel/pmap.h +++ b/i386/intel/pmap.h @@ -86,6 +86,16 @@ typedef phys_addr_t pt_entry_t; #define PTEMASK 0x3ff /* mask for page table index */ #endif /* PAE */ +#ifndef MACH_HYP +#define KPTI 1 +#endif + +extern char _sharedtext_start[], _sharedtext_end[]; +extern char _shareddata_start[], _shareddata_end[]; + +/* These are used by user/kernel switch assembler code to switch page table. */ +extern vm_offset_t kernel_pt, user_pt; + /* * Convert linear offset to page descriptor index */ @@ -164,8 +174,14 @@ typedef volatile long cpu_set; /* set of CPUs - must be <= 32 */ struct pmap { #if ! PAE pt_entry_t *dirbase; /* page directory table */ +#ifdef KPTI + pt_entry_t *dirbase_user; /* page directory table for user */ +#endif #else pt_entry_t *pdpbase; /* page directory pointer table */ +#ifdef KPTI + pt_entry_t *pdpbase_user; /* page directory pointer table for user */ +#endif #endif /* ! PAE */ int ref_count; /* reference count */ decl_simple_lock_data(,lock) @@ -186,12 +202,6 @@ extern void pmap_map_mfn(void *addr, unsigned long mfn); extern void pmap_clear_bootstrap_pagetable(pt_entry_t *addr); #endif /* MACH_PV_PAGETABLES */ -#if PAE -#define set_pmap(pmap) set_cr3(kvtophys((vm_offset_t)(pmap)->pdpbase)) -#else /* PAE */ -#define set_pmap(pmap) set_cr3(kvtophys((vm_offset_t)(pmap)->dirbase)) -#endif /* PAE */ - typedef struct { pt_entry_t *entry; vm_offset_t vaddr; @@ -455,12 +465,33 @@ extern void pmap_zero_page (phys_addr_t); extern void pmap_copy_page (phys_addr_t, phys_addr_t); /* + * pmap_share_kernel_page marks + */ +#ifdef KPTI +extern void pmap_share_kernel_page(pmap_t pmap, vm_offset_t va, int pteflags); +#else +#define pmap_share_kernel_page(pmap, va, pteflags) ((void)0) +#endif + +/* * kvtophys(addr) * * Convert a kernel virtual address to a physical address */ extern phys_addr_t kvtophys (vm_offset_t); +static inline void set_pmap(pmap_t pmap) { +#if PAE + kernel_pt = kvtophys((vm_offset_t)(pmap)->pdpbase); + user_pt = kvtophys((vm_offset_t)(pmap)->pdpbase_user); +#else /* PAE */ + kernel_pt = kvtophys((vm_offset_t)(pmap)->dirbase); + user_pt = kvtophys((vm_offset_t)(pmap)->dirbase_user); +#endif /* PAE */ + /* We are using the kernel page table for now. */ + set_cr3(kernel_pt); +} + #if NCPUS > 1 void signal_cpus( cpu_set use_list, |