/* * Mach Operating System * Copyright (c) 1993,1992,1991,1990 Carnegie Mellon University * Copyright (c) 1991 IBM Corporation * All Rights Reserved. * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation, * and that the nema IBM not be used in advertising or publicity * pertaining to distribution of the software without specific, written * prior permission. * * CARNEGIE MELLON AND IBM ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON AND IBM DISCLAIM ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ #include #include #include #include #include #include #include #include #include #include #include /* * Helpers for thread state as saved in the pcb area, during trap or irq handling */ #define pusha \ pushq %rax ;\ pushq %rcx ;\ pushq %rdx ;\ pushq %rbx ;\ subq $8,%rsp ;\ pushq %rbp ;\ pushq %rsi ;\ pushq %rdi ;\ pushq %r8 ;\ pushq %r9 ;\ pushq %r10 ;\ pushq %r11 ;\ pushq %r12 ;\ pushq %r13 ;\ pushq %r14 ;\ pushq %r15 #define popa \ popq %r15 ;\ popq %r14 ;\ popq %r13 ;\ popq %r12 ;\ popq %r11 ;\ popq %r10 ;\ popq %r9 ;\ popq %r8 ;\ popq %rdi ;\ popq %rsi ;\ popq %rbp ;\ addq $8,%rsp ;\ popq %rbx ;\ popq %rdx ;\ popq %rcx ;\ popq %rax #define PUSH_REGS_ISR \ pushq %rcx ;\ pushq %rdx ;\ pushq %rsi ;\ pushq %rdi ;\ pushq %r8 ;\ pushq %r9 ;\ pushq %r10 ;\ pushq %r11 #define PUSH_AREGS_ISR \ pushq %rax ;\ PUSH_REGS_ISR #define POP_REGS_ISR \ popq %r11 ;\ popq %r10 ;\ popq %r9 ;\ popq %r8 ;\ popq %rdi ;\ popq %rsi ;\ popq %rdx ;\ popq %rcx #define POP_AREGS_ISR \ POP_REGS_ISR ;\ popq %rax /* * Note that we have to load the kernel segment registers even if this * is a trap from the kernel, because the kernel uses user segment * registers for copyin/copyout. * (XXX Would it be smarter just to use fs or gs for that?) */ #ifdef USER32 #define PUSH_SEGMENTS(reg) \ movq %ds,reg ;\ pushq reg ;\ movq %es,reg ;\ pushq reg ;\ pushq %fs ;\ pushq %gs #else #define PUSH_SEGMENTS(reg) #endif #ifdef USER32 #define POP_SEGMENTS(reg) \ popq %gs ;\ popq %fs ;\ popq reg ;\ movq reg,%es ;\ popq reg ;\ movq reg,%ds #else #define POP_SEGMENTS(reg) #endif #ifdef USER32 #define PUSH_SEGMENTS_ISR(reg) \ movq %ds,reg ;\ pushq reg ;\ movq %es,reg ;\ pushq reg ;\ pushq %fs ;\ pushq %gs #else #define PUSH_SEGMENTS_ISR(reg) #endif #ifdef USER32 #define POP_SEGMENTS_ISR(reg) \ popq %gs ;\ popq %fs ;\ popq reg ;\ movq reg,%es ;\ popq reg ;\ movq reg,%ds #else #define POP_SEGMENTS_ISR(reg) #endif #ifdef USER32 #define SET_KERNEL_SEGMENTS(reg) \ mov %ss,reg /* switch to kernel segments */ ;\ mov reg,%ds /* (same as kernel stack segment) */ ;\ mov reg,%es ;\ mov reg,%fs ;\ mov $(PERCPU_DS),reg ;\ mov reg,%gs #else #define SET_KERNEL_SEGMENTS(reg) #endif /* * Fault recovery. */ #define RECOVER_TABLE_START \ .text 2 ;\ DATA(recover_table) ;\ .text #define RECOVER(addr) \ .text 2 ;\ .quad 9f ;\ .quad addr ;\ .text ;\ 9: #define RECOVER_TABLE_END \ .text 2 ;\ .globl EXT(recover_table_end) ;\ LEXT(recover_table_end) ;\ .text /* * Retry table for certain successful faults. */ #define RETRY_TABLE_START \ .text 3 ;\ DATA(retry_table) ;\ .text #define RETRY(addr) \ .text 3 ;\ .quad 9f ;\ .quad addr ;\ .text ;\ 9: #define RETRY_TABLE_END \ .text 3 ;\ .globl EXT(retry_table_end) ;\ LEXT(retry_table_end) ;\ .text /* * Allocate recovery and retry tables. */ RECOVER_TABLE_START RETRY_TABLE_START /* * Timing routines. */ #if STAT_TIME #define TIME_TRAP_UENTRY #define TIME_TRAP_SENTRY #define TIME_TRAP_UEXIT #define TIME_INT_ENTRY #define TIME_INT_EXIT #else /* microsecond timing */ /* * Microsecond timing. * Assumes a free-running microsecond counter. * no TIMER_MAX check needed. */ /* * There is only one current time-stamp per CPU, since only * the time-stamp in the current timer is used. * To save time, we allocate the current time-stamps here. */ .comm EXT(current_tstamp), 4*NCPUS /* * Update time on user trap entry. * 11 instructions (including cli on entry) * Assumes CPU number in %edx. * Uses %eax, %ebx, %ecx. */ #define TIME_TRAP_UENTRY \ pushf /* Save flags */ ;\ cli /* block interrupts */ ;\ movl VA_ETC,%ebx /* get timer value */ ;\ movl CX(EXT(current_tstamp),%rdx),%ecx /* get old time stamp */;\ movl %ebx,CX(EXT(current_tstamp),%rdx) /* set new time stamp */;\ subl %ecx,%ebx /* elapsed = new-old */ ;\ movl CX(EXT(current_timer),%rdx),%ecx /* get current timer */ ;\ addl %ebx,LOW_BITS(%ecx) /* add to low bits */ ;\ jns 0f /* if overflow, */ ;\ call timer_normalize /* normalize timer */ ;\ 0: addl $(TH_SYSTEM_TIMER-TH_USER_TIMER),%ecx ;\ /* switch to sys timer */;\ movl %ecx,CX(EXT(current_timer),%rdx) /* make it current */ ;\ popf /* allow interrupts */ /* * Update time on system call entry. * 11 instructions (including cli on entry) * Assumes CPU number in %edx. * Uses %ebx, %ecx. * Same as TIME_TRAP_UENTRY, but preserves %eax. */ #define TIME_TRAP_SENTRY \ pushf /* Save flags */ ;\ cli /* block interrupts */ ;\ movl VA_ETC,%ebx /* get timer value */ ;\ movl CX(EXT(current_tstamp),%rdx),%ecx /* get old time stamp */;\ movl %ebx,CX(EXT(current_tstamp),%rdx) /* set new time stamp */;\ subl %ecx,%ebx /* elapsed = new-old */ ;\ movl CX(EXT(current_timer),%rdx),%ecx /* get current timer */ ;\ addl %ebx,LOW_BITS(%ecx) /* add to low bits */ ;\ jns 0f /* if overflow, */ ;\ pushq %rax /* save %rax */ ;\ call timer_normalize /* normalize timer */ ;\ popq %rax /* restore %rax */ ;\ 0: addl $(TH_SYSTEM_TIMER-TH_USER_TIMER),%ecx ;\ /* switch to sys timer */;\ movl %ecx,CX(EXT(current_timer),%rdx) /* make it current */ ;\ popf /* allow interrupts */ /* * update time on user trap exit. * 10 instructions. * Assumes CPU number in %edx. * Uses %ebx, %ecx. */ #define TIME_TRAP_UEXIT \ cli /* block interrupts */ ;\ movl VA_ETC,%ebx /* get timer */ ;\ movl CX(EXT(current_tstamp),%rdx),%ecx /* get old time stamp */;\ movl %ebx,CX(EXT(current_tstamp),%rdx) /* set new time stamp */;\ subl %ecx,%ebx /* elapsed = new-old */ ;\ movl CX(EXT(current_timer),%rdx),%ecx /* get current timer */ ;\ addl %ebx,LOW_BITS(%ecx) /* add to low bits */ ;\ jns 0f /* if overflow, */ ;\ call timer_normalize /* normalize timer */ ;\ 0: addl $(TH_USER_TIMER-TH_SYSTEM_TIMER),%ecx ;\ /* switch to user timer */;\ movl %ecx,CX(EXT(current_timer),%rdx) /* make it current */ /* * update time on interrupt entry. * 9 instructions. * Assumes CPU number in %edx. * Leaves old timer in %ebx. * Uses %ecx. */ #define TIME_INT_ENTRY \ movl VA_ETC,%ecx /* get timer */ ;\ movl CX(EXT(current_tstamp),%rdx),%ebx /* get old time stamp */;\ movl %ecx,CX(EXT(current_tstamp),%rdx) /* set new time stamp */;\ subl %ebx,%ecx /* elapsed = new-old */ ;\ movl CX(EXT(current_timer),%rdx),%ebx /* get current timer */ ;\ addl %ecx,LOW_BITS(%ebx) /* add to low bits */ ;\ leal CX(0,%rdx),%ecx /* timer is 16 bytes */ ;\ lea CX(EXT(kernel_timer),%rdx),%ecx /* get interrupt timer*/;\ movl %ecx,CX(EXT(current_timer),%rdx) /* set timer */ /* * update time on interrupt exit. * 11 instructions * Assumes CPU number in %edx, old timer in %ebx. * Uses %eax, %ecx. */ #define TIME_INT_EXIT \ movl VA_ETC,%eax /* get timer */ ;\ movl CX(EXT(current_tstamp),%rdx),%ecx /* get old time stamp */;\ movl %eax,CX(EXT(current_tstamp),%rdx) /* set new time stamp */;\ subl %ecx,%eax /* elapsed = new-old */ ;\ movl CX(EXT(current_timer),%rdx),%ecx /* get current timer */ ;\ addl %eax,LOW_BITS(%ecx) /* add to low bits */ ;\ jns 0f /* if overflow, */ ;\ call timer_normalize /* normalize timer */ ;\ 0: testb $0x80,LOW_BITS+3(%ebx) /* old timer overflow? */;\ jz 0f /* if overflow, */ ;\ movl %ebx,%ecx /* get old timer */ ;\ call timer_normalize /* normalize timer */ ;\ 0: movl %ebx,CX(EXT(current_timer),%rdx) /* set timer */ /* * Normalize timer in ecx. * Preserves edx; clobbers eax. */ .align 2 timer_high_unit: .long TIMER_HIGH_UNIT /* div has no immediate opnd */ timer_normalize: pushq %rdx /* save register */ xorl %edx,%edx /* clear divisor high */ movl LOW_BITS(%ecx),%eax /* get divisor low */ divl timer_high_unit,%eax /* quotient in eax */ /* remainder in edx */ addl %eax,HIGH_BITS_CHECK(%ecx) /* add high_inc to check */ movl %edx,LOW_BITS(%ecx) /* remainder to low_bits */ addl %eax,HIGH_BITS(%ecx) /* add high_inc to high bits */ popq %rdx /* restore register */ ret /* * Switch to a new timer. */ ENTRY(timer_switch) CPU_NUMBER(%edx) /* get this CPU */ movl VA_ETC,%ecx /* get timer */ movl CX(EXT(current_tstamp),%rdx),%eax /* get old time stamp */ movl %ecx,CX(EXT(current_tstamp),%rdx) /* set new time stamp */ subl %ecx,%eax /* elapsed = new - old */ movl CX(EXT(current_timer),%rdx),%ecx /* get current timer */ addl %eax,LOW_BITS(%ecx) /* add to low bits */ jns 0f /* if overflow, */ call timer_normalize /* normalize timer */ 0: movl S_ARG0,%ecx /* get new timer */ movl %ecx,CX(EXT(current_timer),%rdx) /* set timer */ ret /* * Initialize the first timer for a CPU. */ ENTRY(start_timer) CPU_NUMBER(%edx) /* get this CPU */ movl VA_ETC,%ecx /* get timer */ movl %ecx,CX(EXT(current_tstamp),%rdx) /* set initial time stamp */ movl S_ARG0,%ecx /* get timer */ movl %ecx,CX(EXT(current_timer),%rdx) /* set initial timer */ ret #endif /* accurate timing */ /* */ /* * Trap/interrupt entry points. * * All traps must create the i386_saved_state struct on the stack on * entry. Note that: * - CR2 is only used if the trap is a page fault * - user_rsp/user_ss are only used if entering from user space * - v86_regs are used only from V86 threads * (TODO check if V86 is still used with USER32) * * Depending the CPL before entry, the stack might be switched or not; * if entering from user-space the CPU loads TSS->RSP0 in RSP, * otherwise RSP is unchanged. After this, the cpu pushes * SS/RSP/RFLAFS/CS/RIP and optionally ErrorCode and executes the handler. */ /* Try to save/show some information when a double fault happens * We can't recover to a working state, so if we have a debugger wait for it, * otherwise reset */ ENTRY(t_dbl_fault) INT_FIX cli /* disable interrupts that might corrupt the state*/ pusha movq %cr2,%rax movq %rax,R_CR2-R_R15(%rsp) /* CR2 might contain the faulting address */ subq $48,%rsp // FIXME remove when segments are cleaned up movq %rsp,%rdi /* pass the saved state */ call handle_double_fault jmp cpu_shutdown /* reset */ END(t_dbl_fault) /* * General protection or segment-not-present fault. * Check for a GP/NP fault in the kernel_return * sequence; if there, report it as a GP/NP fault on the user's instruction. * * rsp-> 0: trap code (NP or GP) * 8: segment number in error * 16 eip * 24 cs * 32 eflags * 40 old registers (trap is from kernel) */ ENTRY(t_gen_prot) INT_FIX pushq $(T_GENERAL_PROTECTION) /* indicate fault type */ jmp trap_check_kernel_exit /* check for kernel exit sequence */ ENTRY(t_segnp) INT_FIX pushq $(T_SEGMENT_NOT_PRESENT) /* indicate fault type */ trap_check_kernel_exit: #ifdef USER32 testq $(EFL_VM),32(%rsp) /* is trap from V86 mode? */ jnz EXT(alltraps) /* isn`t kernel trap if so */ #endif /* Note: handling KERNEL_RING value by hand */ testq $2,24(%rsp) /* is trap from kernel mode? */ jnz EXT(alltraps) /* if so: */ /* check for the kernel exit sequence */ cmpq $_kret_iret,16(%rsp) /* on IRET? */ je fault_iret #ifdef USER32 cmpq $_kret_popl_ds,16(%rsp) /* popping DS? */ je fault_popl_ds cmpq $_kret_popl_es,16(%rsp) /* popping ES? */ je fault_popl_es cmpq $_kret_popl_fs,16(%rsp) /* popping FS? */ je fault_popl_fs cmpq $_kret_popl_gs,16(%rsp) /* popping GS? */ je fault_popl_gs #endif take_fault: /* if none of the above: */ jmp EXT(alltraps) /* treat as normal trap. */ /* * GP/NP fault on IRET: CS or SS is in error. * All registers contain the user's values. * * on SP is * 0 trap number * 8 errcode * 16 eip * 24 cs --> trapno * 32 efl --> errcode * 40 user eip * 48 user cs * 56 user eflags * 64 user rsp * 72 user ss */ fault_iret: movq %rax,16(%rsp) /* save eax (we don`t need saved eip) */ popq %rax /* get trap number */ movq %rax,24-8(%rsp) /* put in user trap number */ popq %rax /* get error code */ movq %rax,32-16(%rsp) /* put in user errcode */ popq %rax /* restore eax */ jmp EXT(alltraps) /* take fault */ #ifdef USER32 /* * Fault restoring a segment register. The user's registers are still * saved on the stack. The offending segment register has not been * popped. */ fault_popl_ds: popq %rax /* get trap number */ popq %rdx /* get error code */ addq $24,%rsp /* pop stack to user regs */ jmp push_es /* (DS on top of stack) */ fault_popl_es: popq %rax /* get trap number */ popq %rdx /* get error code */ addq $24,%rsp /* pop stack to user regs */ jmp push_fs /* (ES on top of stack) */ fault_popl_fs: popq %rax /* get trap number */ popq %rdx /* get error code */ addq $24,%rsp /* pop stack to user regs */ jmp push_gs /* (FS on top of stack) */ fault_popl_gs: popq %rax /* get trap number */ popq %rdx /* get error code */ addq $24,%rsp /* pop stack to user regs */ jmp push_segregs /* (GS on top of stack) */ push_es: movq %es,%rcx pushq %rcx /* restore es, */ push_fs: pushq %fs /* restore fs, */ push_gs: pushq %gs /* restore gs. */ push_gsbase: pushq $0 pushq $0 #endif push_segregs: movq %rax,R_TRAPNO(%rsp) /* set trap number */ movq %rdx,R_ERR(%rsp) /* set error code */ jmp trap_set_segs /* take trap */ /* * Debug trap. Check for single-stepping across system call into * kernel. If this is the case, taking the debug trap has turned * off single-stepping - save the flags register with the trace * bit set. */ ENTRY(t_debug) INT_FIX #ifdef USER32 testq $(EFL_VM),16(%rsp) /* is trap from V86 mode? */ jnz 0f /* isn`t kernel trap if so */ #endif /* Note: handling KERNEL_RING value by hand */ testq $2,8(%rsp) /* is trap from kernel mode? */ jnz 0f /* if so: */ #ifdef USER32 cmpq $syscall_entry,(%rsp) /* system call entry? */ jne 0f /* if so: */ /* flags are sitting where syscall */ /* wants them */ addq $32,%rsp /* remove eip/cs */ jmp syscall_entry_2 /* continue system call entry */ #else // TODO: implement the 64-bit case ud2 #endif 0: pushq $0 /* otherwise: */ pushq $(T_DEBUG) /* handle as normal */ jmp EXT(alltraps) /* debug fault */ /* * Page fault traps save cr2. */ ENTRY(t_page_fault) INT_FIX pushq $(T_PAGE_FAULT) /* mark a page fault trap */ pusha /* save the general registers */ #ifdef MACH_XEN movq %ss:hyp_shared_info+CR2,%rax #else /* MACH_XEN */ movq %cr2,%rax /* get the faulting address */ #endif /* MACH_XEN */ movq %rax,R_CR2-R_R15(%rsp) /* save in rsp save slot */ jmp trap_push_segs /* continue fault */ /* * All 'exceptions' enter here with: * rsp-> trap number * error code * old eip * old cs * old eflags * old rsp if trapped from user * old ss if trapped from user */ ENTRY(alltraps) pusha /* save the general registers */ trap_push_segs: PUSH_SEGMENTS(%rax) /* and the segment registers */ SET_KERNEL_SEGMENTS(%rax) /* switch to kernel data segment */ trap_set_segs: cld /* clear direction flag */ #ifdef USER32 testl $(EFL_VM),R_EFLAGS(%rsp) /* in V86 mode? */ jnz trap_from_user /* user mode trap if so */ #endif /* Note: handling KERNEL_RING value by hand */ testb $2,R_CS(%rsp) /* user mode trap? */ jz trap_from_kernel /* kernel trap if not */ trap_from_user: CPU_NUMBER(%edx) TIME_TRAP_UENTRY movq CX(EXT(kernel_stack),%rdx),%rbx xchgq %rbx,%rsp /* switch to kernel stack */ /* user regs pointer already set */ _take_trap: movq %rbx,%rdi /* pass register save area to trap */ call EXT(user_trap) /* call user trap routine */ #ifdef USER32 orq %rax,%rax /* emulated syscall? */ jz 1f /* no, just return */ movq R_EAX(%rbx),%rax /* yes, get syscall number */ jmp syscall_entry_3 /* and emulate it */ #endif 1: movq (%rsp),%rsp /* switch back to PCB stack */ /* * Return from trap or system call, checking for ASTs. * On PCB stack. */ _return_from_trap: CPU_NUMBER(%edx) cmpl $0,CX(EXT(need_ast),%rdx) jz _return_to_user /* if we need an AST: */ movq CX(EXT(kernel_stack),%rdx),%rsp /* switch to kernel stack */ call EXT(i386_astintr) /* take the AST */ popq %rsp /* switch back to PCB stack */ jmp _return_from_trap /* and check again (rare) */ /* ASTs after this point will */ /* have to wait */ _return_to_user: TIME_TRAP_UEXIT /* * Return from kernel mode to interrupted thread. */ _return_from_kernel: #ifdef USER32 _kret_popl_gs: popq %gs /* restore segment registers */ _kret_popl_fs: popq %fs _kret_popl_es: popq %rax movq %rax,%es _kret_popl_ds: popq %rax movq %rax,%ds #endif popa /* restore general registers */ addq $16,%rsp /* discard trap number and error code */ _kret_iret: iretq /* return from interrupt */ /* * Trap from kernel mode. No need to switch stacks. */ trap_from_kernel: #if MACH_KDB || MACH_TTD movq %rsp,%rbx /* save current stack */ movq %rsp,%rdx /* on an interrupt stack? */ CPU_NUMBER(%ecx) and $(~(INTSTACK_SIZE-1)),%rdx cmpq CX(EXT(int_stack_base),%rcx),%rdx je 1f /* OK if so */ movl %ecx,%edx cmpq CX(EXT(kernel_stack),%rdx),%rsp /* already on kernel stack? */ ja 0f cmpq MY(ACTIVE_STACK),%rsp ja 1f /* switch if not */ 0: movq CX(EXT(kernel_stack),%rdx),%rsp 1: pushq %rbx /* save old stack */ movq %rbx,%rdi /* pass as parameter */ call EXT(kernel_trap) /* to kernel trap routine */ popq %rsp /* return to old stack */ #else /* MACH_KDB || MACH_TTD */ movq %rsp,%rdi /* pass parameter */ call EXT(kernel_trap) /* to kernel trap routine */ #endif /* MACH_KDB || MACH_TTD */ jmp _return_from_kernel /* * Called as a function, makes the current thread * return from the kernel as if from an exception. */ ENTRY(thread_exception_return) ENTRY(thread_bootstrap_return) movq %rsp,%rcx /* get kernel stack */ or $(KERNEL_STACK_SIZE-1),%rcx movq -7-IKS_SIZE(%rcx),%rsp /* switch back to PCB stack */ jmp _return_from_trap /* * Called as a function, makes the current thread * return from the kernel as if from a syscall. * Takes the syscall's return code as an argument. */ ENTRY(thread_syscall_return) movq S_ARG0,%rax /* get return value */ movq %rsp,%rcx /* get kernel stack */ or $(KERNEL_STACK_SIZE-1),%rcx movq -7-IKS_SIZE(%rcx),%rsp /* switch back to PCB stack */ movq %rax,R_EAX(%rsp) /* save return value */ jmp _return_from_trap ENTRY(call_continuation) movq S_ARG0,%rax /* get continuation */ movq %rsp,%rcx /* get kernel stack */ or $(KERNEL_STACK_SIZE-1),%rcx addq $(-7-IKS_SIZE),%rcx movq %rcx,%rsp /* pop the stack */ xorq %rbp,%rbp /* zero frame pointer */ pushq $0 /* Dummy return address */ jmp *%rax /* goto continuation */ /* IOAPIC has 24 interrupts, put spurious in the same array */ #define INTERRUPT(n) \ .data 2 ;\ .quad 0f ;\ .text ;\ P2ALIGN(TEXT_ALIGN) ;\ 0: ;\ INT_FIX ;\ pushq %rax ;\ movq $(n),%rax ;\ jmp EXT(all_intrs) .data 2 DATA(int_entry_table) .text /* Legacy APIC interrupts or PIC interrupts */ INTERRUPT(0) INTERRUPT(1) INTERRUPT(2) INTERRUPT(3) INTERRUPT(4) INTERRUPT(5) INTERRUPT(6) INTERRUPT(7) INTERRUPT(8) INTERRUPT(9) INTERRUPT(10) INTERRUPT(11) INTERRUPT(12) INTERRUPT(13) INTERRUPT(14) INTERRUPT(15) #ifdef APIC /* APIC PCI interrupts PIRQ A-H */ INTERRUPT(16) INTERRUPT(17) INTERRUPT(18) INTERRUPT(19) INTERRUPT(20) INTERRUPT(21) INTERRUPT(22) INTERRUPT(23) #endif #if NCPUS > 1 INTERRUPT(CALL_AST_CHECK) INTERRUPT(CALL_PMAP_UPDATE) #endif #ifdef APIC /* Spurious interrupt, set irq number to vect number */ INTERRUPT(255) #endif /* XXX handle NMI - at least print a warning like Linux does. */ /* * All interrupts enter here. The cpu might have loaded a new RSP, * depending on the previous CPL, as in alltraps. * Old %eax on stack, interrupt number in %eax; we need to fill the remaining * fields of struct i386_interrupt_state, which might be in the pcb or in the * interrupt stack. */ ENTRY(all_intrs) PUSH_REGS_ISR /* save registers */ cld /* clear direction flag */ PUSH_SEGMENTS_ISR(%rdx) /* save segment registers */ CPU_NUMBER_NO_GS(%ecx) movq %rsp,%rdx /* on an interrupt stack? */ and $(~(INTSTACK_SIZE-1)),%rdx cmpq %ss:CX(EXT(int_stack_base),%rcx),%rdx je int_from_intstack /* if not: */ SET_KERNEL_SEGMENTS(%rdx) /* switch to kernel segments */ CPU_NUMBER(%edx) movq CX(EXT(int_stack_top),%rdx),%rcx xchgq %rcx,%rsp /* switch to interrupt stack */ #if STAT_TIME pushq %rcx /* save pointer to old stack */ #else pushq %rbx /* save %ebx - out of the way */ /* so stack looks the same */ pushq %rcx /* save pointer to old stack */ TIME_INT_ENTRY /* do timing */ #endif #ifdef MACH_LDEBUG incl CX(EXT(in_interrupt),%rdx) #endif call EXT(interrupt) /* call generic interrupt routine */ .globl EXT(return_to_iret) /* ( label for kdb_kintr and hardclock */ LEXT(return_to_iret) /* to find the return from calling interrupt) */ CPU_NUMBER(%edx) #ifdef MACH_LDEBUG decl CX(EXT(in_interrupt),%rdx) #endif #if STAT_TIME #else TIME_INT_EXIT /* do timing */ movq 8(%rsp),%rbx /* restore the extra reg we saved */ #endif popq %rsp /* switch back to old stack */ #ifdef USER32 testl $(EFL_VM),I_EFL(%rsp) /* if in V86 */ jnz 0f /* or */ #endif /* Note: handling KERNEL_RING value by hand */ testb $2,I_CS(%rsp) /* user mode, */ jz 1f /* check for ASTs */ 0: cmpq $0,CX(EXT(need_ast),%rdx) jnz ast_from_interrupt /* take it if so */ 1: POP_SEGMENTS_ISR(%rdx) /* restore segment regs */ POP_AREGS_ISR /* restore registers */ iretq /* return to caller */ int_from_intstack: CPU_NUMBER_NO_GS(%edx) cmpq CX(EXT(int_stack_base),%rdx),%rsp /* seemingly looping? */ jb stack_overflowed /* if not: */ call EXT(interrupt) /* call interrupt routine */ _return_to_iret_i: /* ( label for kdb_kintr) */ POP_SEGMENTS_ISR(%rdx) POP_AREGS_ISR /* restore registers */ /* no ASTs */ iretq stack_overflowed: ud2 /* * Take an AST from an interrupt. * On PCB stack. * sp-> gs -> edx * fs -> ecx * es -> eax * ds -> trapno * edx -> code * ecx * eax * eip * cs * efl * rsp * ss */ ast_from_interrupt: POP_SEGMENTS_ISR(%rdx) /* restore all registers ... */ POP_AREGS_ISR pushq $0 /* zero code */ pushq $0 /* zero trap number */ pusha /* save general registers */ PUSH_SEGMENTS_ISR(%rdx) /* save segment registers */ SET_KERNEL_SEGMENTS(%rdx) /* switch to kernel segments */ CPU_NUMBER(%edx) TIME_TRAP_UENTRY movq CX(EXT(kernel_stack),%rdx),%rsp /* switch to kernel stack */ call EXT(i386_astintr) /* take the AST */ popq %rsp /* back to PCB stack */ jmp _return_from_trap /* return */ #if MACH_KDB /* * kdb_kintr: enter kdb from keyboard interrupt. * Chase down the stack frames until we find one whose return * address is the interrupt handler. At that point, we have: * * frame-> saved %rbp * return address in interrupt handler * saved SPL * saved IRQ * return address == return_to_iret_i * saved %r11 * saved %r10 * saved %r9 * saved %r8 * saved %rdx * saved %rcx * saved %rax * saved %rip * saved %cs * saved %rfl * * OR: * frame-> saved %rbp * return address in interrupt handler * return address == return_to_iret * pointer to save area on old stack * [ saved %ebx, if accurate timing ] * * old stack: saved %gs * saved %fs * saved %es * saved %ds * saved %r11 * saved %r10 * saved %r9 * saved %r8 * saved %rdi * saved %rsi * saved %rdx * saved %rcx * saved %eax * saved %rip * saved %cs * saved %rfl * * Call kdb, passing it that register save area. */ #define RET_OFFSET 32 ENTRY(kdb_kintr) movq %rbp,%rax /* save caller`s frame pointer */ movq $EXT(return_to_iret),%rcx /* interrupt return address 1 */ movq $_return_to_iret_i,%rdx /* interrupt return address 2 */ 0: cmpq RET_OFFSET(%rax),%rcx /* does this frame return to */ /* interrupt handler (1)? */ je 1f cmpq RET_OFFSET(%rax),%rdx /* interrupt handler (2)? */ je 2f /* if not: */ movq (%rax),%rax /* try next frame */ testq %rax,%rax jnz 0b ud2 /* oops, didn't find frame, fix me :/ */ 1: movq $kdb_from_iret,RET_OFFSET(%rax) ret /* returns to kernel/user stack */ 2: movq $kdb_from_iret_i,RET_OFFSET(%rax) /* returns to interrupt stack */ ret /* * On return from keyboard interrupt, we will execute * kdb_from_iret_i * if returning to an interrupt on the interrupt stack * kdb_from_iret * if returning to an interrupt on the user or kernel stack */ kdb_from_iret: /* save regs in known locations */ #if STAT_TIME pushq %rbx /* caller`s %ebx is in reg */ #else movq 8(%rsp),%rax /* get caller`s %ebx */ pushq %rax /* push on stack */ #endif pushq %rbp movq %rsp,%rdi /* pass regs */ call EXT(kdb_kentry) /* to kdb */ popq %rbp #if STAT_TIME popq %rbx #else popq %rax movq %rax,8(%rsp) #endif jmp EXT(return_to_iret) /* normal interrupt return */ kdb_from_iret_i: /* on interrupt stack */ pop %rdx /* restore saved registers */ pop %rcx pop %rax pushq $0 /* zero error code */ pushq $0 /* zero trap number */ pusha /* save general registers */ PUSH_SEGMENTS(%rdx) /* save segment registers */ movq %rsp,%rdx /* pass regs, */ movq $0,%rsi /* code, */ movq $-1,%rdi /* type to kdb */ call EXT(kdb_trap) POP_SEGMENTS(%rdx) /* restore segment registers */ popa /* restore general registers */ addq $16,%rsp // TODO: test it before dropping ud2 movq (%rsp),%rax ud2 iretq #endif /* MACH_KDB */ #if MACH_TTD /* * Same code as that above for the keyboard entry into kdb. */ ENTRY(kttd_intr) // TODO: test it before dropping ud2 ud2 movq %rbp,%rax /* save caller`s frame pointer */ movq $EXT(return_to_iret),%rcx /* interrupt return address 1 */ movq $_return_to_iret_i,%rdx /* interrupt return address 2 */ 0: cmpq 32(%rax),%rcx /* does this frame return to */ /* interrupt handler (1)? */ je 1f cmpq 32(%rax),%rdx /* interrupt handler (2)? */ je 2f /* if not: */ movq (%rax),%rax /* try next frame */ jmp 0b 1: movq $ttd_from_iret,32(%rax) /* returns to kernel/user stack */ ret 2: movq $ttd_from_iret_i,32(%rax) /* returns to interrupt stack */ ret /* * On return from keyboard interrupt, we will execute * ttd_from_iret_i * if returning to an interrupt on the interrupt stack * ttd_from_iret * if returning to an interrupt on the user or kernel stack */ ttd_from_iret: /* save regs in known locations */ #if STAT_TIME pushq %rbx /* caller`s %ebx is in reg */ #else movq 8(%rsp),%rax /* get caller`s %ebx */ pushq %rax /* push on stack */ #endif pushq %rbp pushq %rsi pushq %rdi movq %rsp,%rdi /* pass regs */ call _kttd_netentry /* to kdb */ popq %rdi /* restore registers */ popq %rsi popq %rbp #if STAT_TIME popq %rbx #else popq %rax movq %rax,8(%rsp) #endif jmp EXT(return_to_iret) /* normal interrupt return */ ttd_from_iret_i: /* on interrupt stack */ pop %rdx /* restore saved registers */ pop %rcx pop %rax pushq $0 /* zero error code */ pushq $0 /* zero trap number */ pusha /* save general registers */ PUSH_SEGMENTS_ISR(%rdx) /* save segment registers */ ud2 // TEST it movq %rsp,%rdx /* pass regs, */ movq $0,%rsi /* code, */ movq $-1,%rdi /* type to kdb */ call _kttd_trap POP_SEGMENTS_ISR(%rdx) /* restore segment registers */ popa /* restore general registers */ addq $16,%rsp // TODO: test it before dropping ud2 movq (%rsp),%rax ud2 iretq #endif /* MACH_TTD */ #ifdef USER32 /* * System call enters through a call gate. Flags are not saved - * we must shuffle stack to look like trap save area. * * rsp-> old eip * old cs * old rsp * old ss * * eax contains system call number. */ ENTRY(syscall) syscall_entry: pushf /* save flags as soon as possible */ syscall_entry_2: cld /* clear direction flag */ pushq %rax /* save system call number */ pushq $0 /* clear trap number slot */ pusha /* save the general registers */ PUSH_SEGMENTS(%rdx) /* and the segment registers */ SET_KERNEL_SEGMENTS(%rdx) /* switch to kernel data segment */ /* * Shuffle eflags,eip,cs into proper places */ movq R_EIP(%rsp),%rbx /* eflags are in EIP slot */ movq R_CS(%rsp),%rcx /* eip is in CS slot */ movq R_EFLAGS(%rsp),%rdx /* cs is in EFLAGS slot */ movq %rcx,R_EIP(%rsp) /* fix eip */ movq %rdx,R_CS(%rsp) /* fix cs */ movq %rbx,R_EFLAGS(%rsp) /* fix eflags */ CPU_NUMBER_NO_STACK(%edx) TIME_TRAP_SENTRY movq CX(EXT(kernel_stack),%rdx),%rbx /* get current kernel stack */ xchgq %rbx,%rsp /* switch stacks - %ebx points to */ /* user registers. */ /* user regs pointer already set */ /* * Check for MACH or emulated system call */ syscall_entry_3: movq MY(ACTIVE_THREAD),%rdx /* point to current thread */ movq TH_TASK(%rdx),%rdx /* point to task */ movq TASK_EMUL(%rdx),%rdx /* get emulation vector */ orq %rdx,%rdx /* if none, */ je syscall_native /* do native system call */ movq %rax,%rcx /* copy system call number */ subq DISP_MIN(%rdx),%rcx /* get displacement into syscall */ /* vector table */ jl syscall_native /* too low - native system call */ cmpq DISP_COUNT(%rdx),%rcx /* check range */ jnl syscall_native /* too high - native system call */ movq DISP_VECTOR(%rdx,%rcx,4),%rdx /* get the emulation vector */ orq %rdx,%rdx /* emulated system call if not zero */ jnz syscall_emul /* * Native system call. */ syscall_native: negl %eax /* get system call number */ jl mach_call_range /* out of range if it was positive */ cmpl EXT(mach_trap_count),%eax /* check system call table bounds */ jg mach_call_range /* error if out of range */ #if 0 /* debug hack to show the syscall number on the screen */ movb %al,%dl shrb $4,%dl orb $0x30,%dl movb $0x0f,%dh movw %dx,0xb800a movb %al,%dl andb $0xf,%dl orb $0x30,%dl movb $0xf,%dh movw %dx,0xb800c #endif shll $5,%eax /* manual indexing of mach_trap_t */ xorq %r10,%r10 mov EXT(mach_trap_table)(%rax),%r10 /* get number of arguments */ andq %r10,%r10 jz mach_call_call /* skip argument copy if none */ movq $USER_DS,%rdx /* use user data segment for accesses */ mov %dx,%fs movq %rsp,%r11 /* save kernel ESP for error recovery */ movq R_UESP(%rbx),%rbp /* get user stack pointer */ addq $4,%rbp /* Skip user return address */ movq $VM_MAX_ADDRESS, %rcx cmpq %rcx,%rbp /* Check segment limit by hand */ jae mach_call_addr_push #define PARAM(reg,ereg) \ xorq %reg,%reg ;\ RECOVER(mach_call_addr_push) \ movl %fs:(%rbp),%ereg /* 1st parameter */ ;\ addq $4,%rbp ;\ dec %r10 ;\ jz mach_call_call PARAM(rdi,edi) /* 1st parameter */ PARAM(rsi,esi) /* 2nd parameter */ PARAM(rdx,edx) /* 3rd parameter */ PARAM(rcx,ecx) /* 4th parameter */ PARAM(r8,r8d) /* 5th parameter */ PARAM(r9,r9d) /* 6th parameter */ lea (%rbp,%r10,4),%rbp /* point past last argument */ xorq %r12,%r12 0: subq $4,%rbp RECOVER(mach_call_addr_push) movl %fs:(%rbp),%r12d pushq %r12 /* push argument on stack */ dec %r10 jnz 0b /* loop for all arguments */ mach_call_call: #ifdef DEBUG testb $0xff,EXT(syscall_trace) jz 0f movq %rax,%rdi call EXT(syscall_trace_print) /* will return with syscallofs still (or again) in eax */ 0: #endif /* DEBUG */ call *EXT(mach_trap_table)+8(%rax) /* call procedure */ movq %rsp,%rcx /* get kernel stack */ or $(KERNEL_STACK_SIZE-1),%rcx movq -7-IKS_SIZE(%rcx),%rsp /* switch back to PCB stack */ movq %rax,R_EAX(%rsp) /* save return value */ jmp _return_from_trap /* return to user */ /* * Address out of range. Change to page fault. * %rbp holds failing address. */ mach_call_addr_push: movq %r11,%rsp /* clean parameters from stack */ mach_call_addr: movq %rbp,R_CR2(%rbx) /* set fault address */ movq $(T_PAGE_FAULT),R_TRAPNO(%rbx) /* set page-fault trap */ movq $(T_PF_USER),R_ERR(%rbx) /* set error code - read user space */ jmp _take_trap /* treat as a trap */ /* * System call out of range. Treat as invalid-instruction trap. * (? general protection?) */ mach_call_range: movq $(T_INVALID_OPCODE),R_TRAPNO(%rbx) /* set invalid-operation trap */ movq $0,R_ERR(%rbx) /* clear error code */ jmp _take_trap /* treat as a trap */ /* * User space emulation of system calls. * edx - user address to handle syscall * * User stack will become: * ursp-> eflags * eip * eax still contains syscall number. */ syscall_emul: movq $USER_DS,%rdi /* use user data segment for accesses */ mov %di,%fs /* XXX what about write-protected pages? */ movq R_UESP(%rbx),%rdi /* get user stack pointer */ subq $16,%rdi /* push space for new arguments */ movq $VM_MAX_ADDRESS, %rax cmpq %rax,%rdi /* Check segment limit by hand */ jae syscall_addr movq R_EFLAGS(%rbx),%rax /* move flags */ RECOVER(syscall_addr) movl %eax,%fs:0(%rdi) /* to user stack */ movl R_EIP(%rbx),%eax /* move eip */ RECOVER(syscall_addr) movl %eax,%fs:4(%rdi) /* to user stack */ movq %rdi,R_UESP(%rbx) /* set new user stack pointer */ movq %rdx,R_EIP(%rbx) /* change return address to trap */ movq %rbx,%rsp /* back to PCB stack */ // TODO: test it before dropping ud2 ud2 jmp _return_from_trap /* return to user */ /* * Address error - address is in %edi. */ syscall_addr: movq %rdi,R_CR2(%rbx) /* set fault address */ movq $(T_PAGE_FAULT),R_TRAPNO(%rbx) /* set page-fault trap */ movq $(T_PF_USER),R_ERR(%rbx) /* set error code - read user space */ jmp _take_trap /* treat as a trap */ END(syscall) #else /* USER32 */ /* Entry point for 64-bit syscalls. * On entry we're still on the user stack, so better not use it. Instead we * save the thread state immediately in thread->pcb->iss, then try to invoke * the syscall. * Note: emulated syscalls seem to not be used anymore in GNU/Hurd, so they * are not handled here. * TODO: - for now we assume the return address is canonical, but apparently there can be cases where it's not (see how Linux handles this). Does it apply here? - check that the case where a task is suspended, and later returns via iretq from return_from_trap, works fine in all combinations */ ENTRY(syscall64) /* RFLAGS[32:63] are reserved, so combine syscall num (32 bit) and * eflags in RAX to allow using r11 as temporary register */ shlq $32,%r11 shlq $32,%rax /* make sure bits 32:63 of %rax are zero */ shrq $32,%rax or %r11,%rax /* Save thread state in pcb->iss, as on exception entry. * Since this is triggered synchronously from userspace, we could * save only the callee-preserved status according to the C ABI, * plus RIP and EFLAGS for sysret */ movq MY(ACTIVE_THREAD),%r11 /* point to current thread */ movq TH_PCB(%r11),%r11 /* point to pcb */ addq $ PCB_ISS,%r11 /* point to saved state */ mov %rsp,R_UESP(%r11) /* callee-preserved register */ mov %rcx,R_EIP(%r11) /* syscall places user RIP in RCX */ mov %rbx,R_EBX(%r11) /* callee-preserved register */ mov %rax,%rbx /* Now we can unpack eflags again */ shr $32,%rbx mov %rbx,R_EFLAGS(%r11) /* ... and save them in pcb as well */ mov %rbp,R_EBP(%r11) /* callee-preserved register */ mov %r12,R_R12(%r11) /* callee-preserved register */ mov %r13,R_R13(%r11) /* callee-preserved register */ mov %r14,R_R14(%r11) /* callee-preserved register */ mov %r15,R_R15(%r11) /* callee-preserved register */ /* Save syscall number and args for SYSCALL_EXAMINE/MSG_EXAMINE in glibc. * Note: syscall number is only 32 bit, in EAX, so we sign-extend it in * RAX to mask the EFLAGS bits. */ cdqe /* sign-extend EAX in RAX */ mov %rax,R_EAX(%r11) /* syscall number */ mov %rdi,R_EDI(%r11) /* syscall arg0 */ mov %rsi,R_ESI(%r11) /* syscall arg1 */ mov %rdx,R_EDX(%r11) /* syscall arg2 */ mov %r10,R_R10(%r11) /* syscall arg3 */ mov %r8,R_R8(%r11) /* syscall arg4 */ mov %r9,R_R9(%r11) /* syscall arg5 */ mov %r11,%rbx /* prepare for error handling */ mov %r10,%rcx /* fix arg3 location according to C ABI */ /* switch to kernel stack, then we can enable interrupts */ CPU_NUMBER_NO_STACK(%r11d) movq CX(EXT(kernel_stack),%r11),%rsp sti /* Now we have saved state and args 1-6 are in place. * Before invoking the syscall we do some bound checking and, * if we have more that 6 arguments, we need to copy the * remaining ones to the kernel stack, handling page faults when * accessing the user stack. */ negl %eax /* get system call number */ jl _syscall64_range /* out of range if it was positive */ cmpl EXT(mach_trap_count),%eax /* check system call table bounds */ jg _syscall64_range /* error if out of range */ shll $5,%eax /* manual indexing of mach_trap_t */ /* check if we need to place some arguments on the stack */ _syscall64_args_stack: mov EXT(mach_trap_table)(%rax),%r10 /* get number of arguments */ subq $6,%r10 /* the first 6 args are already in place */ jle _syscall64_call /* skip argument copy if num args <= 6 */ movq R_UESP(%rbx),%r11 /* get user stack pointer */ addq $8,%r11 /* Skip user return address */ lea (%r11,%r10,8),%r11 /* point past last argument */ movq $VM_MAX_ADDRESS, %r12 cmpq %r12,%r11 /* Check segment limit by hand */ jae _syscall64_addr_push 0: subq $8,%r11 RECOVER(_syscall64_addr_push) mov (%r11),%r12 pushq %r12 /* push argument on stack */ dec %r10 jnz 0b /* loop for all remaining arguments */ _syscall64_call: call *EXT(mach_trap_table)+8(%rax) /* call procedure */ _syscall64_check_for_ast: /* Check for ast. */ CPU_NUMBER_NO_GS(%r11d) cmpl $0,CX(EXT(need_ast),%r11) jz _syscall64_restore_state /* Save the syscall return value, both on our stack, for the case * i386_astintr returns normally, and in the PCB stack, in case it * instead calls thread_block(thread_exception_return). */ pushq %rax /* save the return value on our stack */ pushq $0 /* dummy value to keep the stack aligned */ /* Find the PCB stack. */ movq %rsp,%rcx or $(KERNEL_STACK_SIZE-1),%rcx movq -7-IKS_SIZE(%rcx),%rcx movq %rax,R_EAX(%rcx) /* save the return value in the PCB stack */ call EXT(i386_astintr) popq %rax popq %rax /* restore the return value */ jmp _syscall64_check_for_ast /* check again */ _syscall64_restore_state: /* Restore thread state and return to user using sysret. */ cli /* block interrupts when using the user stack in kernel space */ movq MY(ACTIVE_THREAD),%r11 /* point to current thread */ movq TH_PCB(%r11),%r11 /* point to pcb */ addq $ PCB_ISS,%r11 /* point to saved state */ /* Restore syscall args. Note: we can't restore the syscall number in * RAX because it needs to hold the return value.*/ mov R_EDI(%r11),%rdi /* syscall arg0 */ mov R_ESI(%r11),%rsi /* syscall arg1 */ mov R_EDX(%r11),%rdx /* syscall arg2 */ mov R_R10(%r11),%r10 /* syscall arg3 */ mov R_R8(%r11),%r8 /* syscall arg4 */ mov R_R9(%r11),%r9 /* syscall arg5 */ mov R_UESP(%r11),%rsp /* callee-preserved register, * also switch back to user stack */ mov R_EIP(%r11),%rcx /* sysret convention */ mov R_EBX(%r11),%rbx /* callee-preserved register */ mov R_EBP(%r11),%rbp /* callee-preserved register */ mov R_R12(%r11),%r12 /* callee-preserved register */ mov R_R13(%r11),%r13 /* callee-preserved register */ mov R_R14(%r11),%r14 /* callee-preserved register */ mov R_R15(%r11),%r15 /* callee-preserved register */ mov R_EFLAGS(%r11),%r11 /* sysret convention */ sysretq /* fast return to user-space, the thread didn't block */ /* Error handling fragments, from here we jump directly to the trap handler */ _syscall64_addr_push: movq %r11,R_CR2(%rbx) /* set fault address */ movq $(T_PAGE_FAULT),R_TRAPNO(%rbx) /* set page-fault trap */ movq $(T_PF_USER),R_ERR(%rbx) /* set error code - read user space */ jmp _take_trap /* treat as a trap */ _syscall64_range: movq $(T_INVALID_OPCODE),R_TRAPNO(%rbx) /* set invalid-operation trap */ movq $0,R_ERR(%rbx) /* clear error code */ jmp _take_trap /* treat as a trap */ END(syscall64) #endif /* USER32 */ .data DATA(cpu_features) DATA(cpu_features_edx) .long 0 DATA(cpu_features_ecx) .long 0 .text /* Discover what kind of cpu we have; return the family number (3, 4, 5, 6, for 386, 486, 586, 686 respectively). */ ENTRY(discover_x86_cpu_type) /* We are a modern enough processor to have the CPUID instruction; use it to find out what we are. */ movl $1,%eax /* Fetch CPU type info ... */ cpuid /* ... into eax */ movl %ecx,cpu_features_ecx /* Keep a copy */ movl %edx,cpu_features_edx /* Keep a copy */ shrl $8,%eax /* Slide family bits down */ andl $15,%eax /* And select them */ ret /* And return */ /* */ /* * Utility routines. */ ENTRY(copyin) xchgq %rsi,%rdi /* Get user source and kernel destination */ movq $VM_MAX_ADDRESS, %rcx cmpq %rcx,%rsi /* Check segment limit by hand */ jae copyin_fail copyin_remainder: /*cld*/ /* count up: default mode in all GCC code */ movq %rdx,%rcx /* move by longwords first */ shrq $3,%rcx RECOVER(copyin_fail) rep movsq /* move longwords */ movq %rdx,%rcx /* now move remaining bytes */ andq $7,%rcx RECOVER(copyin_fail) rep movsb xorq %rax,%rax /* return 0 for success */ copyin_ret: ret /* and return */ copyin_fail: movq $1,%rax /* return 1 for failure */ jmp copyin_ret /* pop frame and return */ bogus: ud2 ENTRY(copyout) xchgq %rsi,%rdi /* Get user source and kernel destination */ movq $VM_MAX_ADDRESS, %rcx cmpq %rcx,%rdi /* Check segment limit by hand */ jae copyin_fail copyout_remainder: movq %rdx,%rax /* use count */ /*cld*/ /* count up: always this way in GCC code */ movq %rax,%rcx /* move by longwords first */ shrq $3,%rcx RECOVER(copyout_fail) rep movsq movq %rax,%rcx /* now move remaining bytes */ andq $7,%rcx RECOVER(copyout_fail) rep movsb /* move */ xorq %rax,%rax /* return 0 for success */ copyout_ret: ret /* and return */ copyout_fail: movq $1,%rax /* return 1 for failure */ jmp copyout_ret /* pop frame and return */ /* * int inst_fetch(int eip, int cs); * * Fetch instruction byte. Return -1 if invalid address. */ ENTRY(inst_fetch) movq S_ARG1, %rax /* get segment */ movw %ax,%fs /* into FS */ movq S_ARG0, %rax /* get offset */ movq $VM_MAX_ADDRESS, %rcx cmpq %rcx,%rax /* Check segment limit by hand */ jae _inst_fetch_fault RETRY(EXT(inst_fetch)) /* re-load FS on retry */ RECOVER(_inst_fetch_fault) movzbq %fs:(%rax),%rax /* load instruction byte */ ret _inst_fetch_fault: movq $-1,%rax /* return -1 if error */ ret /* * Done with recovery and retry tables. */ RECOVER_TABLE_END RETRY_TABLE_END /* * cpu_shutdown() * Force reboot */ null_idt: .space 8 * 32 null_idtr: .word 8 * 32 - 1 .quad null_idt Entry(cpu_shutdown) lidt null_idtr /* disable the interrupt handler */ xor %rcx,%rcx /* generate a divide by zero */ div %rcx,%rax /* reboot now */ ret /* this will "never" be executed */