diff options
author | Konstantin Belousov <kib@FreeBSD.org> | 2011-09-11 16:05:09 +0000 |
---|---|---|
committer | Konstantin Belousov <kib@FreeBSD.org> | 2011-09-11 16:05:09 +0000 |
commit | 26ccf4f10f9288c19ebe345b0dee7396d6b12ee2 (patch) | |
tree | 0b0a99497de53c0baa2f389ba7ca1c651e4df409 /sys | |
parent | 637b8c6d881a99edd178203161fda260bc054d0d (diff) | |
download | src-26ccf4f10f9288c19ebe345b0dee7396d6b12ee2.tar.gz src-26ccf4f10f9288c19ebe345b0dee7396d6b12ee2.zip |
Inline the syscallenter() and syscallret(). This reduces the time measured
by the syscall entry speed microbenchmarks by ~10% on amd64.
Submitted by: jhb
Approved by: re (bz)
MFC after: 2 weeks
Notes
Notes:
svn path=/head/; revision=225474
Diffstat (limited to 'sys')
-rw-r--r-- | sys/amd64/amd64/trap.c | 2 | ||||
-rw-r--r-- | sys/amd64/ia32/ia32_syscall.c | 2 | ||||
-rw-r--r-- | sys/i386/i386/trap.c | 2 | ||||
-rw-r--r-- | sys/ia64/ia32/ia32_trap.c | 2 | ||||
-rw-r--r-- | sys/ia64/ia64/trap.c | 2 | ||||
-rw-r--r-- | sys/kern/subr_syscall.c | 213 | ||||
-rw-r--r-- | sys/kern/subr_trap.c | 162 | ||||
-rw-r--r-- | sys/powerpc/aim/trap.c | 2 | ||||
-rw-r--r-- | sys/powerpc/booke/trap.c | 2 | ||||
-rw-r--r-- | sys/sparc64/sparc64/trap.c | 2 | ||||
-rw-r--r-- | sys/sys/proc.h | 3 |
11 files changed, 229 insertions, 165 deletions
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c index 4e5f8b8aa9f5..a36a6be13ce3 100644 --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -883,6 +883,8 @@ cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa) return (error); } +#include "../../kern/subr_syscall.c" + /* * syscall - system call request C handler * diff --git a/sys/amd64/ia32/ia32_syscall.c b/sys/amd64/ia32/ia32_syscall.c index 0b46dae7a5e3..d79272a01dcf 100644 --- a/sys/amd64/ia32/ia32_syscall.c +++ b/sys/amd64/ia32/ia32_syscall.c @@ -163,6 +163,8 @@ ia32_fetch_syscall_args(struct thread *td, struct syscall_args *sa) return (error); } +#include "../../kern/subr_syscall.c" + void ia32_syscall(struct trapframe *frame) { diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c index 5a8016c9c69d..dddecaa807ff 100644 --- a/sys/i386/i386/trap.c +++ b/sys/i386/i386/trap.c @@ -1054,6 +1054,8 @@ cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa) return (error); } +#include "../../kern/subr_syscall.c" + /* * syscall - system call request C handler * diff --git a/sys/ia64/ia32/ia32_trap.c b/sys/ia64/ia32/ia32_trap.c index 57f479169560..6755358189a1 100644 --- a/sys/ia64/ia32/ia32_trap.c +++ b/sys/ia64/ia32/ia32_trap.c @@ -144,6 +144,8 @@ ia32_fetch_syscall_args(struct thread *td, struct syscall_args *sa) return (error); } +#include "../../kern/subr_syscall.c" + static void ia32_syscall(struct trapframe *tf) { diff --git a/sys/ia64/ia64/trap.c b/sys/ia64/ia64/trap.c index 4b84f769340e..d587b05868fd 100644 --- a/sys/ia64/ia64/trap.c +++ b/sys/ia64/ia64/trap.c @@ -929,6 +929,8 @@ cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa) return (0); } +#include "../../kern/subr_syscall.c" + /* * Process a system call. * diff --git a/sys/kern/subr_syscall.c b/sys/kern/subr_syscall.c new file mode 100644 index 000000000000..cb0d9295ed2b --- /dev/null +++ b/sys/kern/subr_syscall.c @@ -0,0 +1,213 @@ +/*- + * Copyright (C) 1994, David Greenman + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * Copyright (C) 2010 Konstantin Belousov <kib@freebsd.org> + * + * This code is derived from software contributed to Berkeley by + * the University of Utah, and William Jolitz. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: @(#)trap.c 7.4 (Berkeley) 5/13/91 + */ + +#include "opt_capsicum.h" +#include "opt_ktrace.h" +#include "opt_kdtrace.h" + +__FBSDID("$FreeBSD$"); + +#include <sys/capability.h> +#include <sys/ktr.h> +#ifdef KTRACE +#include <sys/uio.h> +#include <sys/ktrace.h> +#endif +#include <security/audit/audit.h> + +static inline int +syscallenter(struct thread *td, struct syscall_args *sa) +{ + struct proc *p; + int error, traced; + + PCPU_INC(cnt.v_syscall); + p = td->td_proc; + + td->td_pticks = 0; + if (td->td_ucred != p->p_ucred) + cred_update_thread(td); + if (p->p_flag & P_TRACED) { + traced = 1; + PROC_LOCK(p); + td->td_dbgflags &= ~TDB_USERWR; + td->td_dbgflags |= TDB_SCE; + PROC_UNLOCK(p); + } else + traced = 0; + error = (p->p_sysent->sv_fetch_syscall_args)(td, sa); +#ifdef KTRACE + if (KTRPOINT(td, KTR_SYSCALL)) + ktrsyscall(sa->code, sa->narg, sa->args); +#endif + + CTR6(KTR_SYSC, +"syscall: td=%p pid %d %s (%#lx, %#lx, %#lx)", + td, td->td_proc->p_pid, syscallname(p, sa->code), + sa->args[0], sa->args[1], sa->args[2]); + + if (error == 0) { + STOPEVENT(p, S_SCE, sa->narg); + PTRACESTOP_SC(p, td, S_PT_SCE); + if (td->td_dbgflags & TDB_USERWR) { + /* + * Reread syscall number and arguments if + * debugger modified registers or memory. + */ + error = (p->p_sysent->sv_fetch_syscall_args)(td, sa); +#ifdef KTRACE + if (KTRPOINT(td, KTR_SYSCALL)) + ktrsyscall(sa->code, sa->narg, sa->args); +#endif + if (error != 0) + goto retval; + } + +#ifdef CAPABILITY_MODE + /* + * In capability mode, we only allow access to system calls + * flagged with SYF_CAPENABLED. + */ + if (IN_CAPABILITY_MODE(td) && + !(sa->callp->sy_flags & SYF_CAPENABLED)) { + error = ECAPMODE; + goto retval; + } +#endif + + error = syscall_thread_enter(td, sa->callp); + if (error != 0) + goto retval; + +#ifdef KDTRACE_HOOKS + /* + * If the systrace module has registered it's probe + * callback and if there is a probe active for the + * syscall 'entry', process the probe. + */ + if (systrace_probe_func != NULL && sa->callp->sy_entry != 0) + (*systrace_probe_func)(sa->callp->sy_entry, sa->code, + sa->callp, sa->args, 0); +#endif + + AUDIT_SYSCALL_ENTER(sa->code, td); + error = (sa->callp->sy_call)(td, sa->args); + AUDIT_SYSCALL_EXIT(error, td); + + /* Save the latest error return value. */ + td->td_errno = error; + +#ifdef KDTRACE_HOOKS + /* + * If the systrace module has registered it's probe + * callback and if there is a probe active for the + * syscall 'return', process the probe. + */ + if (systrace_probe_func != NULL && sa->callp->sy_return != 0) + (*systrace_probe_func)(sa->callp->sy_return, sa->code, + sa->callp, NULL, (error) ? -1 : td->td_retval[0]); +#endif + syscall_thread_exit(td, sa->callp); + CTR4(KTR_SYSC, "syscall: p=%p error=%d return %#lx %#lx", + p, error, td->td_retval[0], td->td_retval[1]); + } + retval: + if (traced) { + PROC_LOCK(p); + td->td_dbgflags &= ~TDB_SCE; + PROC_UNLOCK(p); + } + (p->p_sysent->sv_set_syscall_retval)(td, error); + return (error); +} + +static inline void +syscallret(struct thread *td, int error, struct syscall_args *sa __unused) +{ + struct proc *p; + int traced; + + p = td->td_proc; + + /* + * Check for misbehavior. + */ + WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning", + syscallname(p, sa->code)); + KASSERT(td->td_critnest == 0, + ("System call %s returning in a critical section", + syscallname(p, sa->code))); + KASSERT(td->td_locks == 0, + ("System call %s returning with %d locks held", + syscallname(p, sa->code), td->td_locks)); + + /* + * Handle reschedule and other end-of-syscall issues + */ + userret(td, td->td_frame); + + CTR4(KTR_SYSC, "syscall %s exit thread %p pid %d proc %s", + syscallname(p, sa->code), td, td->td_proc->p_pid, td->td_name); + +#ifdef KTRACE + if (KTRPOINT(td, KTR_SYSRET)) + ktrsysret(sa->code, error, td->td_retval[0]); +#endif + + if (p->p_flag & P_TRACED) { + traced = 1; + PROC_LOCK(p); + td->td_dbgflags |= TDB_SCX; + PROC_UNLOCK(p); + } else + traced = 0; + /* + * This works because errno is findable through the + * register set. If we ever support an emulation where this + * is not the case, this code will need to be revisited. + */ + STOPEVENT(p, S_SCX, sa->code); + PTRACESTOP_SC(p, td, S_PT_SCX); + if (traced || (td->td_dbgflags & (TDB_EXEC | TDB_FORK)) != 0) { + PROC_LOCK(p); + td->td_dbgflags &= ~(TDB_SCX | TDB_EXEC | TDB_FORK); + PROC_UNLOCK(p); + } +} diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c index 3527ed1fbedf..42adf80f04fe 100644 --- a/sys/kern/subr_trap.c +++ b/sys/kern/subr_trap.c @@ -252,7 +252,6 @@ ast(struct trapframe *framep) mtx_assert(&Giant, MA_NOTOWNED); } -#ifdef HAVE_SYSCALL_ARGS_DEF const char * syscallname(struct proc *p, u_int code) { @@ -264,164 +263,3 @@ syscallname(struct proc *p, u_int code) return (unknown); return (sv->sv_syscallnames[code]); } - -int -syscallenter(struct thread *td, struct syscall_args *sa) -{ - struct proc *p; - int error, traced; - - PCPU_INC(cnt.v_syscall); - p = td->td_proc; - - td->td_pticks = 0; - if (td->td_ucred != p->p_ucred) - cred_update_thread(td); - if (p->p_flag & P_TRACED) { - traced = 1; - PROC_LOCK(p); - td->td_dbgflags &= ~TDB_USERWR; - td->td_dbgflags |= TDB_SCE; - PROC_UNLOCK(p); - } else - traced = 0; - error = (p->p_sysent->sv_fetch_syscall_args)(td, sa); -#ifdef KTRACE - if (KTRPOINT(td, KTR_SYSCALL)) - ktrsyscall(sa->code, sa->narg, sa->args); -#endif - - CTR6(KTR_SYSC, -"syscall: td=%p pid %d %s (%#lx, %#lx, %#lx)", - td, td->td_proc->p_pid, syscallname(p, sa->code), - sa->args[0], sa->args[1], sa->args[2]); - - if (error == 0) { - STOPEVENT(p, S_SCE, sa->narg); - PTRACESTOP_SC(p, td, S_PT_SCE); - if (td->td_dbgflags & TDB_USERWR) { - /* - * Reread syscall number and arguments if - * debugger modified registers or memory. - */ - error = (p->p_sysent->sv_fetch_syscall_args)(td, sa); -#ifdef KTRACE - if (KTRPOINT(td, KTR_SYSCALL)) - ktrsyscall(sa->code, sa->narg, sa->args); -#endif - if (error != 0) - goto retval; - } - -#ifdef CAPABILITY_MODE - /* - * In capability mode, we only allow access to system calls - * flagged with SYF_CAPENABLED. - */ - if (IN_CAPABILITY_MODE(td) && - !(sa->callp->sy_flags & SYF_CAPENABLED)) { - error = ECAPMODE; - goto retval; - } -#endif - - error = syscall_thread_enter(td, sa->callp); - if (error != 0) - goto retval; - -#ifdef KDTRACE_HOOKS - /* - * If the systrace module has registered it's probe - * callback and if there is a probe active for the - * syscall 'entry', process the probe. - */ - if (systrace_probe_func != NULL && sa->callp->sy_entry != 0) - (*systrace_probe_func)(sa->callp->sy_entry, sa->code, - sa->callp, sa->args, 0); -#endif - - AUDIT_SYSCALL_ENTER(sa->code, td); - error = (sa->callp->sy_call)(td, sa->args); - AUDIT_SYSCALL_EXIT(error, td); - - /* Save the latest error return value. */ - td->td_errno = error; - -#ifdef KDTRACE_HOOKS - /* - * If the systrace module has registered it's probe - * callback and if there is a probe active for the - * syscall 'return', process the probe. - */ - if (systrace_probe_func != NULL && sa->callp->sy_return != 0) - (*systrace_probe_func)(sa->callp->sy_return, sa->code, - sa->callp, NULL, (error) ? -1 : td->td_retval[0]); -#endif - syscall_thread_exit(td, sa->callp); - CTR4(KTR_SYSC, "syscall: p=%p error=%d return %#lx %#lx", - p, error, td->td_retval[0], td->td_retval[1]); - } - retval: - if (traced) { - PROC_LOCK(p); - td->td_dbgflags &= ~TDB_SCE; - PROC_UNLOCK(p); - } - (p->p_sysent->sv_set_syscall_retval)(td, error); - return (error); -} - -void -syscallret(struct thread *td, int error, struct syscall_args *sa __unused) -{ - struct proc *p; - int traced; - - p = td->td_proc; - - /* - * Check for misbehavior. - */ - WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning", - syscallname(p, sa->code)); - KASSERT(td->td_critnest == 0, - ("System call %s returning in a critical section", - syscallname(p, sa->code))); - KASSERT(td->td_locks == 0, - ("System call %s returning with %d locks held", - syscallname(p, sa->code), td->td_locks)); - - /* - * Handle reschedule and other end-of-syscall issues - */ - userret(td, td->td_frame); - - CTR4(KTR_SYSC, "syscall %s exit thread %p pid %d proc %s", - syscallname(p, sa->code), td, td->td_proc->p_pid, td->td_name); - -#ifdef KTRACE - if (KTRPOINT(td, KTR_SYSRET)) - ktrsysret(sa->code, error, td->td_retval[0]); -#endif - - if (p->p_flag & P_TRACED) { - traced = 1; - PROC_LOCK(p); - td->td_dbgflags |= TDB_SCX; - PROC_UNLOCK(p); - } else - traced = 0; - /* - * This works because errno is findable through the - * register set. If we ever support an emulation where this - * is not the case, this code will need to be revisited. - */ - STOPEVENT(p, S_SCX, sa->code); - PTRACESTOP_SC(p, td, S_PT_SCX); - if (traced || (td->td_dbgflags & (TDB_EXEC | TDB_FORK)) != 0) { - PROC_LOCK(p); - td->td_dbgflags &= ~(TDB_SCX | TDB_EXEC | TDB_FORK); - PROC_UNLOCK(p); - } -} -#endif /* HAVE_SYSCALL_ARGS_DEF */ diff --git a/sys/powerpc/aim/trap.c b/sys/powerpc/aim/trap.c index 51dee0eabdb5..93feb515728d 100644 --- a/sys/powerpc/aim/trap.c +++ b/sys/powerpc/aim/trap.c @@ -445,6 +445,8 @@ cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa) return (error); } +#include "../../kern/subr_syscall.c" + void syscall(struct trapframe *frame) { diff --git a/sys/powerpc/booke/trap.c b/sys/powerpc/booke/trap.c index fc506bd361ce..091fce08f1fb 100644 --- a/sys/powerpc/booke/trap.c +++ b/sys/powerpc/booke/trap.c @@ -375,6 +375,8 @@ cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa) return (error); } +#include "../../kern/subr_syscall.c" + void syscall(struct trapframe *frame) { diff --git a/sys/sparc64/sparc64/trap.c b/sys/sparc64/sparc64/trap.c index db05f06ba926..7037e462ad0e 100644 --- a/sys/sparc64/sparc64/trap.c +++ b/sys/sparc64/sparc64/trap.c @@ -606,6 +606,8 @@ cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa) return (error); } +#include "../../kern/subr_syscall.c" + /* * Syscall handler * The arguments to the syscall are passed in the out registers by the caller, diff --git a/sys/sys/proc.h b/sys/sys/proc.h index 67adbe5b5899..fb9791303ddc 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -873,9 +873,6 @@ void cpu_switch(struct thread *, struct thread *, struct mtx *); void cpu_throw(struct thread *, struct thread *) __dead2; void unsleep(struct thread *); void userret(struct thread *, struct trapframe *); -struct syscall_args; -int syscallenter(struct thread *, struct syscall_args *); -void syscallret(struct thread *, int, struct syscall_args *); void cpu_exit(struct thread *); void exit1(struct thread *, int) __dead2; |