Kernel and hwpmc(4) support for callchain capture.

Sponsored by:	FreeBSD Foundation and Google Inc.

3 files changed, 77 insertions, 5 deletions

diff --git a/sys/i386/i386/exception.s b/sys/i386/i386/exception.s
index 259327d67207..31d9a66696e5 100644
--- a/sys/i386/i386/exception.s
+++ b/sys/i386/i386/exception.s
@@ -1,8 +1,12 @@
 /*-
  * Copyright (c) 1989, 1990 William F. Jolitz.
  * Copyright (c) 1990 The Regents of the University of California.
+ * Copyright (c) 2007 The FreeBSD Foundation
  * All rights reserved.
  *
+ * Portions of this software were developed by A. Joseph Koshy under
+ * sponsorship from the FreeBSD Foundation and Google, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -31,6 +35,7 @@
  */
 
 #include "opt_apic.h"
+#include "opt_hwpmc_hooks.h"
 #include "opt_npx.h"
 
 #include <machine/asmacros.h>
@@ -42,7 +47,9 @@
 #define	SEL_RPL_MASK	0x0003
 
 	.text
-
+#ifdef HWPMC_HOOKS
+	ENTRY(start_exceptions)
+#endif
 /*****************************************************************************/
 /* Trap handling                                                             */
 /*****************************************************************************/
@@ -261,8 +268,18 @@ doreti:
 	FAKE_MCOUNT($bintr)		/* init "from" bintr -> doreti */
 doreti_next:
 	/*
-	 * Check if ASTs can be handled now.  PSL_VM must be checked first
-	 * since segment registers only have an RPL in non-VM86 mode.
+	 * Check if ASTs can be handled now.  ASTs cannot be safely
+	 * processed when returning from an NMI.
+	 */
+	cmpb	$T_NMI,TF_TRAPNO(%esp)
+#ifdef HWPMC_HOOKS
+	je	doreti_nmi
+#else
+	je	doreti_exit
+#endif
+	/*
+	 * PSL_VM must be checked first since segment registers only
+	 * have an RPL in non-VM86 mode.
 	 */
 	testl	$PSL_VM,TF_EFLAGS(%esp)	/* are we in vm86 mode? */
 	jz	doreti_notvm86
@@ -340,3 +357,32 @@ doreti_popl_fs_fault:
 	movl	$0,TF_ERR(%esp)	/* XXX should be the error code */
 	movl	$T_PROTFLT,TF_TRAPNO(%esp)
 	jmp	alltraps_with_regs_pushed
+#ifdef HWPMC_HOOKS
+doreti_nmi:
+	/*
+	 * Since we are returning from an NMI, check if the current trap
+	 * was from user mode and if so whether the current thread
+	 * needs a user call chain capture.
+	 */
+	testb	$SEL_RPL_MASK,TF_CS(%esp)
+	jz	doreti_exit
+	movl	PCPU(CURTHREAD),%eax	/* curthread present? */
+	orl	%eax,%eax
+	jz	doreti_exit
+	testl	$TDP_CALLCHAIN,TD_PFLAGS(%eax) /* flagged for capture? */
+	jz	doreti_exit
+	/*
+	 * Take the processor out of NMI mode by executing a fake "iret".
+	 */
+	pushfl
+	pushl	%cs
+	pushl	$outofnmi
+	iret
+outofnmi:
+	/*
+	 * Clear interrupts and jump to AST handling code.
+	 */
+	sti
+	jmp	doreti_ast
+	ENTRY(end_exceptions)
+#endif
diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c
index ebec7ab10ca4..d6aa97356d94 100644
--- a/sys/i386/i386/trap.c
+++ b/sys/i386/i386/trap.c
@@ -214,8 +214,7 @@ trap(struct trapframe *frame)
 	 * return immediately.
 	 */
 	if (type == T_NMI && pmc_intr &&
-	    (*pmc_intr)(PCPU_GET(cpuid), (uintptr_t) frame->tf_eip,
-		TRAPF_USERMODE(frame)))
+	    (*pmc_intr)(PCPU_GET(cpuid), frame))
 	    goto out;
 #endif
 
diff --git a/sys/i386/include/pmc_mdep.h b/sys/i386/include/pmc_mdep.h
index 2d4c4a363de7..ffe94dd38329 100644
--- a/sys/i386/include/pmc_mdep.h
+++ b/sys/i386/include/pmc_mdep.h
@@ -1,7 +1,11 @@
 /*-
  * Copyright (c) 2003-2005 Joseph Koshy
+ * Copyright (c) 2007 The FreeBSD Foundation
  * All rights reserved.
  *
+ * Portions of this software were developed by A. Joseph Koshy under
+ * sponsorship from the FreeBSD Foundation and Google, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -73,10 +77,33 @@ union pmc_md_pmc  {
 
 struct pmc;
 
+#define	PMC_TRAPFRAME_TO_PC(TF)	((TF)->tf_eip)
+#define	PMC_TRAPFRAME_TO_FP(TF)	((TF)->tf_ebp)
+#define	PMC_TRAPFRAME_TO_SP(TF)	((TF)->tf_esp)
+
+#define	PMC_IN_KERNEL_STACK(S,START,END)		\
+	((S) >= (START) && (S) < (END))
+#define	PMC_IN_KERNEL(va) (((va) >= USRSTACK) &&	\
+	((va) < VM_MAX_KERNEL_ADDRESS))
+
+#define	PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS)
+
+#define	PMC_IN_TRAP_HANDLER(PC) 			\
+	((PC) >= (uintptr_t) start_exceptions &&	\
+	 (PC) < (uintptr_t) end_exceptions)
+
+#define	PMC_AT_FUNCTION_PROLOGUE_PUSH_BP(I)		\
+	(((I) & 0xffffffff) == 0xe5894855) /* pushq %rbp; movq %rsp,%rbp */
+#define	PMC_AT_FUNCTION_PROLOGUE_MOV_SP_BP(I)		\
+	(((I) & 0x00ffffff) == 0x00e58948) /* movq %rsp,%rbp */
+#define	PMC_AT_FUNCTION_EPILOGUE_RET(I)			\
+	(((I) & 0xFF) == 0xC3)		   /* ret */
+
 /*
  * Prototypes
  */
 
+void	start_exceptions(void), end_exceptions(void);
 void	pmc_x86_lapic_enable_pmc_interrupt(void);
 
 #endif /* _KERNEL */