epoch(9): allow preemptible epochs to compose

- Add tracker argument to preemptible epochs
- Inline epoch read path in kernel and tied modules
- Change in_epoch to take an epoch as argument
- Simplify tfb_tcp_do_segment to not take a ti_locked argument,
  there's no longer any benefit to dropping the pcbinfo lock
  and trying to do so just adds an error prone branchfest to
  these functions
- Remove cases of same function recursion on the epoch as
  recursing is no longer free.
- Remove the the TAILQ_ENTRY and epoch_section from struct
  thread as the tracker field is now stack or heap allocated
  as appropriate.

Tested by: pho and Limelight Networks
Reviewed by: kbowling at llnw dot com
Sponsored by: Limelight Networks
Differential Revision: https://reviews.freebsd.org/D16066

1 files changed, 27 insertions, 24 deletions

diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c
index e09151a6d339..4d7a4535d27b 100644
--- a/sys/dev/hwpmc/hwpmc_mod.c
+++ b/sys/dev/hwpmc/hwpmc_mod.c
@@ -85,6 +85,9 @@ __FBSDID("$FreeBSD$");
 #define free_domain(addr, type) free(addr, type)
 #endif
 
+#define PMC_EPOCH_ENTER() struct epoch_tracker pmc_et; epoch_enter_preempt(global_epoch_preempt, &pmc_et)
+#define PMC_EPOCH_EXIT() epoch_exit_preempt(global_epoch_preempt, &pmc_et)
+
 /*
  * Types
  */
@@ -1752,12 +1755,12 @@ pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm)
 	const struct pmc_process *pp;
 
 	freepath = fullpath = NULL;
-	MPASS(!in_epoch());
+	MPASS(!in_epoch(global_epoch_preempt));
 	pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath);
 
 	pid = td->td_proc->p_pid;
 
-	epoch_enter_preempt(global_epoch_preempt);
+	PMC_EPOCH_ENTER();
 	/* Inform owners of all system-wide sampling PMCs. */
 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
@@ -1778,7 +1781,7 @@ pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm)
   done:
 	if (freepath)
 		free(freepath, M_TEMP);
-	epoch_exit_preempt(global_epoch_preempt);
+	PMC_EPOCH_EXIT();
 }
 
 
@@ -1797,12 +1800,12 @@ pmc_process_munmap(struct thread *td, struct pmckern_map_out *pkm)
 
 	pid = td->td_proc->p_pid;
 
-	epoch_enter_preempt(global_epoch_preempt);
+	PMC_EPOCH_ENTER();
 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
 		pmclog_process_map_out(po, pid, pkm->pm_address,
 		    pkm->pm_address + pkm->pm_size);
-	epoch_exit_preempt(global_epoch_preempt);
+	PMC_EPOCH_EXIT();
 
 	if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
 		return;
@@ -1824,7 +1827,7 @@ pmc_log_kernel_mappings(struct pmc *pm)
 	struct pmc_owner *po;
 	struct pmckern_map_in *km, *kmbase;
 
-	MPASS(in_epoch() || sx_xlocked(&pmc_sx));
+	MPASS(in_epoch(global_epoch_preempt) || sx_xlocked(&pmc_sx));
 	KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
 	    ("[pmc,%d] non-sampling PMC (%p) desires mapping information",
 		__LINE__, (void *) pm));
@@ -2106,13 +2109,13 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
 
 		pk = (struct pmckern_procexec *) arg;
 
-		epoch_enter_preempt(global_epoch_preempt);
+		PMC_EPOCH_ENTER();
 		/* Inform owners of SS mode PMCs of the exec event. */
 		CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
 		    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
 			    pmclog_process_procexec(po, PMC_ID_INVALID,
 				p->p_pid, pk->pm_entryaddr, fullpath);
-		epoch_exit_preempt(global_epoch_preempt);
+		PMC_EPOCH_EXIT();
 
 		PROC_LOCK(p);
 		is_using_hwpmcs = p->p_flag & P_HWPMC;
@@ -2242,7 +2245,7 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
 		break;
 
 	case PMC_FN_MUNMAP:
-		MPASS(in_epoch() || sx_xlocked(&pmc_sx));
+		MPASS(in_epoch(global_epoch_preempt) || sx_xlocked(&pmc_sx));
 		pmc_process_munmap(td, (struct pmckern_map_out *) arg);
 		break;
 
@@ -2479,7 +2482,7 @@ pmc_find_thread_descriptor(struct pmc_process *pp, struct thread *td,
 	if (mode & PMC_FLAG_ALLOCATE) {
 		if ((ptnew = pmc_thread_descriptor_pool_alloc()) == NULL) {
 			wait_flag = M_WAITOK;
-			if ((mode & PMC_FLAG_NOWAIT) || in_epoch())
+			if ((mode & PMC_FLAG_NOWAIT) || in_epoch(global_epoch_preempt))
 				wait_flag = M_NOWAIT;
 
 			ptnew = malloc(THREADENTRY_SIZE, M_PMC,
@@ -5070,11 +5073,11 @@ pmc_process_exit(void *arg __unused, struct proc *p)
 	/*
 	 * Log a sysexit event to all SS PMC owners.
 	 */
-	epoch_enter_preempt(global_epoch_preempt);
+	PMC_EPOCH_ENTER();
 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
 		    pmclog_process_sysexit(po, p->p_pid);
-	epoch_exit_preempt(global_epoch_preempt);
+	PMC_EPOCH_EXIT();
 
 	if (!is_using_hwpmcs)
 		return;
@@ -5255,13 +5258,13 @@ pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc,
 	 * If there are system-wide sampling PMCs active, we need to
 	 * log all fork events to their owner's logs.
 	 */
-	epoch_enter_preempt(global_epoch_preempt);
+	PMC_EPOCH_ENTER();
 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
 		    pmclog_process_procfork(po, p1->p_pid, newproc->p_pid);
 			pmclog_process_proccreate(po, newproc, 1);
 		}
-	epoch_exit_preempt(global_epoch_preempt);
+	PMC_EPOCH_EXIT();
 
 	if (!is_using_hwpmcs)
 		return;
@@ -5327,11 +5330,11 @@ pmc_process_threadcreate(struct thread *td)
 {
 	struct pmc_owner *po;
 
-	epoch_enter_preempt(global_epoch_preempt);
+	PMC_EPOCH_ENTER();
 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
 			pmclog_process_threadcreate(po, td, 1);
-	epoch_exit_preempt(global_epoch_preempt);
+	PMC_EPOCH_EXIT();
 }
 
 static void
@@ -5339,11 +5342,11 @@ pmc_process_threadexit(struct thread *td)
 {
 	struct pmc_owner *po;
 
-	epoch_enter_preempt(global_epoch_preempt);
+	PMC_EPOCH_ENTER();
 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
 			pmclog_process_threadexit(po, td);
-	epoch_exit_preempt(global_epoch_preempt);
+	PMC_EPOCH_EXIT();
 }
 
 static void
@@ -5351,11 +5354,11 @@ pmc_process_proccreate(struct proc *p)
 {
 	struct pmc_owner *po;
 
-	epoch_enter_preempt(global_epoch_preempt);
+	PMC_EPOCH_ENTER();
 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
 			pmclog_process_proccreate(po, p, 1 /* sync */);
-	epoch_exit_preempt(global_epoch_preempt);
+	PMC_EPOCH_EXIT();
 }
 
 static void
@@ -5388,12 +5391,12 @@ pmc_kld_load(void *arg __unused, linker_file_t lf)
 	/*
 	 * Notify owners of system sampling PMCs about KLD operations.
 	 */
-	epoch_enter_preempt(global_epoch_preempt);
+	PMC_EPOCH_ENTER();
 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
 		if (po->po_flags & PMC_PO_OWNS_LOGFILE)
 			pmclog_process_map_in(po, (pid_t) -1,
 			    (uintfptr_t) lf->address, lf->filename);
-	epoch_exit_preempt(global_epoch_preempt);
+	PMC_EPOCH_EXIT();
 
 	/*
 	 * TODO: Notify owners of (all) process-sampling PMCs too.
@@ -5406,12 +5409,12 @@ pmc_kld_unload(void *arg __unused, const char *filename __unused,
 {
 	struct pmc_owner *po;
 
-	epoch_enter_preempt(global_epoch_preempt);
+	PMC_EPOCH_ENTER();
 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
 		if (po->po_flags & PMC_PO_OWNS_LOGFILE)
 			pmclog_process_map_out(po, (pid_t) -1,
 			    (uintfptr_t) address, (uintfptr_t) address + size);
-	epoch_exit_preempt(global_epoch_preempt);
+	PMC_EPOCH_EXIT();
 
 	/*
 	 * TODO: Notify owners of process-sampling PMCs.