- Add support for sending IPIs with USIII and greater sun4u CPUs.

  These CPUs use an enhanced layout of the interrupt vector dispatch
  and dispatch status registers in order to allow sending IPIs to
  multiple targets simultaneously. Thus support for these CPUs was
  put in a newly added cheetah_ipi_selected(). This is intended to
  be pointed to by cpu_ipi_selected, which now is a function pointer,
  in order to avoid cpu_impl checks once booted. Alternatively it
  can point to spitfire_ipi_selected(), which was renamed from
  cpu_ipi_selected(). Consequently cpu_ipi_send() was also renamed
  to spitfire_ipi_send() (there's no need for a cheetah equivalent
  of this so far). Initialization of the cpu_ipi_selected pointer
  and other requirements is done in mp_init(), which was renamed
  from mp_tramp_alloc(), as cpu_mp_start() isn't called on UP
  systems while cpu_ipi_selected() is. As a side-effect this allows
  to make mp_tramp static to sys/sparc64/sparc64/mp_machdep.c.
  For the sake of avoiding #ifdef SMP and for keeping the history in
  place cheetah_ipi_selected() and spitfire_ipi_{selected,send}()
  where not put into/moved to sys/sparc64/sparc64/{cheetah,spitfire}.c
- Add some CTASSERTs and KASSERTs ensuring that MAXCPU doesn't
  exceed the data types we use to store the CPU bit fields or the
  number of USIII and greater CPUs supported by the current
  cheetah_ipi_selected() implementation (which for JBus-CPUs is
  only 4; that should be fine though as according to OpenSolaris
  there are no sun4u machines with more than 4 JBus-CPUs).
- In cpu_mp_start() don't enumerate and start more than MAXCPU CPUs
  as we can't handle more than that.
- In cpu_mp_start() check for upa-portid vs. portid depending on
  cpu_impl for consistency with nexus(4).
- In spitfire_ipi_selected() add KASSERTs ensuring that a CPU isn't
  told to IPI itself as sun4u CPUs just can't do that.
- In spitfire_ipi_send() do a MEMBAR #Sync after writing the
  interrupt vector data as we want to make sure the payload was
  actually written before we trigger the dispatch.
- In spitfire_ipi_send() also verify IDR_BUSY when checking whether
  the dispatch was successful as it has to be cleared for this to
  be the case.
- Remove some redundant variables.

3 files changed, 132 insertions, 42 deletions

diff --git a/sys/sparc64/include/smp.h b/sys/sparc64/include/smp.h
index 324a0032802a..01127e83adc3 100644
--- a/sys/sparc64/include/smp.h
+++ b/sys/sparc64/include/smp.h
@@ -39,10 +39,15 @@
 #include <machine/pcb.h>
 #include <machine/tte.h>
 
-#define	IDR_BUSY	(1<<0)
-#define	IDR_NACK	(1<<1)
+#define	IDR_BUSY			0x0000000000000001ULL
+#define	IDR_NACK			0x0000000000000002ULL
+#define	IDR_CHEETAH_ALL_BUSY		0x5555555555555555ULL
+#define	IDR_CHEETAH_ALL_NACK		(~IDR_CHEETAH_ALL_BUSY)
+#define	IDR_CHEETAH_MAX_BN_PAIRS	32
+#define	IDR_JALAPENO_MAX_BN_PAIRS	4
 
 #define	IDC_ITID_SHIFT			14
+#define	IDC_BN_SHIFT			24
 
 #define	IPI_AST		PIL_AST
 #define	IPI_RENDEZVOUS	PIL_RENDEZVOUS
@@ -80,19 +85,19 @@ extern struct pcb stoppcbs[];
 void	cpu_mp_bootstrap(struct pcpu *pc);
 void	cpu_mp_shutdown(void);
 
-void	cpu_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2);
+typedef	void cpu_ipi_selected_t(u_int, u_long, u_long, u_long);
+extern	cpu_ipi_selected_t *cpu_ipi_selected;
 
 void	ipi_selected(u_int cpus, u_int ipi);
 void	ipi_all(u_int ipi);
 void	ipi_all_but_self(u_int ipi);
 
-vm_offset_t mp_tramp_alloc(void);
+void	mp_init(void);
 
 extern	struct mtx ipi_mtx;
 extern	struct ipi_cache_args ipi_cache_args;
 extern	struct ipi_tlb_args ipi_tlb_args;
 
-extern	vm_offset_t mp_tramp;
 extern	char *mp_tramp_code;
 extern	u_long mp_tramp_code_len;
 extern	u_long mp_tramp_tlb_slots;
diff --git a/sys/sparc64/sparc64/machdep.c b/sys/sparc64/sparc64/machdep.c
index 6a589b49ec05..889d676cf387 100644
--- a/sys/sparc64/sparc64/machdep.c
+++ b/sys/sparc64/sparc64/machdep.c
@@ -373,7 +373,7 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec)
 	}
 
 #ifdef SMP
-	mp_tramp = mp_tramp_alloc();
+	mp_init();
 #endif
 
 	/*
diff --git a/sys/sparc64/sparc64/mp_machdep.c b/sys/sparc64/sparc64/mp_machdep.c
index 4c604f4866a4..d005f8ac6328 100644
--- a/sys/sparc64/sparc64/mp_machdep.c
+++ b/sys/sparc64/sparc64/mp_machdep.c
@@ -65,6 +65,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mutex.h>
 #include <sys/pcpu.h>
 #include <sys/proc.h>
+#include <sys/sched.h>
 #include <sys/smp.h>
 
 #include <vm/vm.h>
@@ -87,6 +88,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/tick.h>
 #include <machine/tlb.h>
 #include <machine/tte.h>
+#include <machine/ver.h>
 
 static ih_func_t cpu_ipi_ast;
 static ih_func_t cpu_ipi_stop;
@@ -104,40 +106,59 @@ struct	pcb stoppcbs[MAXCPU];
 
 struct	mtx ipi_mtx;
 
-vm_offset_t mp_tramp;
+cpu_ipi_selected_t *cpu_ipi_selected;
 
-static u_int		cpuid_to_mid[MAXCPU];
-static volatile u_int	shutdown_cpus;
+static vm_offset_t mp_tramp;
+static u_int cpuid_to_mid[MAXCPU];
+static int isjbus;
+static volatile u_int shutdown_cpus;
 
-static void cpu_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2);
 static void cpu_mp_unleash(void *v);
+static void spitfire_ipi_send(u_int, u_long, u_long, u_long);
 static void sun4u_startcpu(phandle_t cpu, void *func, u_long arg);
 static void sun4u_stopself(void);
 
+static cpu_ipi_selected_t cheetah_ipi_selected;
+static cpu_ipi_selected_t spitfire_ipi_selected;
+
 SYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL);
 
-vm_offset_t
-mp_tramp_alloc(void)
+CTASSERT(MAXCPU <= IDR_CHEETAH_MAX_BN_PAIRS);
+CTASSERT(MAXCPU <= sizeof(u_int) * NBBY);
+CTASSERT(MAXCPU <= sizeof(int) * NBBY);
+
+void
+mp_init(void)
 {
 	struct tte *tp;
-	char *v;
 	int i;
 
-	v = OF_claim(NULL, PAGE_SIZE, PAGE_SIZE);
-	if (v == NULL)
+	mp_tramp = (vm_offset_t)OF_claim(NULL, PAGE_SIZE, PAGE_SIZE);
+	if (mp_tramp == (vm_offset_t)-1)
 		panic("%s", __func__);
-	bcopy(mp_tramp_code, v, mp_tramp_code_len);
-	*(u_long *)(v + mp_tramp_tlb_slots) = kernel_tlb_slots;
-	*(u_long *)(v + mp_tramp_func) = (u_long)mp_startup;
-	tp = (struct tte *)(v + mp_tramp_code_len);
+	bcopy(mp_tramp_code, (void *)mp_tramp, mp_tramp_code_len);
+	*(vm_offset_t *)(mp_tramp + mp_tramp_tlb_slots) = kernel_tlb_slots;
+	*(vm_offset_t *)(mp_tramp + mp_tramp_func) = (vm_offset_t)mp_startup;
+	tp = (struct tte *)(mp_tramp + mp_tramp_code_len);
 	for (i = 0; i < kernel_tlb_slots; i++) {
 		tp[i].tte_vpn = TV_VPN(kernel_tlbs[i].te_va, TS_4M);
 		tp[i].tte_data = TD_V | TD_4M | TD_PA(kernel_tlbs[i].te_pa) |
 		    TD_L | TD_CP | TD_CV | TD_P | TD_W;
 	}
-	for (i = 0; i < PAGE_SIZE; i += sizeof(long))
-		flush(v + i);
-	return (vm_offset_t)v;
+	for (i = 0; i < PAGE_SIZE; i += sizeof(vm_offset_t))
+		flush(mp_tramp + i);
+
+	/*
+	 * On UP systems cpu_ipi_selected() can be called while
+	 * cpu_mp_start() wasn't so initialize these here.
+	 */
+	if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi ||
+	    cpu_impl == CPU_IMPL_ULTRASPARCIIIip)
+		isjbus = 1;
+	if (cpu_impl >= CPU_IMPL_ULTRASPARCIII)
+		cpu_ipi_selected = cheetah_ipi_selected;
+	else
+		cpu_ipi_selected = spitfire_ipi_selected;
 }
 
 /*
@@ -148,19 +169,16 @@ cpu_mp_setmaxid(void)
 {
 	char buf[128];
 	phandle_t child;
-	phandle_t root;
 	int cpus;
 
 	all_cpus = 1 << curcpu;
 	mp_ncpus = 1;
 
 	cpus = 0;
-	root = OF_peer(0);
-	for (child = OF_child(root); child != 0; child = OF_peer(child)) {
+	for (child = OF_child(OF_peer(0)); child != 0; child = OF_peer(child))
 		if (OF_getprop(child, "device_type", buf, sizeof(buf)) > 0 &&
 		    strcmp(buf, "cpu") == 0)
 			cpus++;
-	}
 	mp_maxid = cpus - 1;
 }
 
@@ -184,10 +202,6 @@ sun4u_startcpu(phandle_t cpu, void *func, u_long arg)
 	} args = {
 		(cell_t)"SUNW,start-cpu",
 		3,
-		0,
-		0,
-		0,
-		0
 	};
 
 	args.cpu = cpu;
@@ -208,8 +222,6 @@ sun4u_stopself(void)
 		cell_t	nreturns;
 	} args = {
 		(cell_t)"SUNW,stop-self",
-		0,
-		0,
 	};
 
 	openfirmware_exit(&args);
@@ -228,7 +240,6 @@ cpu_mp_start(void)
 	register_t s;
 	vm_offset_t va;
 	phandle_t child;
-	phandle_t root;
 	u_int clock;
 	u_int mid;
 	int cpuid;
@@ -242,14 +253,14 @@ cpu_mp_start(void)
 
 	cpuid_to_mid[curcpu] = PCPU_GET(mid);
 
-	root = OF_peer(0);
 	csa = &cpu_start_args;
-	for (child = OF_child(root); child != 0; child = OF_peer(child)) {
+	for (child = OF_child(OF_peer(0)); child != 0 && mp_ncpus <= MAXCPU;
+	    child = OF_peer(child)) {
 		if (OF_getprop(child, "device_type", buf, sizeof(buf)) <= 0 ||
 		    strcmp(buf, "cpu") != 0)
 			continue;
-		if (OF_getprop(child, "upa-portid", &mid, sizeof(mid)) <= 0 &&
-		    OF_getprop(child, "portid", &mid, sizeof(mid)) <= 0)
+		if (OF_getprop(child, cpu_impl < CPU_IMPL_ULTRASPARCIII ?
+		    "upa-portid" : "portid", &mid, sizeof(mid)) <= 0)
 			panic("%s: can't get module ID", __func__);
 		if (mid == PCPU_GET(mid))
 			continue;
@@ -282,6 +293,9 @@ cpu_mp_start(void)
 
 		all_cpus |= 1 << cpuid;
 	}
+	KASSERT(!isjbus || mp_ncpus <= IDR_JALAPENO_MAX_BN_PAIRS,
+	    ("%s: can only IPI a maximum of %d JBus-CPUs",
+	    __func__, IDR_JALAPENO_MAX_BN_PAIRS));
 	PCPU_SET(other_cpus, all_cpus & ~(1 << curcpu));
 	smp_active = 1;
 }
@@ -413,20 +427,22 @@ cpu_ipi_stop(struct trapframe *tf)
 	CTR2(KTR_SMP, "%s: restarted %d", __func__, curcpu);
 }
 
-void
-cpu_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2)
+static void
+spitfire_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2)
 {
 	u_int cpu;
 
+	KASSERT((cpus & (1 << curcpu)) == 0,
+	    ("%s: CPU can't IPI itself", __func__));
 	while (cpus) {
 		cpu = ffs(cpus) - 1;
 		cpus &= ~(1 << cpu);
-		cpu_ipi_send(cpuid_to_mid[cpu], d0, d1, d2);
+		spitfire_ipi_send(cpuid_to_mid[cpu], d0, d1, d2);
 	}
 }
 
 static void
-cpu_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2)
+spitfire_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2)
 {
 	register_t s;
 	u_long ids;
@@ -439,6 +455,7 @@ cpu_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2)
 		stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0);
 		stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1);
 		stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2);
+		membar(Sync);
 		stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT),
 		    ASI_SDB_INTR_W, 0);
 		/*
@@ -455,7 +472,7 @@ cpu_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2)
 		    IDR_BUSY) != 0)
 			;
 		intr_restore(s);
-		if ((ids & IDR_NACK) == 0)
+		if ((ids & (IDR_BUSY | IDR_NACK)) == 0)
 			return;
 		/*
 		 * Leave interrupts enabled for a bit before retrying
@@ -475,6 +492,74 @@ cpu_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2)
 		panic("%s: couldn't send IPI", __func__);
 }
 
+static void
+cheetah_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2)
+{
+	register_t s;
+	u_long ids;
+	u_int bnp;
+	u_int cpu;
+	int i;
+
+	KASSERT((cpus & (1 << curcpu)) == 0,
+	    ("%s: CPU can't IPI itself", __func__));
+	KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) &
+	    IDR_CHEETAH_ALL_BUSY) == 0,
+	    ("%s: outstanding dispatch", __func__));
+	if (cpus == 0)
+		return;
+	ids = 0;
+	for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) {
+		s = intr_disable();
+		stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0);
+		stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1);
+		stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2);
+		membar(Sync);
+		bnp = 0;
+		for (cpu = 0; cpu < mp_ncpus; cpu++) {
+			if ((cpus & (1 << cpu)) != 0) {
+				stxa(AA_INTR_SEND |
+				    (cpuid_to_mid[cpu] << IDC_ITID_SHIFT) |
+				    (isjbus ? 0 : bnp << IDC_BN_SHIFT),
+				    ASI_SDB_INTR_W, 0);
+				membar(Sync);
+				bnp++;
+			}
+		}
+		while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) &
+		    IDR_CHEETAH_ALL_BUSY) != 0)
+			;
+		intr_restore(s);
+		if ((ids & (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0)
+			return;
+		bnp = 0;
+		for (cpu = 0; cpu < mp_ncpus; cpu++) {
+			if ((cpus & (1 << cpu)) != 0) {
+				if ((ids & (IDR_NACK << (isjbus ?
+				    (2 * cpuid_to_mid[cpu]) :
+				    (2 * bnp)))) == 0)
+					cpus &= ~(1 << cpu);
+				bnp++;
+			}
+		}
+		/*
+		 * Leave interrupts enabled for a bit before retrying
+		 * in order to avoid deadlocks if the other CPUs are
+		 * also trying to send IPIs.
+		 */
+		DELAY(2 * bnp);
+	}
+	if (
+#ifdef KDB
+	    kdb_active ||
+#endif
+	    panicstr != NULL)
+		printf("%s: couldn't send IPI (cpus=0x%u ids=0x%lu)\n",
+		    __func__, cpus, ids);
+	else
+		panic("%s: couldn't send IPI", __func__);
+}
+
 void
 ipi_selected(u_int cpus, u_int ipi)
 {