make MSI-X the default and allocate up to mp_ncpus queues per port

MFC after: 3 days

4 files changed, 86 insertions, 59 deletions

diff --git a/sys/dev/cxgb/cxgb_adapter.h b/sys/dev/cxgb/cxgb_adapter.h
index 33a5e4a83154..6b32460d0295 100644
--- a/sys/dev/cxgb/cxgb_adapter.h
+++ b/sys/dev/cxgb/cxgb_adapter.h
@@ -226,6 +226,7 @@ struct sge_qset {
        	unsigned long           txq_stopped;       /* which Tx queues are stopped */
 	uint64_t                port_stats[SGE_PSTAT_MAX];
 	struct port_info        *port;
+	int                     idx; /* qset # */
 };
 
 struct sge {
@@ -410,7 +411,7 @@ void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p);
 /*
  * XXX figure out how we can return this to being private to sge
  */
-#define desc_reclaimable(q) ((q)->processed - (q)->cleaned - TX_MAX_DESC)
+#define desc_reclaimable(q) ((int)((q)->processed - (q)->cleaned - TX_MAX_DESC))
 
 #define container_of(p, stype, field) ((stype *)(((uint8_t *)(p)) - offsetof(stype, field))) 
 
diff --git a/sys/dev/cxgb/cxgb_main.c b/sys/dev/cxgb/cxgb_main.c
index ab79f68e828e..8074fbe78529 100644
--- a/sys/dev/cxgb/cxgb_main.c
+++ b/sys/dev/cxgb/cxgb_main.c
@@ -176,7 +176,7 @@ DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
  * msi = 1 : only consider MSI and pin interrupts
  * msi = 0: force pin interrupts
  */
-static int msi_allowed = 0;
+static int msi_allowed = 2;
 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed);
 
 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
@@ -303,9 +303,10 @@ cxgb_controller_attach(device_t dev)
 	device_t child;
 	const struct adapter_info *ai;
 	struct adapter *sc;
-	int i, msi_count = 0, error = 0;
+	int i, msi_needed, msi_count = 0, error = 0;
 	uint32_t vers;
-	
+	int port_qsets = 1;
+	    
 	sc = device_get_softc(dev);
 	sc->dev = dev;
 
@@ -336,14 +337,18 @@ cxgb_controller_attach(device_t dev)
 	 * interrupt pin model.
 	 */
 #ifdef MSI_SUPPORTED
+
 	sc->msix_regs_rid = 0x20;
 	if ((msi_allowed >= 2) &&
 	    (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 	    &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
 
-		msi_count = SGE_MSIX_COUNT;
+		msi_needed = msi_count = 5;
+
 		if ((pci_alloc_msix(dev, &msi_count) != 0) ||
-		    (msi_count != SGE_MSIX_COUNT)) {
+		    (msi_count != msi_needed)) {
+			device_printf(dev, "msix allocation failed"
+			    " will try msi\n");
 			msi_count = 0;
 			pci_release_msi(dev);
 			bus_release_resource(dev, SYS_RES_MEMORY,
@@ -353,14 +358,12 @@ cxgb_controller_attach(device_t dev)
 			sc->flags |= USING_MSIX;
 			cxgb_intr = t3_intr_msix;
 		}
-
-		printf("allocated %d msix intrs\n", msi_count);
 	}
 
 	if ((msi_allowed >= 1) && (msi_count == 0)) {
 		msi_count = 1;
 		if (pci_alloc_msi(dev, &msi_count)) {
-			device_printf(dev, "alloc msi failed\n");
+			device_printf(dev, "alloc msi failed - will try INTx\n");
 			msi_count = 0;
 			pci_release_msi(dev);
 		} else {
@@ -371,6 +374,7 @@ cxgb_controller_attach(device_t dev)
 	}
 #endif
 	if (msi_count == 0) {
+		device_printf(dev, "using line interrupts\n");
 		sc->irq_rid = 0;
 		cxgb_intr = t3b_intr;
 	}
@@ -420,11 +424,14 @@ cxgb_controller_attach(device_t dev)
 	}
 	t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
 
+	if (sc->flags & USING_MSIX)
+		port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus);
+
 	/*
 	 * Create a child device for each MAC.  The ethernet attachment
 	 * will be done in these children.
-	 */
-	for (i = 0; i < (sc)->params.nports; ++i) {
+	 */	
+	for (i = 0; i < (sc)->params.nports; i++) {
 		if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
 			device_printf(dev, "failed to add child port\n");
 			error = EINVAL;
@@ -432,12 +439,8 @@ cxgb_controller_attach(device_t dev)
 		}
 		sc->portdev[i] = child;
 		sc->port[i].adapter = sc;
-#ifdef MULTIQ
-		sc->port[i].nqsets = mp_ncpus;
-#else		
-		sc->port[i].nqsets = 1;
-#endif		
-		sc->port[i].first_qset = i;
+		sc->port[i].nqsets = port_qsets;
+		sc->port[i].first_qset = i*port_qsets;
 		sc->port[i].port = i;
 		device_set_softc(child, &sc->port[i]);
 	}
@@ -512,20 +515,20 @@ cxgb_free(struct adapter *sc)
 {
 	int i;
 
-	for (i = 0; i < (sc)->params.nports; ++i) {
-		if (sc->portdev[i] != NULL)
-			device_delete_child(sc->dev, sc->portdev[i]);
-	}
-	
+	callout_drain(&sc->cxgb_tick_ch);
+
 	t3_sge_deinit_sw(sc);
 
 	if (sc->tq != NULL) {
 		taskqueue_drain(sc->tq, &sc->ext_intr_task);
 		taskqueue_free(sc->tq);
 	}
-
-	callout_drain(&sc->cxgb_tick_ch);
 	
+	for (i = 0; i < (sc)->params.nports; ++i) {
+		if (sc->portdev[i] != NULL)
+			device_delete_child(sc->dev, sc->portdev[i]);
+	}
+		
 	bus_generic_detach(sc->dev);
 
 	t3_free_sge_resources(sc);
@@ -589,7 +592,7 @@ setup_sge_qsets(adapter_t *sc)
 	u_int ntxq = 3;
 
 	if ((err = t3_sge_alloc(sc)) != 0) {
-		printf("t3_sge_alloc returned %d\n", err);
+		device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
 		return (err);
 	}
 
@@ -602,12 +605,12 @@ setup_sge_qsets(adapter_t *sc)
 		struct port_info *pi = &sc->port[i];
 
 		for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
-			err = t3_sge_alloc_qset(sc, qset_idx, 1,
+			err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
 			    (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
 			    &sc->params.sge.qset[qset_idx], ntxq, pi);
 			if (err) {
 				t3_free_sge_resources(sc);
-				printf("t3_sge_alloc_qset failed with %d\n", err);
+				device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", err);
 				return (err);
 			}
 		}
@@ -628,6 +631,7 @@ cxgb_setup_msix(adapter_t *sc, int msix_count)
 		device_printf(sc->dev, "Cannot allocate msix interrupt\n");
 		return (EINVAL);
 	}
+
 	if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
 #ifdef INTR_FILTERS
 			NULL,
@@ -636,7 +640,6 @@ cxgb_setup_msix(adapter_t *sc, int msix_count)
 		device_printf(sc->dev, "Cannot set up interrupt\n");
 		return (EINVAL);
 	}
-
 	for (i = 0, k = 0; i < (sc)->params.nports; ++i) {
 		nqsets = sc->port[i].nqsets;
 		for (j = 0; j < nqsets; ++j, k++) {
@@ -665,6 +668,8 @@ cxgb_setup_msix(adapter_t *sc, int msix_count)
 			}
 		}
 	}
+
+
 	return (0);
 }
 
@@ -1109,9 +1114,11 @@ cxgb_init_locked(struct port_info *p)
 	ADAPTER_UNLOCK(p->adapter);
 	t3_intr_enable(sc);
 	t3_port_intr_enable(sc, p->port);
+
 	if ((p->adapter->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX)
 		bind_qsets(sc);
 	p->adapter->flags |= QUEUES_BOUND;
+
 	callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz,
 	    cxgb_tick, sc);
 	
@@ -1125,6 +1132,8 @@ cxgb_set_rxmode(struct port_info *p)
 {
 	struct t3_rx_mode rm;
 	struct cmac *mac = &p->mac;
+
+	mtx_assert(&p->lock, MA_OWNED);
 	
 	t3_init_rx_mode(&rm, p);
 	t3_mac_set_rx_mode(mac, &rm);
@@ -1138,7 +1147,6 @@ cxgb_stop_locked(struct port_info *p)
 	mtx_assert(&p->lock, MA_OWNED);
 	mtx_assert(&p->adapter->lock, MA_NOTOWNED);
 		
-	callout_stop(&p->adapter->cxgb_tick_ch);
 	ifp = p->ifp;
 
 	ADAPTER_LOCK(p->adapter);
@@ -1185,8 +1193,9 @@ cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
 			error = ether_ioctl(ifp, command, data);
 		break;
 	case SIOCSIFFLAGS:
-		PORT_LOCK(p);
+
 		if (ifp->if_flags & IFF_UP) {
+			PORT_LOCK(p);			
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 				flags = p->if_flags;
 				if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
@@ -1195,13 +1204,23 @@ cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
 			
 			} else
 				cxgb_init_locked(p);
+			p->if_flags = ifp->if_flags;
+			PORT_UNLOCK(p);
 		} else {
+			callout_drain(&p->adapter->cxgb_tick_ch);
+			PORT_LOCK(p);
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 				cxgb_stop_locked(p);
+			} else {
+				adapter_t *sc = p->adapter;
+				callout_reset(&sc->cxgb_tick_ch,
+				    sc->params.stats_update_period * hz,
+				    cxgb_tick, sc);
 			}
+			PORT_UNLOCK(p);
 		}
-		p->if_flags = ifp->if_flags;
-		PORT_UNLOCK(p);
+
+
 		break;
 	case SIOCSIFMEDIA:
 	case SIOCGIFMEDIA:
@@ -1370,8 +1389,13 @@ cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 static void
 cxgb_async_intr(void *data)
 {
+	adapter_t *sc = data;
+
 	if (cxgb_debug)
-		printf("cxgb_async_intr\n");
+		device_printf(sc->dev, "cxgb_async_intr\n");
+
+	t3_slow_intr_handler(sc);
+
 }
 
 static void
diff --git a/sys/dev/cxgb/cxgb_osdep.h b/sys/dev/cxgb/cxgb_osdep.h
index 83bb3250f12b..087cf5ac0658 100644
--- a/sys/dev/cxgb/cxgb_osdep.h
+++ b/sys/dev/cxgb/cxgb_osdep.h
@@ -82,6 +82,7 @@ struct t3_mbuf_hdr {
 
 #define CXGB_TX_CLEANUP_THRESHOLD 32
 
+#define LOG_WARNING 1
 
 #ifdef DEBUG_PRINT
 #define DPRINTF printf
diff --git a/sys/dev/cxgb/cxgb_sge.c b/sys/dev/cxgb/cxgb_sge.c
index b6d97eb5c97a..5e9d6d107bd6 100644
--- a/sys/dev/cxgb/cxgb_sge.c
+++ b/sys/dev/cxgb/cxgb_sge.c
@@ -52,6 +52,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/sched.h>
 #include <sys/smp.h>
+#include <sys/systm.h>
 
 #include <netinet/in_systm.h>
 #include <netinet/in.h>
@@ -456,7 +457,7 @@ refill_fl(adapter_t *sc, struct sge_fl *q, int n)
 		m = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, q->buf_size);
 		
 		if (m == NULL) {
-			printf("Failed to allocate mbuf\n");
+			log(LOG_WARNING, "Failed to allocate mbuf\n");
 			goto done;
 		}
 		
@@ -470,7 +471,7 @@ refill_fl(adapter_t *sc, struct sge_fl *q, int n)
 		err = bus_dmamap_load_mbuf_sg(sc->rx_jumbo_dmat, sd->map, m, &seg,
 		    &nsegs, BUS_DMA_NOWAIT);
 		if (err != 0) {
-			printf("failure in refill_fl %d\n", err);
+			log(LOG_WARNING, "failure in refill_fl %d\n", err);
 			m_freem(m);
 			return;
 		}
@@ -581,12 +582,10 @@ static void
 sge_slow_intr_handler(void *arg, int ncount)
 {
 	adapter_t *sc = arg;
-	
+
 	t3_slow_intr_handler(sc);
 }
 
-
-
 static void
 sge_timer_cb(void *arg)
 {
@@ -660,24 +659,24 @@ static void
 sge_timer_reclaim(void *arg, int ncount)
 {
 	adapter_t *sc = arg;
-	int i, j, nqsets = 0;
+	int i, nqsets = 0;
 	struct sge_qset *qs;
 	struct sge_txq *txq;
 	struct mtx *lock;
 	struct mbuf *m_vec[TX_CLEAN_MAX_DESC];
-	int n;
+	int n, reclaimable;
 	/* 
 	 * XXX assuming these quantities are allowed to change during operation
 	 */
-	for (i = 0; i < sc->params.nports; i++) {		
-		for (j = 0; j < sc->port[i].nqsets; j++)
-			nqsets++;
-	}
+	for (i = 0; i < sc->params.nports; i++) 
+		nqsets += sc->port[i].nqsets;
+
 	for (i = 0; i < nqsets; i++) {
 		qs = &sc->sge.qs[i];
 		txq = &qs->txq[TXQ_ETH];
-		if (desc_reclaimable(txq) > 0) {
-			mtx_lock(&txq->lock);
+		reclaimable = desc_reclaimable(txq);
+		if (reclaimable > 0) {
+			mtx_lock(&txq->lock);			
 			n = reclaim_completed_tx(sc, txq, TX_CLEAN_MAX_DESC, m_vec);
 			mtx_unlock(&txq->lock);
 			
@@ -687,7 +686,8 @@ sge_timer_reclaim(void *arg, int ncount)
 		} 
 		    
 		txq = &qs->txq[TXQ_OFLD];
-		if (desc_reclaimable(txq) > 0) {
+		reclaimable = desc_reclaimable(txq);
+		if (reclaimable > 0) {
 			mtx_lock(&txq->lock);
 			n = reclaim_completed_tx(sc, txq, TX_CLEAN_MAX_DESC, m_vec);
 			mtx_unlock(&txq->lock);
@@ -697,17 +697,16 @@ sge_timer_reclaim(void *arg, int ncount)
 			}
 		}
 
-		
 		lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock :
 			    &sc->sge.qs[0].rspq.lock;
 
 		if (mtx_trylock(lock)) {
 			/* XXX currently assume that we are *NOT* polling */
 			uint32_t status = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS);
-			
-			if (qs->fl[0].credits < qs->fl[0].size)
+
+			if (qs->fl[0].credits < qs->fl[0].size - 16)
 				__refill_fl(sc, &qs->fl[0]);
-			if (qs->fl[1].credits < qs->fl[1].size)
+			if (qs->fl[1].credits < qs->fl[1].size - 16)
 				__refill_fl(sc, &qs->fl[1]);
 			
 			if (status & (1 << qs->rspq.cntxt_id)) {
@@ -1850,11 +1849,12 @@ check_ring_db(adapter_t *adap, struct sge_qset *qs,
  * to work around lack of ithread affinity
  */
 static void
-bind_ithread(void)
+bind_ithread(int cpu)
 {
+	KASSERT(cpu < mp_ncpus, ("invalid cpu identifier"));
 	if (mp_ncpus > 1) {
 		mtx_lock_spin(&sched_lock);
-		sched_bind(curthread, 1);
+		sched_bind(curthread, cpu);
 		mtx_unlock_spin(&sched_lock);
 	}
 
@@ -1884,7 +1884,7 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
 	unsigned int sleeping = 0;
 	int lro = qs->lro.enabled;
 		
-	static int pinned = 0;
+	static uint8_t pinned[MAXCPU];
 
 #ifdef DEBUG	
 	static int last_holdoff = 0;
@@ -1893,9 +1893,12 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
 		last_holdoff = rspq->holdoff_tmr;
 	}
 #endif	
-	if (pinned == 0) {
-		bind_ithread();
-		pinned = 1;
+	if (pinned[qs->rspq.cntxt_id * adap->params.nports] == 0) {
+		/*
+		 * Assumes that cntxt_id < mp_ncpus
+		 */
+		bind_ithread(qs->rspq.cntxt_id);
+		pinned[qs->rspq.cntxt_id * adap->params.nports] = 1;
 	}
 	rspq->next_holdoff = rspq->holdoff_tmr;
 
@@ -2088,8 +2091,6 @@ t3_intr_msix(void *data)
 	adapter_t *adap = qs->port->adapter;
 	struct sge_rspq *rspq = &qs->rspq;
 
-	if (cxgb_debug)
-		printf("got msi-x interrupt\n");
 	mtx_lock(&rspq->lock);
 	if (process_responses_gts(adap, rspq) == 0) {
 #ifdef notyet