- move WR_LEN in to cxgb_adapter.h add PIO_LEN to make intent clearer

- move cxgb_tx_common in to cxgb_multiq.c and rename to cxgb_tx
- move cxgb_tx_common dependencies
- further simplify cxgb_dequeue_packet for the non-multiqueue case
- only launch one service thread per port in the non-multiq case
- remove dead cleaning code from cxgb_sge.c
- simplify PIO case substantially in by returning directly from mbuf collapse
  and just using m_copydata
- remove gratuitous m_gethdr in the rx path
- clarify freeing of mbufs in collapse

6 files changed, 191 insertions, 249 deletions

diff --git a/sys/dev/cxgb/cxgb_adapter.h b/sys/dev/cxgb/cxgb_adapter.h
index df8aa2b1831b..6910ba95a476 100644
--- a/sys/dev/cxgb/cxgb_adapter.h
+++ b/sys/dev/cxgb/cxgb_adapter.h
@@ -149,13 +149,24 @@ enum {				/* adapter flags */
 #define FL_Q_SIZE	4096
 #define JUMBO_Q_SIZE	1024
 #define RSPQ_Q_SIZE	1024
+#if 0
 #define TX_ETH_Q_SIZE	1024
+#else
+#define TX_ETH_Q_SIZE	64
+#endif
 
 enum { TXQ_ETH = 0,
        TXQ_OFLD = 1,
        TXQ_CTRL = 2, };
 
 
+/* 
+ * work request size in bytes
+ */
+#define WR_LEN (WR_FLITS * 8)
+#define PIO_LEN (WR_LEN - sizeof(struct cpl_tx_pkt))
+
+
 /* careful, the following are set on priv_flags and must not collide with
  * IFF_ flags!
  */
@@ -288,7 +299,6 @@ struct sge_txq {
 	struct mtx      lock;
 	struct sg_ent  txq_sgl[TX_MAX_SEGS / 2 + 1];
 	bus_dma_segment_t txq_segs[TX_MAX_SEGS];
-	struct mbuf     *txq_m_vec[TX_WR_COUNT_MAX];
 #define TXQ_NAME_LEN  32
 	char            lockbuf[TXQ_NAME_LEN];
 };
@@ -610,10 +620,7 @@ void cxgb_pcpu_shutdown_threads(struct adapter *sc);
 void cxgb_pcpu_startup_threads(struct adapter *sc);
 
 int process_responses(adapter_t *adap, struct sge_qset *qs, int budget);
-int cxgb_tx_common(struct ifnet *ifp, struct sge_qset  *qs, uint32_t txmax);
 void t3_free_qset(adapter_t *sc, struct sge_qset *q);
-int cxgb_dequeue_packet(struct ifnet *, struct sge_txq *, struct mbuf **);
 void cxgb_start(struct ifnet *ifp);
 void refill_fl_service(adapter_t *adap, struct sge_fl *fl);
-int reclaim_completed_tx(struct sge_txq *q, int reclaim_min);
 #endif
diff --git a/sys/dev/cxgb/cxgb_main.c b/sys/dev/cxgb/cxgb_main.c
index c808bcae69b1..e5057585a85e 100644
--- a/sys/dev/cxgb/cxgb_main.c
+++ b/sys/dev/cxgb/cxgb_main.c
@@ -281,21 +281,6 @@ struct cxgb_ident {
 
 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
 
-static __inline void
-check_pkt_coalesce(struct sge_qset *qs)
-{
-	struct adapter *sc;
-	struct sge_txq *txq;
-
-	txq = &qs->txq[TXQ_ETH];
-	sc = qs->port->adapter;
-
-	if (sc->tunq_fill[qs->idx] && (txq->in_use < (txq->size - (txq->size>>2)))) 
-		sc->tunq_fill[qs->idx] = 0;
-	else if (!sc->tunq_fill[qs->idx] && (txq->in_use > (txq->size - (txq->size>>2)))) 
-		sc->tunq_fill[qs->idx] = 1;
-}
-
 static __inline char
 t3rev2char(struct adapter *adapter)
 {
@@ -1865,64 +1850,6 @@ cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
 	return (error);
 }
 
-int
-cxgb_tx_common(struct ifnet *ifp, struct sge_qset *qs, uint32_t txmax)
-{
-	struct sge_txq *txq;
-	int err, in_use_init, count;
-	struct mbuf **m_vec;
-
-	txq = &qs->txq[TXQ_ETH];
-	m_vec = txq->txq_m_vec;
-	in_use_init = txq->in_use;
-	err = 0;
-	while ((txq->in_use - in_use_init < txmax) &&
-	    (txq->size > txq->in_use + TX_MAX_DESC)) {
-		check_pkt_coalesce(qs);
-		count = cxgb_dequeue_packet(ifp, txq, m_vec);
-		if (count == 0)
-			break;
-		ETHER_BPF_MTAP(ifp, m_vec[0]);
-		
-		if ((err = t3_encap(qs, m_vec, count)) != 0)
-			break;
-		txq->txq_enqueued += count;
-	}
-#if 0 /* !MULTIQ */
-	if (__predict_false(err)) {
-		if (err == ENOMEM) {
-			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
-			IFQ_LOCK(&ifp->if_snd);
-			IFQ_DRV_PREPEND(&ifp->if_snd, m_vec[0]);
-			IFQ_UNLOCK(&ifp->if_snd);
-		}
-	}
-	if (err == 0 && m_vec[0] == NULL) {
-		err = ENOBUFS;
-	}
-	else if ((err == 0) &&  (txq->size <= txq->in_use + TX_MAX_DESC) &&
-	    (ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
-		setbit(&qs->txq_stopped, TXQ_ETH);
-		ifp->if_drv_flags |= IFF_DRV_OACTIVE;
-		err = ENOSPC;
-	}
-#else
-	if ((err == 0) &&  (txq->size <= txq->in_use + TX_MAX_DESC)) {
-		err = ENOSPC;
-		setbit(&qs->txq_stopped, TXQ_ETH);
-	}
-	if (err == ENOMEM) {
-		int i;
-		/*
-		 * Sub-optimal :-/
-		 */
-		for (i = 0; i < count; i++)
-			m_freem(m_vec[i]);
-	}
-#endif
-	return (err);
-}
-
 static int
 cxgb_media_change(struct ifnet *ifp)
 {
diff --git a/sys/dev/cxgb/cxgb_multiq.c b/sys/dev/cxgb/cxgb_multiq.c
index fc180aa8e755..537b1db8681d 100644
--- a/sys/dev/cxgb/cxgb_multiq.c
+++ b/sys/dev/cxgb/cxgb_multiq.c
@@ -115,6 +115,9 @@ static void cxgb_pcpu_start_proc(void *arg);
 #ifdef IFNET_MULTIQUEUE
 static int cxgb_pcpu_cookie_to_qidx(struct port_info *, uint32_t cookie);
 #endif
+static int cxgb_tx(struct sge_qset *qs, uint32_t txmax);
+
+
 static inline int
 cxgb_pcpu_enqueue_packet_(struct sge_qset *qs, struct mbuf *m)
 {
@@ -124,7 +127,6 @@ cxgb_pcpu_enqueue_packet_(struct sge_qset *qs, struct mbuf *m)
 #ifndef IFNET_MULTIQUEUE
 	panic("not expecting enqueue without multiqueue");
 #endif	
-	
 	KASSERT(m != NULL, ("null mbuf"));
 	KASSERT(m->m_type == MT_DATA, ("bad mbuf type %d", m->m_type));
 	if (qs->qs_flags & QS_EXITING) {
@@ -164,8 +166,8 @@ cxgb_pcpu_enqueue_packet(struct ifnet *ifp, struct mbuf *m)
 	return (err);
 }
 
-int
-cxgb_dequeue_packet(struct ifnet *unused, struct sge_txq *txq, struct mbuf **m_vec)
+static int
+cxgb_dequeue_packet(struct sge_txq *txq, struct mbuf **m_vec)
 {
 	struct mbuf *m;
 	struct sge_qset *qs;
@@ -176,10 +178,16 @@ cxgb_dequeue_packet(struct ifnet *unused, struct sge_txq *txq, struct mbuf **m_v
 
 	if (txq->immpkt != NULL)
 		panic("immediate packet set");
-#endif
-
 	mtx_assert(&txq->lock, MA_OWNED);
 
+	IFQ_DRV_DEQUEUE(&pi->ifp->if_snd, m);
+	if (m == NULL)
+		return (0);
+	
+	m_vec[0] = m;
+	return (1);
+#endif
+
 	coalesced = count = size = 0;
 	qs = txq_to_qset(txq, TXQ_ETH);
 	if (qs->qs_flags & QS_EXITING)
@@ -193,39 +201,20 @@ cxgb_dequeue_packet(struct ifnet *unused, struct sge_txq *txq, struct mbuf **m_v
 	}
 	sc = qs->port->adapter;
 
-#ifndef IFNET_MULTIQUEUE
-	/*
-	 * This is terrible from a cache and locking efficiency standpoint
-	 * but then again ... so is ifnet.
-	 */
-	while (((qs->qs_flags & QS_EXITING) == 0) && !IFQ_DRV_IS_EMPTY(&pi->ifp->if_snd) && !buf_ring_full(&txq->txq_mr)) {
-		
-		struct mbuf *m = NULL;
-
-		IFQ_DRV_DEQUEUE(&pi->ifp->if_snd, m);
-		if (m) {
-			KASSERT(m->m_type == MT_DATA, ("bad mbuf type %d", m->m_type));
-			if (buf_ring_enqueue(&txq->txq_mr, m))
-				panic("ring full");
-		} else
-			break;
-	}
-#endif    
 	m = buf_ring_dequeue(&txq->txq_mr);
-	if (m == NULL)
+	if (m == NULL) 
 		return (0);
 
-	buf_ring_scan(&txq->txq_mr, m, __FILE__, __LINE__);
-	KASSERT(m->m_type == MT_DATA, ("bad mbuf type %d", m->m_type));
+	count = 1;
+	KASSERT(m->m_type == MT_DATA,
+	    ("m=%p is bad mbuf type %d from ring cons=%d prod=%d", m,
+		m->m_type, txq->txq_mr.br_cons, txq->txq_mr.br_prod));
 	m_vec[0] = m;
-	if (m->m_pkthdr.tso_segsz > 0 || m->m_pkthdr.len > TX_WR_SIZE_MAX || m->m_next != NULL ||
-	    (cxgb_pcpu_tx_coalesce == 0)) {
-		return (1);
+	if (m->m_pkthdr.tso_segsz > 0 || m->m_pkthdr.len > TX_WR_SIZE_MAX ||
+	    m->m_next != NULL || (cxgb_pcpu_tx_coalesce == 0)) {
+		return (count);
 	}
-#ifndef IFNET_MULTIQUEUE
-	panic("coalesce not supported yet");
-#endif	
-	count = 1;
+
 	size = m->m_pkthdr.len;
 	for (m = buf_ring_peek(&txq->txq_mr); m != NULL;
 	     m = buf_ring_peek(&txq->txq_mr)) {
@@ -381,13 +370,15 @@ cxgb_pcpu_free(struct sge_qset *qs)
 {
 	struct mbuf *m;
 	struct sge_txq *txq = &qs->txq[TXQ_ETH];
-	
+
+	mtx_lock(&txq->lock);
 	while ((m = mbufq_dequeue(&txq->sendq)) != NULL) 
 		m_freem(m);
 	while ((m = buf_ring_dequeue(&txq->txq_mr)) != NULL) 
 		m_freem(m);
 
 	t3_free_tx_desc_all(txq);
+	mtx_unlock(&txq->lock);
 }
 
 static int
@@ -400,6 +391,7 @@ cxgb_pcpu_reclaim_tx(struct sge_txq *txq)
 	KASSERT(qs->qs_cpuid == curcpu, ("cpu qset mismatch cpuid=%d curcpu=%d",
 			qs->qs_cpuid, curcpu));
 #endif
+	mtx_assert(&txq->lock, MA_OWNED);
 	
 	reclaimable = desc_reclaimable(txq);
 	if (reclaimable == 0)
@@ -429,6 +421,8 @@ cxgb_pcpu_start_(struct sge_qset *qs, struct mbuf *immpkt, int tx_flush)
 	sc = pi->adapter;
 	txq = &qs->txq[TXQ_ETH];
 	
+	mtx_assert(&txq->lock, MA_OWNED);
+	
  retry:	
 	if (!pi->link_config.link_ok)
 		initerr = ENXIO;
@@ -474,10 +468,12 @@ cxgb_pcpu_start_(struct sge_qset *qs, struct mbuf *immpkt, int tx_flush)
 		DPRINTF("stopped=%d flush=%d max_desc=%d\n",
 		    stopped, flush, max_desc);
 	
-	err = flush ? cxgb_tx_common(qs->port->ifp, qs, max_desc) : ENOSPC;
+	err = flush ? cxgb_tx(qs, max_desc) : ENOSPC;
 
 
-	if ((tx_flush && flush && err == 0) && !buf_ring_empty(&txq->txq_mr)) {
+	if ((tx_flush && flush && err == 0) &&
+	    (!buf_ring_empty(&txq->txq_mr)  ||
+		!IFQ_DRV_IS_EMPTY(&pi->ifp->if_snd))) {
 		struct thread *td = curthread;
 
 		if (++i > 1) {
@@ -525,7 +521,8 @@ cxgb_pcpu_start(struct ifnet *ifp, struct mbuf *immpkt)
 	
 	txq = &qs->txq[TXQ_ETH];
 
-	if (((sc->tunq_coalesce == 0) || (buf_ring_count(&txq->txq_mr) >= TX_WR_COUNT_MAX) ||
+	if (((sc->tunq_coalesce == 0) ||
+		(buf_ring_count(&txq->txq_mr) >= TX_WR_COUNT_MAX) ||
 		(cxgb_pcpu_tx_coalesce == 0)) && mtx_trylock(&txq->lock)) {
 		if (cxgb_debug)
 			printf("doing immediate transmit\n");
@@ -658,12 +655,18 @@ cxgb_pcpu_cookie_to_qidx(struct port_info *pi, uint32_t cookie)
 void
 cxgb_pcpu_startup_threads(struct adapter *sc)
 {
-	int i, j;
+	int i, j, nqsets;
 	struct proc *p;
-	
+
+
 	for (i = 0; i < (sc)->params.nports; ++i) {
 			struct port_info *pi = adap2pinfo(sc, i);
-		
+
+#ifdef IFNET_MULTIQUEUE
+	nqsets = pi->nqsets;
+#else
+	nqsets = 1;
+#endif	
 		for (j = 0; j < pi->nqsets; ++j) {
 			struct sge_qset *qs;
 
@@ -701,3 +704,84 @@ cxgb_pcpu_shutdown_threads(struct adapter *sc)
 		}
 	}
 }
+
+static __inline void
+check_pkt_coalesce(struct sge_qset *qs)
+{
+	struct adapter *sc;
+	struct sge_txq *txq;
+
+	txq = &qs->txq[TXQ_ETH];
+	sc = qs->port->adapter;
+
+	if (sc->tunq_fill[qs->idx] && (txq->in_use < (txq->size - (txq->size>>2)))) 
+		sc->tunq_fill[qs->idx] = 0;
+	else if (!sc->tunq_fill[qs->idx] && (txq->in_use > (txq->size - (txq->size>>2)))) 
+		sc->tunq_fill[qs->idx] = 1;
+}
+
+static int
+cxgb_tx(struct sge_qset *qs, uint32_t txmax)
+{
+	struct sge_txq *txq;
+	struct ifnet *ifp = qs->port->ifp;
+	int i, err, in_use_init, count;
+	struct mbuf *m_vec[TX_WR_COUNT_MAX];
+
+	txq = &qs->txq[TXQ_ETH];
+	ifp = qs->port->ifp;
+	in_use_init = txq->in_use;
+	err = 0;
+	
+	for (i = 0; i < TX_WR_COUNT_MAX; i++)
+		m_vec[i] = NULL;
+
+	mtx_assert(&txq->lock, MA_OWNED);
+	while ((txq->in_use - in_use_init < txmax) &&
+	    (txq->size > txq->in_use + TX_MAX_DESC)) {
+		check_pkt_coalesce(qs);
+		count = cxgb_dequeue_packet(txq, m_vec);
+		if (count == 0) {
+			err = ENOBUFS;
+			break;
+		}
+		ETHER_BPF_MTAP(ifp, m_vec[0]);
+		
+		if ((err = t3_encap(qs, m_vec, count)) != 0)
+			break;
+		txq->txq_enqueued += count;
+		m_vec[0] = NULL;
+	}
+#if 0 /* !MULTIQ */
+	if (__predict_false(err)) {
+		if (err == ENOMEM) {
+			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
+			IFQ_LOCK(&ifp->if_snd);
+			IFQ_DRV_PREPEND(&ifp->if_snd, m_vec[0]);
+			IFQ_UNLOCK(&ifp->if_snd);
+		}
+	}
+	else if ((err == 0) &&  (txq->size <= txq->in_use + TX_MAX_DESC) &&
+	    (ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
+		setbit(&qs->txq_stopped, TXQ_ETH);
+		ifp->if_drv_flags |= IFF_DRV_OACTIVE;
+		err = ENOSPC;
+	}
+#else
+	if ((err == 0) &&  (txq->size <= txq->in_use + TX_MAX_DESC)) {
+		err = ENOSPC;
+		setbit(&qs->txq_stopped, TXQ_ETH);
+	}
+	if (err == ENOMEM) {
+		int i;
+		/*
+		 * Sub-optimal :-/
+		 */
+		printf("ENOMEM!!!");
+		for (i = 0; i < count; i++)
+			m_freem(m_vec[i]);
+	}
+#endif
+	return (err);
+}
+
diff --git a/sys/dev/cxgb/cxgb_sge.c b/sys/dev/cxgb/cxgb_sge.c
index b1e56aaa0e70..c854d9d15837 100644
--- a/sys/dev/cxgb/cxgb_sge.c
+++ b/sys/dev/cxgb/cxgb_sge.c
@@ -26,6 +26,8 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGE.
 
 ***************************************************************************/
+#define DEBUG_BUFRING
+
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
@@ -72,7 +74,7 @@ __FBSDID("$FreeBSD$");
 
 int      txq_fills = 0;
 static int bogus_imm = 0;
-static int recycle_enable = 1;
+static int recycle_enable = 0;
 extern int cxgb_txq_buf_ring_size;
 int cxgb_cached_allocations;
 int cxgb_cached;
@@ -92,10 +94,6 @@ extern int cxgb_use_16k_clusters;
  */
 #define TX_RECLAIM_PERIOD       (hz >> 1)
 
-/* 
- * work request size in bytes
- */
-#define WR_LEN (WR_FLITS * 8)
 
 /* 
  * Values for sge_txq.flags
@@ -218,13 +216,6 @@ reclaim_completed_tx_(struct sge_txq *q, int reclaim_min)
 	return (reclaim);
 }
 
-int
-reclaim_completed_tx(struct sge_txq *q, int reclaim_min)
-{
-	return reclaim_completed_tx_(q, reclaim_min);
-}
-
-	
 /**
  *	should_restart_tx - are there enough resources to restart a Tx queue?
  *	@q: the Tx queue
@@ -672,8 +663,7 @@ alloc_ring(adapter_t *sc, size_t nelem, size_t elem_size, size_t sw_size,
 
 	if (sw_size) {
 		len = nelem * sw_size;
-		s = malloc(len, M_DEVBUF, M_WAITOK);
-		bzero(s, len);
+		s = malloc(len, M_DEVBUF, M_WAITOK|M_ZERO);
 		*(void **)sdesc = s;
 	}
 	if (parent_entry_tag == NULL)
@@ -736,17 +726,16 @@ sge_timer_cb(void *arg)
 	struct sge_qset *qs;
 	struct sge_txq  *txq;
 	int i, j;
-	int reclaim_eth, reclaim_ofl, refill_rx;
+	int reclaim_ofl, refill_rx;
 
 	for (i = 0; i < sc->params.nports; i++) 
 		for (j = 0; j < sc->port[i].nqsets; j++) {
 			qs = &sc->sge.qs[i + j];
 			txq = &qs->txq[0];
-			reclaim_eth = txq[TXQ_ETH].processed - txq[TXQ_ETH].cleaned;
 			reclaim_ofl = txq[TXQ_OFLD].processed - txq[TXQ_OFLD].cleaned;
 			refill_rx = ((qs->fl[0].credits < qs->fl[0].size) || 
 			    (qs->fl[1].credits < qs->fl[1].size));
-			if (reclaim_eth || reclaim_ofl || refill_rx) {
+			if (reclaim_ofl || refill_rx) {
 				pi = &sc->port[i];
 				taskqueue_enqueue(pi->tq, &pi->timer_reclaim_task);
 				break;
@@ -834,31 +823,14 @@ refill_rspq(adapter_t *sc, const struct sge_rspq *q, u_int credits)
 static __inline void
 sge_txq_reclaim_(struct sge_txq *txq, int force)
 {
-	int reclaimable, n;
-	struct port_info *pi;
 
-	pi = txq->port;
-reclaim_more:
-	n = 0;
-	if ((reclaimable = desc_reclaimable(txq)) < 16)
+	if (desc_reclaimable(txq) < 16)
 		return;
 	if (mtx_trylock(&txq->lock) == 0) 
 		return;
-	n = reclaim_completed_tx_(txq, 16);
+	reclaim_completed_tx_(txq, 16);
 	mtx_unlock(&txq->lock);
 
-	if (pi && pi->ifp->if_drv_flags & IFF_DRV_OACTIVE &&
-	    txq->size - txq->in_use >= TX_START_MAX_DESC) {
-		struct sge_qset *qs = txq_to_qset(txq, TXQ_ETH);
-
-		txq_fills++;
-		pi->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
-		clrbit(&qs->txq_stopped, TXQ_ETH);		
-		taskqueue_enqueue(pi->tq, &pi->start_task);
-	}
-
-	if (n)
-		goto reclaim_more;
 }
 
 static void
@@ -886,8 +858,6 @@ sge_timer_reclaim(void *arg, int ncount)
 #endif 
 	for (i = 0; i < nqsets; i++) {
 		qs = &sc->sge.qs[i];
-		txq = &qs->txq[TXQ_ETH];
-		sge_txq_reclaim_(txq, FALSE);
 
 		txq = &qs->txq[TXQ_OFLD];
 		sge_txq_reclaim_(txq, FALSE);
@@ -1184,7 +1154,7 @@ write_wr_hdr_sgl(unsigned int ndesc, struct tx_desc *txd, struct txq_state *txqs
 			 * is freed all clusters will be freed
 			 * with it
 			 */
-			txsd->mi.mi_base = NULL;
+			KASSERT(txsd->mi.mi_base == NULL, ("overwrting valid entry mi_base==%p", txsd->mi.mi_base));
 			wrp = (struct work_request_hdr *)txd;
 			wrp->wr_hi = htonl(V_WR_DATATYPE(1) |
 			    V_WR_SGLSFLT(1)) | wr_hi;
@@ -1244,6 +1214,7 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
 		
 	DPRINTF("t3_encap cpu=%d ", curcpu);
 
+	mi = NULL;
 	pi = qs->port;
 	sc = pi->adapter;
 	txq = &qs->txq[TXQ_ETH];
@@ -1252,9 +1223,13 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
 	sgl = txq->txq_sgl;
 	segs = txq->txq_segs;
 	m0 = *m;
+	
 	DPRINTF("t3_encap port_id=%d qsidx=%d ", pi->port_id, pi->first_qset);
 	DPRINTF("mlen=%d txpkt_intf=%d tx_chan=%d\n", m[0]->m_pkthdr.len, pi->txpkt_intf, pi->tx_chan);
-
+	if (cxgb_debug)
+		printf("mi_base=%p cidx=%d pidx=%d\n\n", txsd->mi.mi_base, txq->cidx, txq->pidx);
+	
+	mtx_assert(&txq->lock, MA_OWNED);
 	cntrl = V_TXPKT_INTF(pi->txpkt_intf);
 /*
  * XXX need to add VLAN support for 6.x
@@ -1263,7 +1238,9 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
 	if  (m0->m_pkthdr.csum_flags & (CSUM_TSO))
 		tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz);
 #endif
-	
+	KASSERT(txsd->mi.mi_base == NULL, ("overwrting valid entry mi_base==%p",
+		txsd->mi.mi_base));
+
 	if (count > 1) {
 		panic("count > 1 not support in CVS\n");
 		if ((err = busdma_map_sg_vec(m, &m0, segs, count)))
@@ -1276,13 +1253,10 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
 	} 
 	KASSERT(m0->m_pkthdr.len, ("empty packet nsegs=%d count=%d", nsegs, count));
 
-	if (m0->m_type == MT_DATA)
-		DPRINTF("mbuf type=%d tags:%d head=%p", m0->m_type, !SLIST_EMPTY(&m0->m_pkthdr.tags),
-		    SLIST_FIRST(&m0->m_pkthdr.tags));
-	 
-	mi_collapse_mbuf(&txsd->mi, m0);
-	mi = &txsd->mi;
-
+	if (m0->m_pkthdr.len > PIO_LEN) {
+		mi_collapse_mbuf(&txsd->mi, m0);
+		mi = &txsd->mi;
+	}
 	if (count > 1) {
 		struct cpl_tx_pkt_batch *cpl_batch = (struct cpl_tx_pkt_batch *)txd;
 		int i, fidx;
@@ -1373,25 +1347,11 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
 		mlen = m0->m_pkthdr.len;
 		cpl->len = htonl(mlen | 0x80000000);
 
-		if (mlen <= WR_LEN - sizeof(*cpl)) {
+		if (mlen <= PIO_LEN) {
 			txq_prod(txq, 1, &txqs);
-			
-			DPRINTF("mlen==%d max=%ld\n", mlen, (WR_LEN - sizeof(*cpl)));
-			if (mi->mi_type != MT_IOVEC &&
-			    mi->mi_type != MT_CLIOVEC) 
-				memcpy(&txd->flit[2], mi->mi_data, mlen);
-			else {
-				/*
-				 * XXX mbuf_iovec
-				 */
-#if 0
-				m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]);
-#endif
-				printf("bailing on m_copydata\n");
-			}
-			m_freem_iovec(&txsd->mi);
-			txsd->mi.mi_base = NULL;
-
+			m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]);
+			m_freem(m0);
+			m0 = NULL;
 			flits = (mlen + 7) / 8 + 2;
 			cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) |
 					  V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) |
@@ -1797,13 +1757,18 @@ t3_sge_stop(adapter_t *sc)
 	
 	for (nqsets = i = 0; i < (sc)->params.nports; i++) 
 		nqsets += sc->port[i].nqsets;
-	
+#ifdef notyet
+	/*
+	 * 
+	 * XXX
+	 */
 	for (i = 0; i < nqsets; ++i) {
 		struct sge_qset *qs = &sc->sge.qs[i];
 		
 		taskqueue_drain(sc->tq, &qs->txq[TXQ_OFLD].qresume_task);
 		taskqueue_drain(sc->tq, &qs->txq[TXQ_CTRL].qresume_task);
 	}
+#endif
 }
 
 /**
@@ -1839,7 +1804,8 @@ t3_free_tx_desc(struct sge_txq *q, int reclaimable)
 				bus_dmamap_unload(q->entry_tag, txsd->map);
 				txsd->flags &= ~TX_SW_DESC_MAPPED;
 			}
-			m_freem_iovec(&txsd->mi);
+			m_freem_iovec(&txsd->mi);	
+			buf_ring_scan(&q->txq_mr, txsd->mi.mi_base, __FILE__, __LINE__);
 			txsd->mi.mi_base = NULL;
 
 #if defined(DIAGNOSTIC) && 0
@@ -2513,7 +2479,7 @@ init_cluster_mbuf(caddr_t cl, int flags, int type, uma_zone_t zone)
 
 static int
 get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
-    struct t3_mbuf_hdr *mh, struct rsp_desc *r, struct mbuf *m)
+    struct t3_mbuf_hdr *mh, struct rsp_desc *r)
 {
 
 	unsigned int len_cq =  ntohl(r->len_cq);
@@ -2522,6 +2488,7 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
 	uint32_t len = G_RSPD_LEN(len_cq);
 	uint32_t flags = ntohl(r->flags);
 	uint8_t sopeop = G_RSPD_SOP_EOP(flags);
+	struct mbuf *m;
 	uint32_t *ref;
 	int ret = 0;
 
@@ -2536,13 +2503,13 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
 		cl = mtod(m0, void *);
 		memcpy(cl, sd->data, len);
 		recycle_rx_buf(adap, fl, fl->cidx);
-		*m = m0;
+		m = m0;
 	} else {
 	skip_recycle:
 		int flags = 0;
 		bus_dmamap_unload(fl->entry_tag, sd->map);
 		cl = sd->rxsd_cl;
-		*m = m0 = (struct mbuf *)cl;
+		m = m0 = (struct mbuf *)cl;
 
 		m0->m_len = len;
 		if ((sopeop == RSPQ_SOP_EOP) ||
@@ -2561,8 +2528,7 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
 	case RSPQ_NSOP_NEOP:
 		DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m));
 		if (mh->mh_tail == NULL) {
-			if (cxgb_debug)
-				printf("discarding intermediate descriptor entry\n");
+			printf("discarding intermediate descriptor entry\n");
 			m_freem(m);
 			break;
 		}
@@ -2798,18 +2764,7 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
 			int drop_thresh = eth ? SGE_RX_DROP_THRES : 0;
 			
 #ifdef DISABLE_MBUF_IOVEC
-			struct mbuf *m;
-			m = m_gethdr(M_DONTWAIT, MT_DATA);
-
-			if (m == NULL) {
-				log(LOG_WARNING, "failed to get mbuf for packet\n");
-				budget_left--;
-				break;	
-			} else {
-				m->m_next = m->m_nextpkt = NULL;
-			}
-
-			eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r, m);
+			eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r);
 #else
 			eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mbuf, r);
 #ifdef IFNET_MULTIQUEUE
diff --git a/sys/dev/cxgb/sys/mvec.h b/sys/dev/cxgb/sys/mvec.h
index f873f567cd5a..57a6478cd530 100644
--- a/sys/dev/cxgb/sys/mvec.h
+++ b/sys/dev/cxgb/sys/mvec.h
@@ -232,13 +232,15 @@ m_freem_iovec(struct mbuf_iovec *mi)
 	struct mbuf *m;
 
 	switch (mi->mi_type) {
+	case EXT_MBUF:
+		m_free_fast((struct mbuf *)mi->mi_base);
+		break;
 	case EXT_IOVEC:
 	case EXT_CLIOVEC:
 	case EXT_JMPIOVEC:
 		m = (struct mbuf *)mi->mi_base;
 		m_free_iovec(m, mi->mi_type);
 		break;
-	case EXT_MBUF:
 	case EXT_CLUSTER:
 	case EXT_JUMBOP:
 	case EXT_JUMBO9:
diff --git a/sys/dev/cxgb/sys/uipc_mvec.c b/sys/dev/cxgb/sys/uipc_mvec.c
index 3f6ecf631346..9dc902b69663 100644
--- a/sys/dev/cxgb/sys/uipc_mvec.c
+++ b/sys/dev/cxgb/sys/uipc_mvec.c
@@ -162,7 +162,7 @@ _mcl_collapse_mbuf(struct mbuf_iovec *mi, struct mbuf *m)
 	}
 	KASSERT(mi->mi_len != 0, ("miov has len 0"));
 	KASSERT(mi->mi_type > 0, ("mi_type is invalid"));
-
+	KASSERT(mi->mi_base, ("mi_base is invalid"));
 	return (n);
 }
 
@@ -204,40 +204,15 @@ busdma_map_sg_collapse(struct mbuf **m, bus_dma_segment_t *segs, int *nsegs)
 	if (n->m_flags & M_PKTHDR && !SLIST_EMPTY(&n->m_pkthdr.tags)) 
 		m_tag_delete_chain(n, NULL);
 
+	if (n->m_pkthdr.len <= PIO_LEN)
+		return (0);
 retry:
 	seg_count = 0;
 	if (n->m_next == NULL) {
 		busdma_map_mbuf_fast(n, segs);
 		*nsegs = 1;
-
 		return (0);
 	}
-
-	if (n->m_pkthdr.len <= 104) {
-		caddr_t data;
-
-		if ((m0 = m_gethdr(M_DONTWAIT, MT_DATA)) == NULL) 
-			return (ENOMEM);
-
-		data = m0->m_data;
-		memcpy(m0, n, sizeof(struct m_hdr) + sizeof(struct pkthdr));
-		m0->m_data = data;
-		m0->m_len = n->m_pkthdr.len;
-		m0->m_flags &= ~M_EXT;
-		m0->m_next = NULL;
-		m0->m_type = n->m_type;
-		n->m_flags &= ~M_PKTHDR;
-		while (n) {
-			memcpy(data, n->m_data, n->m_len);
-			data += n->m_len;
-			n = n->m_next;
-		}
-		m_freem(*m);
-		n = m0;
-		*m = n;
-		goto retry;
-	}
-	
 	while (n && seg_count < TX_MAX_SEGS) {
 		marray[seg_count] = n;
 
@@ -249,18 +224,6 @@ retry:
 
 		n = n->m_next;
 	}
-#if 0
-	/*
-	 * XXX needs more careful consideration
-	 */
-	if (__predict_false(seg_count == 1)) {
-		n = marray[0];
-		if (n != *m)
-			
-		/* XXX */
-		goto retry;
-	}
-#endif	
 	if (seg_count == 0) {
 		if (cxgb_debug)
 			printf("empty segment chain\n");
@@ -302,16 +265,20 @@ retry:
 	}
 	n = *m;
 	while (n) {
-		if (((n->m_flags & (M_EXT|M_NOFREE)) == M_EXT) &&
-		    (n->m_len > 0) && (n->m_ext.ext_type != EXT_PACKET) )
+		if (n->m_ext.ext_type == EXT_PACKET)
+			goto skip; 
+		else if (n->m_len == 0)
+			/* do nothing */;
+		else if ((n->m_flags & (M_EXT|M_NOFREE)) == M_EXT)
 			n->m_flags &= ~M_EXT; 
-		else if ((n->m_len > 0) || (n->m_ext.ext_type == EXT_PACKET)) {
-			n = n->m_next;
-			continue;
-		}
+		else
+			goto skip;
 		mhead = n->m_next;
 		m_free(n);
 		n = mhead;
+		continue;
+	skip:
+		n = n->m_next;
 	}
 	*nsegs = seg_count;
 	*m = m0;