IFLIB updates

- unconditionally enable BUS_DMA on non-x86 architectures
- speed up rxd zeroing via customized function
- support out of order updates to rxd's
- add prefetching to hardware descriptor rings
- only prefetch on 10G or faster hardware
- add seperate tx queue intr function
- preliminary rework of NETMAP interfaces, WIP

Submitted by:	Matt Macy <mmacy@nextbsd.org>
Sponsored by:	Limelight Networks

1 files changed, 69 insertions, 45 deletions

diff --git a/sys/net/iflib.h b/sys/net/iflib.h
index 9c8a8c95e13e..9af87a2c09b7 100644
--- a/sys/net/iflib.h
+++ b/sys/net/iflib.h
@@ -37,7 +37,13 @@
 #include <sys/nv.h>
 #include <sys/gtaskqueue.h>
 
-
+/*
+ * The value type for indexing, limits max descriptors
+ * to 65535 can be conditionally redefined to uint32_t
+ * in the future if the need arises.
+ */
+typedef uint16_t qidx_t;
+#define QIDX_INVALID 0xFFFF
 /*
  * Most cards can handle much larger TSO requests
  * but the FreeBSD TCP stack will break on larger
@@ -63,7 +69,7 @@ typedef struct if_int_delay_info  *if_int_delay_info_t;
 
 typedef struct if_rxd_frag {
 	uint8_t irf_flid;
-	uint16_t irf_idx;
+	qidx_t irf_idx;
 	uint16_t irf_len;
 } *if_rxd_frag_t;
 
@@ -73,47 +79,61 @@ typedef struct if_rxd_info {
 	uint16_t iri_vtag;		/* vlan tag - if flag set */
 	/* XXX redundant with the new irf_len field */
 	uint16_t iri_len;		/* packet length */
-	uint16_t iri_cidx;		/* consumer index of cq */
+	qidx_t iri_cidx;		/* consumer index of cq */
 	struct ifnet *iri_ifp;		/* some drivers >1 interface per softc */
 
 	/* updated by driver */
-	uint16_t iri_flags;		/* mbuf flags for packet */
+	if_rxd_frag_t iri_frags;
 	uint32_t iri_flowid;		/* RSS hash for packet */
 	uint32_t iri_csum_flags;	/* m_pkthdr csum flags */
+
 	uint32_t iri_csum_data;		/* m_pkthdr csum data */
+	uint8_t iri_flags;		/* mbuf flags for packet */
 	uint8_t	 iri_nfrags;		/* number of fragments in packet */
 	uint8_t	 iri_rsstype;		/* RSS hash type */
 	uint8_t	 iri_pad;		/* any padding in the received data */
-	if_rxd_frag_t iri_frags;
 } *if_rxd_info_t;
 
+typedef struct if_rxd_update {
+	uint64_t	*iru_paddrs;
+	caddr_t		*iru_vaddrs;
+	qidx_t		*iru_idxs;
+	qidx_t		iru_pidx;
+	uint16_t	iru_qsidx;
+	uint16_t	iru_count;
+	uint16_t	iru_buf_size;
+	uint8_t		iru_flidx;
+} *if_rxd_update_t;
+
 #define IPI_TX_INTR	0x1		/* send an interrupt when this packet is sent */
 #define IPI_TX_IPV4	0x2		/* ethertype IPv4 */
 #define IPI_TX_IPV6	0x4		/* ethertype IPv6 */
 
 typedef struct if_pkt_info {
-	uint32_t			ipi_len;	/* packet length */
-	bus_dma_segment_t		*ipi_segs;	/* physical addresses */
-	uint16_t			ipi_qsidx;	/* queue set index */
-	uint16_t			ipi_nsegs;	/* number of segments */
-	uint16_t			ipi_ndescs;	/* number of descriptors used by encap */
-	uint16_t			ipi_flags;	/* iflib per-packet flags */
-	uint32_t			ipi_pidx;	/* start pidx for encap */
-	uint32_t			ipi_new_pidx;	/* next available pidx post-encap */
+	bus_dma_segment_t	*ipi_segs;	/* physical addresses */
+	uint32_t		ipi_len;	/* packet length */
+	uint16_t		ipi_qsidx;	/* queue set index */
+	qidx_t			ipi_nsegs;	/* number of segments */
+
+	qidx_t			ipi_ndescs;	/* number of descriptors used by encap */
+	uint16_t		ipi_flags;	/* iflib per-packet flags */
+	qidx_t			ipi_pidx;	/* start pidx for encap */
+	qidx_t			ipi_new_pidx;	/* next available pidx post-encap */
 	/* offload handling */
-	uint64_t			ipi_csum_flags;	/* packet checksum flags */
-	uint16_t			ipi_tso_segsz;	/* tso segment size */
-	uint16_t			ipi_mflags;	/* packet mbuf flags */
-	uint16_t			ipi_vtag;	/* VLAN tag */
-	uint16_t			ipi_etype;	/* ether header type */
-	uint8_t				ipi_ehdrlen;	/* ether header length */
-	uint8_t				ipi_ip_hlen;	/* ip header length */
-	uint8_t				ipi_tcp_hlen;	/* tcp header length */
-	uint8_t				ipi_tcp_hflags;	/* tcp header flags */
-	uint8_t				ipi_ipproto;	/* ip protocol */
-	/* implied padding */
-	uint32_t			ipi_tcp_seq;	/* tcp seqno */
-	uint32_t			ipi_tcp_sum;	/* tcp csum */
+	uint8_t			ipi_ehdrlen;	/* ether header length */
+	uint8_t			ipi_ip_hlen;	/* ip header length */
+	uint8_t			ipi_tcp_hlen;	/* tcp header length */
+	uint8_t			ipi_ipproto;	/* ip protocol */
+
+	uint32_t		ipi_csum_flags;	/* packet checksum flags */
+	uint16_t		ipi_tso_segsz;	/* tso segment size */
+	uint16_t		ipi_vtag;	/* VLAN tag */
+	uint16_t		ipi_etype;	/* ether header type */
+	uint8_t			ipi_tcp_hflags;	/* tcp header flags */
+	uint8_t			ipi_mflags;	/* packet mbuf flags */
+
+	uint32_t		ipi_tcp_seq;	/* tcp seqno */
+	uint32_t		ipi_tcp_sum;	/* tcp csum */
 } *if_pkt_info_t;
 
 typedef struct if_irq {
@@ -156,15 +176,13 @@ typedef struct pci_vendor_info {
 
 typedef struct if_txrx {
 	int (*ift_txd_encap) (void *, if_pkt_info_t);
-	void (*ift_txd_flush) (void *, uint16_t, uint32_t);
-	int (*ift_txd_credits_update) (void *, uint16_t, uint32_t, bool);
+	void (*ift_txd_flush) (void *, uint16_t, qidx_t pidx);
+	int (*ift_txd_credits_update) (void *, uint16_t qsidx, bool clear);
 
-	int (*ift_rxd_available) (void *, uint16_t qsidx, uint32_t pidx,
-	    int budget);
+	int (*ift_rxd_available) (void *, uint16_t qsidx, qidx_t pidx, qidx_t budget);
 	int (*ift_rxd_pkt_get) (void *, if_rxd_info_t ri);
-	void (*ift_rxd_refill) (void * , uint16_t qsidx, uint8_t flidx, uint32_t pidx,
-							uint64_t *paddrs, caddr_t *vaddrs, uint16_t count, uint16_t buf_size);
-	void (*ift_rxd_flush) (void *, uint16_t qsidx, uint8_t flidx, uint32_t pidx);
+	void (*ift_rxd_refill) (void * , if_rxd_update_t iru);
+	void (*ift_rxd_flush) (void *, uint16_t qsidx, uint8_t flidx, qidx_t pidx);
 	int (*ift_legacy_intr) (void *);
 } *if_txrx_t;
 
@@ -179,6 +197,10 @@ typedef struct if_softc_ctx {
 
 	uint32_t isc_txqsizes[8];
 	uint32_t isc_rxqsizes[8];
+	/* is there such thing as a descriptor that is more than 248 bytes ? */
+	uint8_t isc_txd_size[8];
+	uint8_t isc_rxd_size[8];
+
 	int isc_max_txqsets;
 	int isc_max_rxqsets;
 	int isc_tx_tso_segments_max;
@@ -203,22 +225,14 @@ typedef struct if_softc_ctx {
 struct if_shared_ctx {
 	int isc_magic;
 	driver_t *isc_driver;
-	int isc_nfl;
-	int isc_flags;
 	bus_size_t isc_q_align;
 	bus_size_t isc_tx_maxsize;
 	bus_size_t isc_tx_maxsegsize;
 	bus_size_t isc_rx_maxsize;
 	bus_size_t isc_rx_maxsegsize;
 	int isc_rx_nsegments;
-	int isc_rx_process_limit;
-	int isc_ntxqs;			/* # of tx queues per tx qset - usually 1 */
-	int isc_nrxqs;			/* # of rx queues per rx qset - intel 1, chelsio 2, broadcom 3 */
 	int isc_admin_intrcnt;		/* # of admin/link interrupts */
 
-
-	int isc_tx_reclaim_thresh;
-
 	/* fields necessary for probe */
 	pci_vendor_info_t *isc_vendor_info;
 	char *isc_driver_version;
@@ -231,6 +245,14 @@ struct if_shared_ctx {
 	int isc_ntxd_min[8];
 	int isc_ntxd_default[8];
 	int isc_ntxd_max[8];
+
+	/* actively used during operation */
+	int isc_nfl __aligned(CACHE_LINE_SIZE);
+	int isc_ntxqs;			/* # of tx queues per tx qset - usually 1 */
+	int isc_nrxqs;			/* # of rx queues per rx qset - intel 1, chelsio 2, broadcom 3 */
+	int isc_rx_process_limit;
+	int isc_tx_reclaim_thresh;
+	int isc_flags;
 };
 
 typedef struct iflib_dma_info {
@@ -244,8 +266,9 @@ typedef struct iflib_dma_info {
 #define IFLIB_MAGIC 0xCAFEF00D
 
 typedef enum {
-	IFLIB_INTR_TX,
 	IFLIB_INTR_RX,
+	IFLIB_INTR_TX,
+	IFLIB_INTR_RXTX,
 	IFLIB_INTR_ADMIN,
 	IFLIB_INTR_IOV,
 } iflib_intr_type_t;
@@ -279,6 +302,10 @@ typedef enum {
  * Interface doesn't expect in_pseudo for th_sum
  */
 #define IFLIB_TSO_INIT_IP	0x20
+/*
+ * Interface doesn't align IP header
+ */
+#define IFLIB_DO_RX_FIXUP	0x40
 
 
 
@@ -298,9 +325,6 @@ if_shared_ctx_t iflib_get_sctx(if_ctx_t ctx);
 
 void iflib_set_mac(if_ctx_t ctx, uint8_t mac[ETHER_ADDR_LEN]);
 
-
-
-
 /*
  * If the driver can plug cleanly in to newbus use these
  */