diff options
Diffstat (limited to 'sys/dev/mlx4/mlx4_en')
| -rw-r--r-- | sys/dev/mlx4/mlx4_en/en.h | 27 | ||||
| -rw-r--r-- | sys/dev/mlx4/mlx4_en/mlx4_en_cq.c | 4 | ||||
| -rw-r--r-- | sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c | 70 | ||||
| -rw-r--r-- | sys/dev/mlx4/mlx4_en/mlx4_en_port.c | 1 | ||||
| -rw-r--r-- | sys/dev/mlx4/mlx4_en/mlx4_en_rx.c | 194 | ||||
| -rw-r--r-- | sys/dev/mlx4/mlx4_en/mlx4_en_tx.c | 140 |
6 files changed, 261 insertions, 175 deletions
diff --git a/sys/dev/mlx4/mlx4_en/en.h b/sys/dev/mlx4/mlx4_en/en.h index b4abecb84c24..fb765a698c6c 100644 --- a/sys/dev/mlx4/mlx4_en/en.h +++ b/sys/dev/mlx4/mlx4_en/en.h @@ -54,6 +54,7 @@ #include <dev/mlx4/cmd.h> #include <netinet/tcp_lro.h> +#include <netinet/netdump/netdump.h> #include "en_port.h" #include <dev/mlx4/stats.h> @@ -74,6 +75,15 @@ #define MAX_RX_RINGS 128 #define MIN_RX_RINGS 4 #define TXBB_SIZE 64 + +#ifndef MLX4_EN_MAX_RX_SEGS +#define MLX4_EN_MAX_RX_SEGS 1 /* or 8 */ +#endif + +#ifndef MLX4_EN_MAX_RX_BYTES +#define MLX4_EN_MAX_RX_BYTES MCLBYTES +#endif + #define HEADROOM (2048 / TXBB_SIZE + 1) #define INIT_OWNER_BIT 0xffffffff #define STAMP_STRIDE 64 @@ -271,10 +281,8 @@ struct mlx4_en_tx_ring { u32 doorbell_qpn; u8 *buf; u16 poll_cnt; - int blocked; struct mlx4_en_tx_info *tx_info; u8 queue_index; - struct buf_ring *br; u32 last_nr_txbb; struct mlx4_qp qp; struct mlx4_qp_context context; @@ -298,10 +306,12 @@ struct mlx4_en_tx_ring { }; struct mlx4_en_rx_desc { - /* actual number of entries depends on rx ring stride */ - struct mlx4_wqe_data_seg data[0]; + struct mlx4_wqe_data_seg data[MLX4_EN_MAX_RX_SEGS]; }; +/* the size of the structure above must be power of two */ +CTASSERT(powerof2(sizeof(struct mlx4_en_rx_desc))); + struct mlx4_en_rx_mbuf { bus_dmamap_t dma_map; struct mbuf *mbuf; @@ -310,7 +320,7 @@ struct mlx4_en_rx_mbuf { struct mlx4_en_rx_spare { bus_dmamap_t dma_map; struct mbuf *mbuf; - u64 paddr_be; + bus_dma_segment_t segs[MLX4_EN_MAX_RX_SEGS]; }; struct mlx4_en_rx_ring { @@ -320,7 +330,6 @@ struct mlx4_en_rx_ring { u32 size ; /* number of Rx descs*/ u32 actual_size; u32 size_mask; - u16 stride; u16 log_stride; u16 cqn; /* index of port CQ associated with this ring */ u32 prod; @@ -328,6 +337,7 @@ struct mlx4_en_rx_ring { u32 buf_size; u8 fcs_del; u32 rx_mb_size; + u32 rx_mr_key_be; int qpn; u8 *buf; struct mlx4_en_rx_mbuf *mbuf; @@ -560,7 +570,6 @@ struct mlx4_en_priv { int registered; int gone; int allocated; - int stride; unsigned char current_mac[ETH_ALEN + 2]; u64 mac; int mac_index; @@ -786,6 +795,7 @@ int mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); void mlx4_en_tx_irq(struct mlx4_cq *mcq); u16 mlx4_en_select_queue(struct net_device *dev, struct mbuf *mb); +int mlx4_en_xmit(struct mlx4_en_priv *priv, int tx_ind, struct mbuf **mbp); int mlx4_en_transmit(struct ifnet *dev, struct mbuf *m); int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring **pring, @@ -805,8 +815,7 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, u32 size, int node); void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring **pring, - u32 size, u16 stride); -void mlx4_en_tx_que(void *context, int pending); + u32 size); void mlx4_en_rx_que(void *context, int pending); int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv); void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, diff --git a/sys/dev/mlx4/mlx4_en/mlx4_en_cq.c b/sys/dev/mlx4/mlx4_en/mlx4_en_cq.c index f19ce2f1e1ef..1b198778bfdd 100644 --- a/sys/dev/mlx4/mlx4_en/mlx4_en_cq.c +++ b/sys/dev/mlx4/mlx4_en/mlx4_en_cq.c @@ -44,6 +44,10 @@ static void mlx4_en_cq_event(struct mlx4_cq *cq, enum mlx4_event event) return; } +static void mlx4_en_tx_que(void *arg, int pending) +{ + +} int mlx4_en_create_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq, diff --git a/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c b/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c index ca7e0d74e59f..ecddc21bec3b 100644 --- a/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c +++ b/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c @@ -53,6 +53,8 @@ #include "en.h" #include "en_port.h" +NETDUMP_DEFINE(mlx4_en); + static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv); static void mlx4_en_sysctl_conf(struct mlx4_en_priv *priv); @@ -1574,9 +1576,12 @@ static void mlx4_en_restart(struct work_struct *work) if (priv->blocked == 0 || priv->port_up == 0) return; for (i = 0; i < priv->tx_ring_num; i++) { + int watchdog_time; + ring = priv->tx_ring[i]; - if (ring->blocked && - ring->watchdog_time + MLX4_EN_WATCHDOG_TIMEOUT < ticks) + watchdog_time = READ_ONCE(ring->watchdog_time); + if (watchdog_time != 0 && + time_after(ticks, ring->watchdog_time)) goto reset; } return; @@ -1678,7 +1683,7 @@ void mlx4_en_free_resources(struct mlx4_en_priv *priv) for (i = 0; i < priv->rx_ring_num; i++) { if (priv->rx_ring[i]) mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], - priv->prof->rx_ring_size, priv->stride); + priv->prof->rx_ring_size); if (priv->rx_cq[i]) mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); } @@ -1729,8 +1734,7 @@ err: for (i = 0; i < priv->rx_ring_num; i++) { if (priv->rx_ring[i]) mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], - prof->rx_ring_size, - priv->stride); + prof->rx_ring_size); if (priv->rx_cq[i]) mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); } @@ -2232,9 +2236,6 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, goto out; } - priv->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + - DS_SIZE); - mlx4_en_sysctl_conf(priv); err = mlx4_en_alloc_resources(priv); @@ -2308,6 +2309,8 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO); + NETDUMP_SET(dev, mlx4_en); + en_warn(priv, "Using %d TX rings\n", prof->tx_ring_num); en_warn(priv, "Using %d RX rings\n", prof->rx_ring_num); @@ -2889,3 +2892,54 @@ static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv) CTLFLAG_RD, &rx_ring->errors, 0, "RX soft errors"); } } + +#ifdef NETDUMP +static void +mlx4_en_netdump_init(struct ifnet *dev, int *nrxr, int *ncl, int *clsize) +{ + struct mlx4_en_priv *priv; + + priv = if_getsoftc(dev); + mutex_lock(&priv->mdev->state_lock); + *nrxr = priv->rx_ring_num; + *ncl = NETDUMP_MAX_IN_FLIGHT; + *clsize = priv->rx_mb_size; + mutex_unlock(&priv->mdev->state_lock); +} + +static void +mlx4_en_netdump_event(struct ifnet *dev, enum netdump_ev event) +{ +} + +static int +mlx4_en_netdump_transmit(struct ifnet *dev, struct mbuf *m) +{ + struct mlx4_en_priv *priv; + int err; + + priv = if_getsoftc(dev); + if ((if_getdrvflags(dev) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != + IFF_DRV_RUNNING || !priv->link_state) + return (ENOENT); + + err = mlx4_en_xmit(priv, 0, &m); + if (err != 0 && m != NULL) + m_freem(m); + return (err); +} + +static int +mlx4_en_netdump_poll(struct ifnet *dev, int count) +{ + struct mlx4_en_priv *priv; + + priv = if_getsoftc(dev); + if ((if_getdrvflags(dev) & IFF_DRV_RUNNING) == 0 || !priv->link_state) + return (ENOENT); + + mlx4_poll_interrupts(priv->mdev->dev); + + return (0); +} +#endif /* NETDUMP */ diff --git a/sys/dev/mlx4/mlx4_en/mlx4_en_port.c b/sys/dev/mlx4/mlx4_en/mlx4_en_port.c index 4f3b29b5b296..7e8d2c1550e4 100644 --- a/sys/dev/mlx4/mlx4_en/mlx4_en_port.c +++ b/sys/dev/mlx4/mlx4_en/mlx4_en_port.c @@ -256,7 +256,6 @@ int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset) priv->pkstats.rx_length_errors = be32_to_cpu(mlx4_en_stats->RdropLength); priv->pkstats.rx_over_errors = be32_to_cpu(mlx4_en_stats->RdropOvflw); priv->pkstats.rx_crc_errors = be32_to_cpu(mlx4_en_stats->RCRC); - priv->pkstats.rx_dropped = be32_to_cpu(mlx4_en_stats->RdropOvflw); priv->pkstats.tx_dropped = be32_to_cpu(mlx4_en_stats->TDROP); /* RX stats */ diff --git a/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c b/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c index 7266cfa83ad6..c9222e1df55c 100644 --- a/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c +++ b/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c @@ -44,14 +44,13 @@ #include "en.h" - +#if (MLX4_EN_MAX_RX_SEGS == 1) static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, int index) { - struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) - (ring->buf + (ring->stride * index)); - int possible_frags; + struct mlx4_en_rx_desc *rx_desc = + ((struct mlx4_en_rx_desc *)ring->buf) + index; int i; /* Set size and memtype fields */ @@ -63,38 +62,75 @@ static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv, * stride, remaining (unused) fragments must be padded with * null address/size and a special memory key: */ - possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE; - for (i = 1; i < possible_frags; i++) { + for (i = 1; i < MLX4_EN_MAX_RX_SEGS; i++) { rx_desc->data[i].byte_count = 0; rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); rx_desc->data[i].addr = 0; } } +#endif + +static inline struct mbuf * +mlx4_en_alloc_mbuf(struct mlx4_en_rx_ring *ring) +{ + struct mbuf *mb; + +#if (MLX4_EN_MAX_RX_SEGS == 1) + mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); + if (likely(mb != NULL)) + mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size; +#else + mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MLX4_EN_MAX_RX_BYTES); + if (likely(mb != NULL)) { + struct mbuf *mb_head = mb; + int i; + + mb->m_len = MLX4_EN_MAX_RX_BYTES; + mb->m_pkthdr.len = MLX4_EN_MAX_RX_BYTES; + + for (i = 1; i != MLX4_EN_MAX_RX_SEGS; i++) { + if (mb_head->m_pkthdr.len >= ring->rx_mb_size) + break; + mb = (mb->m_next = m_getjcl(M_NOWAIT, MT_DATA, 0, MLX4_EN_MAX_RX_BYTES)); + if (unlikely(mb == NULL)) { + m_freem(mb_head); + return (NULL); + } + mb->m_len = MLX4_EN_MAX_RX_BYTES; + mb_head->m_pkthdr.len += MLX4_EN_MAX_RX_BYTES; + } + /* rewind to first mbuf in chain */ + mb = mb_head; + } +#endif + return (mb); +} static int -mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, - __be64 *pdma, struct mlx4_en_rx_mbuf *mb_list) +mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_desc *rx_desc, + struct mlx4_en_rx_mbuf *mb_list) { - bus_dma_segment_t segs[1]; + bus_dma_segment_t segs[MLX4_EN_MAX_RX_SEGS]; bus_dmamap_t map; struct mbuf *mb; int nsegs; int err; +#if (MLX4_EN_MAX_RX_SEGS != 1) + int i; +#endif /* try to allocate a new spare mbuf */ if (unlikely(ring->spare.mbuf == NULL)) { - mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); + mb = mlx4_en_alloc_mbuf(ring); if (unlikely(mb == NULL)) return (-ENOMEM); - /* setup correct length */ - mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size; /* make sure IP header gets aligned */ m_adj(mb, MLX4_NET_IP_ALIGN); /* load spare mbuf into BUSDMA */ err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, ring->spare.dma_map, - mb, segs, &nsegs, BUS_DMA_NOWAIT); + mb, ring->spare.segs, &nsegs, BUS_DMA_NOWAIT); if (unlikely(err != 0)) { m_freem(mb); return (err); @@ -102,8 +138,14 @@ mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, /* store spare info */ ring->spare.mbuf = mb; - ring->spare.paddr_be = cpu_to_be64(segs[0].ds_addr); +#if (MLX4_EN_MAX_RX_SEGS != 1) + /* zero remaining segs */ + for (i = nsegs; i != MLX4_EN_MAX_RX_SEGS; i++) { + ring->spare.segs[i].ds_addr = 0; + ring->spare.segs[i].ds_len = 0; + } +#endif bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, BUS_DMASYNC_PREREAD); } @@ -115,13 +157,10 @@ mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, bus_dmamap_unload(ring->dma_tag, mb_list->dma_map); } - mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); + mb = mlx4_en_alloc_mbuf(ring); if (unlikely(mb == NULL)) goto use_spare; - /* setup correct length */ - mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size; - /* make sure IP header gets aligned */ m_adj(mb, MLX4_NET_IP_ALIGN); @@ -132,7 +171,20 @@ mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, goto use_spare; } - *pdma = cpu_to_be64(segs[0].ds_addr); +#if (MLX4_EN_MAX_RX_SEGS == 1) + rx_desc->data[0].addr = cpu_to_be64(segs[0].ds_addr); +#else + for (i = 0; i != nsegs; i++) { + rx_desc->data[i].byte_count = cpu_to_be32(segs[i].ds_len); + rx_desc->data[i].lkey = ring->rx_mr_key_be; + rx_desc->data[i].addr = cpu_to_be64(segs[i].ds_addr); + } + for (; i != MLX4_EN_MAX_RX_SEGS; i++) { + rx_desc->data[i].byte_count = 0; + rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); + rx_desc->data[i].addr = 0; + } +#endif mb_list->mbuf = mb; bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_PREREAD); @@ -149,7 +201,21 @@ use_spare: ring->spare.mbuf = NULL; /* store physical address */ - *pdma = ring->spare.paddr_be; +#if (MLX4_EN_MAX_RX_SEGS == 1) + rx_desc->data[0].addr = cpu_to_be64(ring->spare.segs[0].ds_addr); +#else + for (i = 0; i != MLX4_EN_MAX_RX_SEGS; i++) { + if (ring->spare.segs[i].ds_len != 0) { + rx_desc->data[i].byte_count = cpu_to_be32(ring->spare.segs[i].ds_len); + rx_desc->data[i].lkey = ring->rx_mr_key_be; + rx_desc->data[i].addr = cpu_to_be64(ring->spare.segs[i].ds_addr); + } else { + rx_desc->data[i].byte_count = 0; + rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); + rx_desc->data[i].addr = 0; + } + } +#endif return (0); } @@ -167,13 +233,13 @@ static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, int index) { - struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) - (ring->buf + (index * ring->stride)); + struct mlx4_en_rx_desc *rx_desc = + ((struct mlx4_en_rx_desc *)ring->buf) + index; struct mlx4_en_rx_mbuf *mb_list = ring->mbuf + index; mb_list->mbuf = NULL; - if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) { + if (mlx4_en_alloc_buf(ring, rx_desc, mb_list)) { priv->port_stats.rx_alloc_failed++; return (-ENOMEM); } @@ -321,7 +387,7 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ MJUM16BYTES, /* maxsize */ - 1, /* nsegments */ + MLX4_EN_MAX_RX_SEGS, /* nsegments */ MJUM16BYTES, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockfuncarg */ @@ -334,10 +400,9 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, ring->cons = 0; ring->size = size; ring->size_mask = size - 1; - ring->stride = roundup_pow_of_two( - sizeof(struct mlx4_en_rx_desc) + DS_SIZE); - ring->log_stride = ffs(ring->stride) - 1; - ring->buf_size = ring->size * ring->stride + TXBB_SIZE; + + ring->log_stride = ilog2(sizeof(struct mlx4_en_rx_desc)); + ring->buf_size = (ring->size * sizeof(struct mlx4_en_rx_desc)) + TXBB_SIZE; tmp = size * sizeof(struct mlx4_en_rx_mbuf); @@ -398,11 +463,11 @@ err_ring: int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) { struct mlx4_en_rx_ring *ring; +#if (MLX4_EN_MAX_RX_SEGS == 1) int i; +#endif int ring_ind; int err; - int stride = roundup_pow_of_two( - sizeof(struct mlx4_en_rx_desc) + DS_SIZE); for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { ring = priv->rx_ring[ring_ind]; @@ -413,8 +478,7 @@ int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn; ring->rx_mb_size = priv->rx_mb_size; - ring->stride = stride; - if (ring->stride <= TXBB_SIZE) { + if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE) { /* Stamp first unused send wqe */ __be32 *ptr = (__be32 *)ring->buf; __be32 stamp = cpu_to_be32(1 << STAMP_SHIFT); @@ -423,15 +487,18 @@ int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) ring->buf += TXBB_SIZE; } - ring->log_stride = ffs(ring->stride) - 1; - ring->buf_size = ring->size * ring->stride; + ring->log_stride = ilog2(sizeof(struct mlx4_en_rx_desc)); + ring->buf_size = ring->size * sizeof(struct mlx4_en_rx_desc); memset(ring->buf, 0, ring->buf_size); mlx4_en_update_rx_prod_db(ring); +#if (MLX4_EN_MAX_RX_SEGS == 1) /* Initialize all descriptors */ for (i = 0; i < ring->size; i++) mlx4_en_init_rx_desc(priv, ring, i); +#endif + ring->rx_mr_key_be = cpu_to_be32(priv->mdev->mr.key); #ifdef INET /* Configure lro mngr */ @@ -466,7 +533,7 @@ err_buffers: while (ring_ind >= 0) { ring = priv->rx_ring[ring_ind]; - if (ring->stride <= TXBB_SIZE) + if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE) ring->buf -= TXBB_SIZE; ring_ind--; } @@ -477,14 +544,14 @@ err_buffers: void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring **pring, - u32 size, u16 stride) + u32 size) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_rx_ring *ring = *pring; uint32_t x; mlx4_en_unmap_buffer(&ring->wqres.buf); - mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); + mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * sizeof(struct mlx4_en_rx_desc) + TXBB_SIZE); for (x = 0; x != size; x++) bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map); /* free spare mbuf, if any */ @@ -511,7 +578,7 @@ void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, tcp_lro_free(&ring->lro); #endif mlx4_en_free_rx_buf(priv, ring); - if (ring->stride <= TXBB_SIZE) + if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE) ring->buf -= TXBB_SIZE; } @@ -557,21 +624,61 @@ mlx4_en_rx_mb(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_mbuf *mb_list, int length) { +#if (MLX4_EN_MAX_RX_SEGS != 1) + struct mbuf *mb_head; +#endif struct mbuf *mb; + /* optimise reception of small packets */ + if (length <= (MHLEN - MLX4_NET_IP_ALIGN) && + (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) { + + /* set packet length */ + mb->m_pkthdr.len = mb->m_len = length; + + /* make sure IP header gets aligned */ + mb->m_data += MLX4_NET_IP_ALIGN; + + bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, + BUS_DMASYNC_POSTREAD); + + bcopy(mtod(mb_list->mbuf, caddr_t), mtod(mb, caddr_t), length); + + return (mb); + } + /* get mbuf */ mb = mb_list->mbuf; /* collect used fragment while atomically replacing it */ - if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) + if (mlx4_en_alloc_buf(ring, rx_desc, mb_list)) return (NULL); /* range check hardware computed value */ - if (unlikely(length > mb->m_len)) - length = mb->m_len; + if (unlikely(length > mb->m_pkthdr.len)) + length = mb->m_pkthdr.len; +#if (MLX4_EN_MAX_RX_SEGS == 1) /* update total packet length in packet header */ mb->m_len = mb->m_pkthdr.len = length; +#else + mb->m_pkthdr.len = length; + for (mb_head = mb; mb != NULL; mb = mb->m_next) { + if (mb->m_len > length) + mb->m_len = length; + length -= mb->m_len; + if (likely(length == 0)) { + if (likely(mb->m_next != NULL)) { + /* trim off empty mbufs */ + m_freem(mb->m_next); + mb->m_next = NULL; + } + break; + } + } + /* rewind to first mbuf in chain */ + mb = mb_head; +#endif return (mb); } @@ -660,8 +767,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, cons_index & size)) { mb_list = ring->mbuf + index; - rx_desc = (struct mlx4_en_rx_desc *) - (ring->buf + (index << ring->log_stride)); + rx_desc = ((struct mlx4_en_rx_desc *)ring->buf) + index; /* * make sure we read the CQE after we read the ownership bit @@ -830,7 +936,7 @@ static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn, qp->event = mlx4_en_sqp_event; memset(context, 0, sizeof *context); - mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0, + mlx4_en_fill_qp_context(priv, ring->actual_size, sizeof(struct mlx4_en_rx_desc), 0, 0, qpn, ring->cqn, -1, context); context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma); diff --git a/sys/dev/mlx4/mlx4_en/mlx4_en_tx.c b/sys/dev/mlx4/mlx4_en/mlx4_en_tx.c index a3692712767a..e07964f3225e 100644 --- a/sys/dev/mlx4/mlx4_en/mlx4_en_tx.c +++ b/sys/dev/mlx4/mlx4_en/mlx4_en_tx.c @@ -94,15 +94,6 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, mtx_init(&ring->tx_lock.m, "mlx4 tx", NULL, MTX_DEF); mtx_init(&ring->comp_lock.m, "mlx4 comp", NULL, MTX_DEF); - /* Allocate the buf ring */ - ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF, - M_WAITOK, &ring->tx_lock.m); - if (ring->br == NULL) { - en_err(priv, "Failed allocating tx_info ring\n"); - err = -ENOMEM; - goto err_free_dma_tag; - } - tmp = size * sizeof(struct mlx4_en_tx_info); ring->tx_info = kzalloc_node(tmp, GFP_KERNEL, node); if (!ring->tx_info) { @@ -190,8 +181,6 @@ err_dma_map: err_info: vfree(ring->tx_info); err_ring: - buf_ring_free(ring->br, M_DEVBUF); -err_free_dma_tag: bus_dma_tag_destroy(ring->dma_tag); done: kfree(ring); @@ -206,7 +195,6 @@ void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, uint32_t x; en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn); - buf_ring_free(ring->br, M_DEVBUF); if (ring->bf_enabled) mlx4_bf_free(mdev->dev, &ring->bf); mlx4_qp_remove(mdev->dev, &ring->qp); @@ -236,8 +224,8 @@ int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, ring->cons = 0xffffffff; ring->last_nr_txbb = 1; ring->poll_cnt = 0; - ring->blocked = 0; memset(ring->buf, 0, ring->buf_size); + ring->watchdog_time = 0; ring->qp_state = MLX4_QP_STATE_RST; ring->doorbell_qpn = ring->qp.qpn << 8; @@ -429,14 +417,6 @@ static int mlx4_en_process_tx_cq(struct net_device *dev, wmb(); ring->cons += txbbs_skipped; - /* Wakeup Tx queue if it was stopped and ring is not full */ - if (unlikely(ring->blocked) && !mlx4_en_tx_ring_is_full(ring)) { - ring->blocked = 0; - if (atomic_fetchadd_int(&priv->blocked, -1) == 1) - atomic_clear_int(&dev->if_drv_flags ,IFF_DRV_OACTIVE); - priv->port_stats.wake_queue++; - ring->wake_queue++; - } return (0); } @@ -648,7 +628,7 @@ static void mlx4_bf_copy(void __iomem *dst, volatile unsigned long *src, unsigne __iowrite64_copy(dst, __DEVOLATILE(void *, src), bytecnt / 8); } -static int mlx4_en_xmit(struct mlx4_en_priv *priv, int tx_ind, struct mbuf **mbp) +int mlx4_en_xmit(struct mlx4_en_priv *priv, int tx_ind, struct mbuf **mbp) { enum { DS_FACT = TXBB_SIZE / DS_SIZE_ALIGNMENT, @@ -682,15 +662,6 @@ static int mlx4_en_xmit(struct mlx4_en_priv *priv, int tx_ind, struct mbuf **mbp /* check if TX ring is full */ if (unlikely(mlx4_en_tx_ring_is_full(ring))) { - /* every full native Tx ring stops queue */ - if (ring->blocked == 0) - atomic_add_int(&priv->blocked, 1); - /* Set HW-queue-is-full flag */ - atomic_set_int(&ifp->if_drv_flags, IFF_DRV_OACTIVE); - priv->port_stats.queue_stopped++; - ring->blocked = 1; - ring->queue_stopped++; - /* Use interrupts to find out when queue opened */ mlx4_en_arm_cq(priv, priv->tx_cq[tx_ind]); return (ENOBUFS); @@ -956,73 +927,28 @@ tx_drop: } static int -mlx4_en_transmit_locked(struct ifnet *dev, int tx_ind, struct mbuf *m) +mlx4_en_transmit_locked(struct ifnet *ifp, int tx_ind, struct mbuf *mb) { - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_tx_ring *ring; - struct mbuf *next; - int enqueued, err = 0; - - ring = priv->tx_ring[tx_ind]; - if ((dev->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING || priv->port_up == 0) { - if (m != NULL) - err = drbr_enqueue(dev, ring->br, m); - return (err); - } + struct mlx4_en_priv *priv = netdev_priv(ifp); + struct mlx4_en_tx_ring *ring = priv->tx_ring[tx_ind]; + int err = 0; - enqueued = 0; - if (m != NULL) - /* - * If we can't insert mbuf into drbr, try to xmit anyway. - * We keep the error we got so we could return that after xmit. - */ - err = drbr_enqueue(dev, ring->br, m); - - /* Process the queue */ - while ((next = drbr_peek(dev, ring->br)) != NULL) { - if (mlx4_en_xmit(priv, tx_ind, &next) != 0) { - if (next == NULL) { - drbr_advance(dev, ring->br); - } else { - drbr_putback(dev, ring->br, next); - } - break; - } - drbr_advance(dev, ring->br); - enqueued++; - if ((dev->if_drv_flags & IFF_DRV_RUNNING) == 0) - break; + if (unlikely((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || + READ_ONCE(priv->port_up) == 0)) { + m_freem(mb); + return (ENETDOWN); } - if (enqueued > 0) - ring->watchdog_time = ticks; - - return (err); -} - -void -mlx4_en_tx_que(void *context, int pending) -{ - struct mlx4_en_tx_ring *ring; - struct mlx4_en_priv *priv; - struct net_device *dev; - struct mlx4_en_cq *cq; - int tx_ind; - cq = context; - dev = cq->dev; - priv = dev->if_softc; - tx_ind = cq->ring; - ring = priv->tx_ring[tx_ind]; - - if (priv->port_up != 0 && - (dev->if_drv_flags & IFF_DRV_RUNNING) != 0) { - mlx4_en_xmit_poll(priv, tx_ind); - spin_lock(&ring->tx_lock); - if (!drbr_empty(dev, ring->br)) - mlx4_en_transmit_locked(dev, tx_ind, NULL); - spin_unlock(&ring->tx_lock); + if (mlx4_en_xmit(priv, tx_ind, &mb) != 0) { + /* NOTE: m_freem() is NULL safe */ + m_freem(mb); + err = ENOBUFS; + if (ring->watchdog_time == 0) + ring->watchdog_time = ticks + MLX4_EN_WATCHDOG_TIMEOUT; + } else { + ring->watchdog_time = 0; } + return (err); } int @@ -1030,7 +956,6 @@ mlx4_en_transmit(struct ifnet *dev, struct mbuf *m) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_tx_ring *ring; - struct mlx4_en_cq *cq; int i, err = 0; if (priv->port_up == 0) { @@ -1047,16 +972,14 @@ mlx4_en_transmit(struct ifnet *dev, struct mbuf *m) } ring = priv->tx_ring[i]; - if (spin_trylock(&ring->tx_lock)) { - err = mlx4_en_transmit_locked(dev, i, m); - spin_unlock(&ring->tx_lock); - /* Poll CQ here */ - mlx4_en_xmit_poll(priv, i); - } else { - err = drbr_enqueue(dev, ring->br, m); - cq = priv->tx_cq[i]; - taskqueue_enqueue(cq->tq, &cq->cq_task); - } + + spin_lock(&ring->tx_lock); + + err = mlx4_en_transmit_locked(dev, i, m); + spin_unlock(&ring->tx_lock); + + /* Poll CQ here */ + mlx4_en_xmit_poll(priv, i); #if __FreeBSD_version >= 1100000 if (unlikely(err != 0)) @@ -1072,18 +995,9 @@ void mlx4_en_qflush(struct ifnet *dev) { struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_tx_ring *ring; - struct mbuf *m; if (priv->port_up == 0) return; - for (int i = 0; i < priv->tx_ring_num; i++) { - ring = priv->tx_ring[i]; - spin_lock(&ring->tx_lock); - while ((m = buf_ring_dequeue_sc(ring->br)) != NULL) - m_freem(m); - spin_unlock(&ring->tx_lock); - } if_qflush(dev); } |