aboutsummaryrefslogtreecommitdiff
path: root/sys/dev/mlx4/mlx4_en
diff options
context:
space:
mode:
Diffstat (limited to 'sys/dev/mlx4/mlx4_en')
-rw-r--r--sys/dev/mlx4/mlx4_en/en.h27
-rw-r--r--sys/dev/mlx4/mlx4_en/mlx4_en_cq.c4
-rw-r--r--sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c70
-rw-r--r--sys/dev/mlx4/mlx4_en/mlx4_en_port.c1
-rw-r--r--sys/dev/mlx4/mlx4_en/mlx4_en_rx.c194
-rw-r--r--sys/dev/mlx4/mlx4_en/mlx4_en_tx.c140
6 files changed, 261 insertions, 175 deletions
diff --git a/sys/dev/mlx4/mlx4_en/en.h b/sys/dev/mlx4/mlx4_en/en.h
index b4abecb84c24..fb765a698c6c 100644
--- a/sys/dev/mlx4/mlx4_en/en.h
+++ b/sys/dev/mlx4/mlx4_en/en.h
@@ -54,6 +54,7 @@
#include <dev/mlx4/cmd.h>
#include <netinet/tcp_lro.h>
+#include <netinet/netdump/netdump.h>
#include "en_port.h"
#include <dev/mlx4/stats.h>
@@ -74,6 +75,15 @@
#define MAX_RX_RINGS 128
#define MIN_RX_RINGS 4
#define TXBB_SIZE 64
+
+#ifndef MLX4_EN_MAX_RX_SEGS
+#define MLX4_EN_MAX_RX_SEGS 1 /* or 8 */
+#endif
+
+#ifndef MLX4_EN_MAX_RX_BYTES
+#define MLX4_EN_MAX_RX_BYTES MCLBYTES
+#endif
+
#define HEADROOM (2048 / TXBB_SIZE + 1)
#define INIT_OWNER_BIT 0xffffffff
#define STAMP_STRIDE 64
@@ -271,10 +281,8 @@ struct mlx4_en_tx_ring {
u32 doorbell_qpn;
u8 *buf;
u16 poll_cnt;
- int blocked;
struct mlx4_en_tx_info *tx_info;
u8 queue_index;
- struct buf_ring *br;
u32 last_nr_txbb;
struct mlx4_qp qp;
struct mlx4_qp_context context;
@@ -298,10 +306,12 @@ struct mlx4_en_tx_ring {
};
struct mlx4_en_rx_desc {
- /* actual number of entries depends on rx ring stride */
- struct mlx4_wqe_data_seg data[0];
+ struct mlx4_wqe_data_seg data[MLX4_EN_MAX_RX_SEGS];
};
+/* the size of the structure above must be power of two */
+CTASSERT(powerof2(sizeof(struct mlx4_en_rx_desc)));
+
struct mlx4_en_rx_mbuf {
bus_dmamap_t dma_map;
struct mbuf *mbuf;
@@ -310,7 +320,7 @@ struct mlx4_en_rx_mbuf {
struct mlx4_en_rx_spare {
bus_dmamap_t dma_map;
struct mbuf *mbuf;
- u64 paddr_be;
+ bus_dma_segment_t segs[MLX4_EN_MAX_RX_SEGS];
};
struct mlx4_en_rx_ring {
@@ -320,7 +330,6 @@ struct mlx4_en_rx_ring {
u32 size ; /* number of Rx descs*/
u32 actual_size;
u32 size_mask;
- u16 stride;
u16 log_stride;
u16 cqn; /* index of port CQ associated with this ring */
u32 prod;
@@ -328,6 +337,7 @@ struct mlx4_en_rx_ring {
u32 buf_size;
u8 fcs_del;
u32 rx_mb_size;
+ u32 rx_mr_key_be;
int qpn;
u8 *buf;
struct mlx4_en_rx_mbuf *mbuf;
@@ -560,7 +570,6 @@ struct mlx4_en_priv {
int registered;
int gone;
int allocated;
- int stride;
unsigned char current_mac[ETH_ALEN + 2];
u64 mac;
int mac_index;
@@ -786,6 +795,7 @@ int mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
void mlx4_en_tx_irq(struct mlx4_cq *mcq);
u16 mlx4_en_select_queue(struct net_device *dev, struct mbuf *mb);
+int mlx4_en_xmit(struct mlx4_en_priv *priv, int tx_ind, struct mbuf **mbp);
int mlx4_en_transmit(struct ifnet *dev, struct mbuf *m);
int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring **pring,
@@ -805,8 +815,7 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
u32 size, int node);
void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring **pring,
- u32 size, u16 stride);
-void mlx4_en_tx_que(void *context, int pending);
+ u32 size);
void mlx4_en_rx_que(void *context, int pending);
int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv);
void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
diff --git a/sys/dev/mlx4/mlx4_en/mlx4_en_cq.c b/sys/dev/mlx4/mlx4_en/mlx4_en_cq.c
index f19ce2f1e1ef..1b198778bfdd 100644
--- a/sys/dev/mlx4/mlx4_en/mlx4_en_cq.c
+++ b/sys/dev/mlx4/mlx4_en/mlx4_en_cq.c
@@ -44,6 +44,10 @@ static void mlx4_en_cq_event(struct mlx4_cq *cq, enum mlx4_event event)
return;
}
+static void mlx4_en_tx_que(void *arg, int pending)
+{
+
+}
int mlx4_en_create_cq(struct mlx4_en_priv *priv,
struct mlx4_en_cq **pcq,
diff --git a/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c b/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c
index ca7e0d74e59f..ecddc21bec3b 100644
--- a/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c
+++ b/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c
@@ -53,6 +53,8 @@
#include "en.h"
#include "en_port.h"
+NETDUMP_DEFINE(mlx4_en);
+
static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv);
static void mlx4_en_sysctl_conf(struct mlx4_en_priv *priv);
@@ -1574,9 +1576,12 @@ static void mlx4_en_restart(struct work_struct *work)
if (priv->blocked == 0 || priv->port_up == 0)
return;
for (i = 0; i < priv->tx_ring_num; i++) {
+ int watchdog_time;
+
ring = priv->tx_ring[i];
- if (ring->blocked &&
- ring->watchdog_time + MLX4_EN_WATCHDOG_TIMEOUT < ticks)
+ watchdog_time = READ_ONCE(ring->watchdog_time);
+ if (watchdog_time != 0 &&
+ time_after(ticks, ring->watchdog_time))
goto reset;
}
return;
@@ -1678,7 +1683,7 @@ void mlx4_en_free_resources(struct mlx4_en_priv *priv)
for (i = 0; i < priv->rx_ring_num; i++) {
if (priv->rx_ring[i])
mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i],
- priv->prof->rx_ring_size, priv->stride);
+ priv->prof->rx_ring_size);
if (priv->rx_cq[i])
mlx4_en_destroy_cq(priv, &priv->rx_cq[i]);
}
@@ -1729,8 +1734,7 @@ err:
for (i = 0; i < priv->rx_ring_num; i++) {
if (priv->rx_ring[i])
mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i],
- prof->rx_ring_size,
- priv->stride);
+ prof->rx_ring_size);
if (priv->rx_cq[i])
mlx4_en_destroy_cq(priv, &priv->rx_cq[i]);
}
@@ -2232,9 +2236,6 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
goto out;
}
- priv->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
- DS_SIZE);
-
mlx4_en_sysctl_conf(priv);
err = mlx4_en_alloc_resources(priv);
@@ -2308,6 +2309,8 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL);
ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO);
+ NETDUMP_SET(dev, mlx4_en);
+
en_warn(priv, "Using %d TX rings\n", prof->tx_ring_num);
en_warn(priv, "Using %d RX rings\n", prof->rx_ring_num);
@@ -2889,3 +2892,54 @@ static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv)
CTLFLAG_RD, &rx_ring->errors, 0, "RX soft errors");
}
}
+
+#ifdef NETDUMP
+static void
+mlx4_en_netdump_init(struct ifnet *dev, int *nrxr, int *ncl, int *clsize)
+{
+ struct mlx4_en_priv *priv;
+
+ priv = if_getsoftc(dev);
+ mutex_lock(&priv->mdev->state_lock);
+ *nrxr = priv->rx_ring_num;
+ *ncl = NETDUMP_MAX_IN_FLIGHT;
+ *clsize = priv->rx_mb_size;
+ mutex_unlock(&priv->mdev->state_lock);
+}
+
+static void
+mlx4_en_netdump_event(struct ifnet *dev, enum netdump_ev event)
+{
+}
+
+static int
+mlx4_en_netdump_transmit(struct ifnet *dev, struct mbuf *m)
+{
+ struct mlx4_en_priv *priv;
+ int err;
+
+ priv = if_getsoftc(dev);
+ if ((if_getdrvflags(dev) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
+ IFF_DRV_RUNNING || !priv->link_state)
+ return (ENOENT);
+
+ err = mlx4_en_xmit(priv, 0, &m);
+ if (err != 0 && m != NULL)
+ m_freem(m);
+ return (err);
+}
+
+static int
+mlx4_en_netdump_poll(struct ifnet *dev, int count)
+{
+ struct mlx4_en_priv *priv;
+
+ priv = if_getsoftc(dev);
+ if ((if_getdrvflags(dev) & IFF_DRV_RUNNING) == 0 || !priv->link_state)
+ return (ENOENT);
+
+ mlx4_poll_interrupts(priv->mdev->dev);
+
+ return (0);
+}
+#endif /* NETDUMP */
diff --git a/sys/dev/mlx4/mlx4_en/mlx4_en_port.c b/sys/dev/mlx4/mlx4_en/mlx4_en_port.c
index 4f3b29b5b296..7e8d2c1550e4 100644
--- a/sys/dev/mlx4/mlx4_en/mlx4_en_port.c
+++ b/sys/dev/mlx4/mlx4_en/mlx4_en_port.c
@@ -256,7 +256,6 @@ int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset)
priv->pkstats.rx_length_errors = be32_to_cpu(mlx4_en_stats->RdropLength);
priv->pkstats.rx_over_errors = be32_to_cpu(mlx4_en_stats->RdropOvflw);
priv->pkstats.rx_crc_errors = be32_to_cpu(mlx4_en_stats->RCRC);
- priv->pkstats.rx_dropped = be32_to_cpu(mlx4_en_stats->RdropOvflw);
priv->pkstats.tx_dropped = be32_to_cpu(mlx4_en_stats->TDROP);
/* RX stats */
diff --git a/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c b/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c
index 7266cfa83ad6..c9222e1df55c 100644
--- a/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c
+++ b/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c
@@ -44,14 +44,13 @@
#include "en.h"
-
+#if (MLX4_EN_MAX_RX_SEGS == 1)
static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring,
int index)
{
- struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *)
- (ring->buf + (ring->stride * index));
- int possible_frags;
+ struct mlx4_en_rx_desc *rx_desc =
+ ((struct mlx4_en_rx_desc *)ring->buf) + index;
int i;
/* Set size and memtype fields */
@@ -63,38 +62,75 @@ static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv,
* stride, remaining (unused) fragments must be padded with
* null address/size and a special memory key:
*/
- possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE;
- for (i = 1; i < possible_frags; i++) {
+ for (i = 1; i < MLX4_EN_MAX_RX_SEGS; i++) {
rx_desc->data[i].byte_count = 0;
rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD);
rx_desc->data[i].addr = 0;
}
}
+#endif
+
+static inline struct mbuf *
+mlx4_en_alloc_mbuf(struct mlx4_en_rx_ring *ring)
+{
+ struct mbuf *mb;
+
+#if (MLX4_EN_MAX_RX_SEGS == 1)
+ mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size);
+ if (likely(mb != NULL))
+ mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size;
+#else
+ mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MLX4_EN_MAX_RX_BYTES);
+ if (likely(mb != NULL)) {
+ struct mbuf *mb_head = mb;
+ int i;
+
+ mb->m_len = MLX4_EN_MAX_RX_BYTES;
+ mb->m_pkthdr.len = MLX4_EN_MAX_RX_BYTES;
+
+ for (i = 1; i != MLX4_EN_MAX_RX_SEGS; i++) {
+ if (mb_head->m_pkthdr.len >= ring->rx_mb_size)
+ break;
+ mb = (mb->m_next = m_getjcl(M_NOWAIT, MT_DATA, 0, MLX4_EN_MAX_RX_BYTES));
+ if (unlikely(mb == NULL)) {
+ m_freem(mb_head);
+ return (NULL);
+ }
+ mb->m_len = MLX4_EN_MAX_RX_BYTES;
+ mb_head->m_pkthdr.len += MLX4_EN_MAX_RX_BYTES;
+ }
+ /* rewind to first mbuf in chain */
+ mb = mb_head;
+ }
+#endif
+ return (mb);
+}
static int
-mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring,
- __be64 *pdma, struct mlx4_en_rx_mbuf *mb_list)
+mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_desc *rx_desc,
+ struct mlx4_en_rx_mbuf *mb_list)
{
- bus_dma_segment_t segs[1];
+ bus_dma_segment_t segs[MLX4_EN_MAX_RX_SEGS];
bus_dmamap_t map;
struct mbuf *mb;
int nsegs;
int err;
+#if (MLX4_EN_MAX_RX_SEGS != 1)
+ int i;
+#endif
/* try to allocate a new spare mbuf */
if (unlikely(ring->spare.mbuf == NULL)) {
- mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size);
+ mb = mlx4_en_alloc_mbuf(ring);
if (unlikely(mb == NULL))
return (-ENOMEM);
- /* setup correct length */
- mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size;
/* make sure IP header gets aligned */
m_adj(mb, MLX4_NET_IP_ALIGN);
/* load spare mbuf into BUSDMA */
err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, ring->spare.dma_map,
- mb, segs, &nsegs, BUS_DMA_NOWAIT);
+ mb, ring->spare.segs, &nsegs, BUS_DMA_NOWAIT);
if (unlikely(err != 0)) {
m_freem(mb);
return (err);
@@ -102,8 +138,14 @@ mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring,
/* store spare info */
ring->spare.mbuf = mb;
- ring->spare.paddr_be = cpu_to_be64(segs[0].ds_addr);
+#if (MLX4_EN_MAX_RX_SEGS != 1)
+ /* zero remaining segs */
+ for (i = nsegs; i != MLX4_EN_MAX_RX_SEGS; i++) {
+ ring->spare.segs[i].ds_addr = 0;
+ ring->spare.segs[i].ds_len = 0;
+ }
+#endif
bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map,
BUS_DMASYNC_PREREAD);
}
@@ -115,13 +157,10 @@ mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring,
bus_dmamap_unload(ring->dma_tag, mb_list->dma_map);
}
- mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size);
+ mb = mlx4_en_alloc_mbuf(ring);
if (unlikely(mb == NULL))
goto use_spare;
- /* setup correct length */
- mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size;
-
/* make sure IP header gets aligned */
m_adj(mb, MLX4_NET_IP_ALIGN);
@@ -132,7 +171,20 @@ mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring,
goto use_spare;
}
- *pdma = cpu_to_be64(segs[0].ds_addr);
+#if (MLX4_EN_MAX_RX_SEGS == 1)
+ rx_desc->data[0].addr = cpu_to_be64(segs[0].ds_addr);
+#else
+ for (i = 0; i != nsegs; i++) {
+ rx_desc->data[i].byte_count = cpu_to_be32(segs[i].ds_len);
+ rx_desc->data[i].lkey = ring->rx_mr_key_be;
+ rx_desc->data[i].addr = cpu_to_be64(segs[i].ds_addr);
+ }
+ for (; i != MLX4_EN_MAX_RX_SEGS; i++) {
+ rx_desc->data[i].byte_count = 0;
+ rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD);
+ rx_desc->data[i].addr = 0;
+ }
+#endif
mb_list->mbuf = mb;
bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_PREREAD);
@@ -149,7 +201,21 @@ use_spare:
ring->spare.mbuf = NULL;
/* store physical address */
- *pdma = ring->spare.paddr_be;
+#if (MLX4_EN_MAX_RX_SEGS == 1)
+ rx_desc->data[0].addr = cpu_to_be64(ring->spare.segs[0].ds_addr);
+#else
+ for (i = 0; i != MLX4_EN_MAX_RX_SEGS; i++) {
+ if (ring->spare.segs[i].ds_len != 0) {
+ rx_desc->data[i].byte_count = cpu_to_be32(ring->spare.segs[i].ds_len);
+ rx_desc->data[i].lkey = ring->rx_mr_key_be;
+ rx_desc->data[i].addr = cpu_to_be64(ring->spare.segs[i].ds_addr);
+ } else {
+ rx_desc->data[i].byte_count = 0;
+ rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD);
+ rx_desc->data[i].addr = 0;
+ }
+ }
+#endif
return (0);
}
@@ -167,13 +233,13 @@ static int
mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring, int index)
{
- struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *)
- (ring->buf + (index * ring->stride));
+ struct mlx4_en_rx_desc *rx_desc =
+ ((struct mlx4_en_rx_desc *)ring->buf) + index;
struct mlx4_en_rx_mbuf *mb_list = ring->mbuf + index;
mb_list->mbuf = NULL;
- if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) {
+ if (mlx4_en_alloc_buf(ring, rx_desc, mb_list)) {
priv->port_stats.rx_alloc_failed++;
return (-ENOMEM);
}
@@ -321,7 +387,7 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
MJUM16BYTES, /* maxsize */
- 1, /* nsegments */
+ MLX4_EN_MAX_RX_SEGS, /* nsegments */
MJUM16BYTES, /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockfuncarg */
@@ -334,10 +400,9 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
ring->cons = 0;
ring->size = size;
ring->size_mask = size - 1;
- ring->stride = roundup_pow_of_two(
- sizeof(struct mlx4_en_rx_desc) + DS_SIZE);
- ring->log_stride = ffs(ring->stride) - 1;
- ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
+
+ ring->log_stride = ilog2(sizeof(struct mlx4_en_rx_desc));
+ ring->buf_size = (ring->size * sizeof(struct mlx4_en_rx_desc)) + TXBB_SIZE;
tmp = size * sizeof(struct mlx4_en_rx_mbuf);
@@ -398,11 +463,11 @@ err_ring:
int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
{
struct mlx4_en_rx_ring *ring;
+#if (MLX4_EN_MAX_RX_SEGS == 1)
int i;
+#endif
int ring_ind;
int err;
- int stride = roundup_pow_of_two(
- sizeof(struct mlx4_en_rx_desc) + DS_SIZE);
for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
ring = priv->rx_ring[ring_ind];
@@ -413,8 +478,7 @@ int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn;
ring->rx_mb_size = priv->rx_mb_size;
- ring->stride = stride;
- if (ring->stride <= TXBB_SIZE) {
+ if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE) {
/* Stamp first unused send wqe */
__be32 *ptr = (__be32 *)ring->buf;
__be32 stamp = cpu_to_be32(1 << STAMP_SHIFT);
@@ -423,15 +487,18 @@ int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
ring->buf += TXBB_SIZE;
}
- ring->log_stride = ffs(ring->stride) - 1;
- ring->buf_size = ring->size * ring->stride;
+ ring->log_stride = ilog2(sizeof(struct mlx4_en_rx_desc));
+ ring->buf_size = ring->size * sizeof(struct mlx4_en_rx_desc);
memset(ring->buf, 0, ring->buf_size);
mlx4_en_update_rx_prod_db(ring);
+#if (MLX4_EN_MAX_RX_SEGS == 1)
/* Initialize all descriptors */
for (i = 0; i < ring->size; i++)
mlx4_en_init_rx_desc(priv, ring, i);
+#endif
+ ring->rx_mr_key_be = cpu_to_be32(priv->mdev->mr.key);
#ifdef INET
/* Configure lro mngr */
@@ -466,7 +533,7 @@ err_buffers:
while (ring_ind >= 0) {
ring = priv->rx_ring[ring_ind];
- if (ring->stride <= TXBB_SIZE)
+ if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE)
ring->buf -= TXBB_SIZE;
ring_ind--;
}
@@ -477,14 +544,14 @@ err_buffers:
void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring **pring,
- u32 size, u16 stride)
+ u32 size)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_rx_ring *ring = *pring;
uint32_t x;
mlx4_en_unmap_buffer(&ring->wqres.buf);
- mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE);
+ mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * sizeof(struct mlx4_en_rx_desc) + TXBB_SIZE);
for (x = 0; x != size; x++)
bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map);
/* free spare mbuf, if any */
@@ -511,7 +578,7 @@ void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
tcp_lro_free(&ring->lro);
#endif
mlx4_en_free_rx_buf(priv, ring);
- if (ring->stride <= TXBB_SIZE)
+ if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE)
ring->buf -= TXBB_SIZE;
}
@@ -557,21 +624,61 @@ mlx4_en_rx_mb(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring,
struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_mbuf *mb_list,
int length)
{
+#if (MLX4_EN_MAX_RX_SEGS != 1)
+ struct mbuf *mb_head;
+#endif
struct mbuf *mb;
+ /* optimise reception of small packets */
+ if (length <= (MHLEN - MLX4_NET_IP_ALIGN) &&
+ (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) {
+
+ /* set packet length */
+ mb->m_pkthdr.len = mb->m_len = length;
+
+ /* make sure IP header gets aligned */
+ mb->m_data += MLX4_NET_IP_ALIGN;
+
+ bus_dmamap_sync(ring->dma_tag, mb_list->dma_map,
+ BUS_DMASYNC_POSTREAD);
+
+ bcopy(mtod(mb_list->mbuf, caddr_t), mtod(mb, caddr_t), length);
+
+ return (mb);
+ }
+
/* get mbuf */
mb = mb_list->mbuf;
/* collect used fragment while atomically replacing it */
- if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list))
+ if (mlx4_en_alloc_buf(ring, rx_desc, mb_list))
return (NULL);
/* range check hardware computed value */
- if (unlikely(length > mb->m_len))
- length = mb->m_len;
+ if (unlikely(length > mb->m_pkthdr.len))
+ length = mb->m_pkthdr.len;
+#if (MLX4_EN_MAX_RX_SEGS == 1)
/* update total packet length in packet header */
mb->m_len = mb->m_pkthdr.len = length;
+#else
+ mb->m_pkthdr.len = length;
+ for (mb_head = mb; mb != NULL; mb = mb->m_next) {
+ if (mb->m_len > length)
+ mb->m_len = length;
+ length -= mb->m_len;
+ if (likely(length == 0)) {
+ if (likely(mb->m_next != NULL)) {
+ /* trim off empty mbufs */
+ m_freem(mb->m_next);
+ mb->m_next = NULL;
+ }
+ break;
+ }
+ }
+ /* rewind to first mbuf in chain */
+ mb = mb_head;
+#endif
return (mb);
}
@@ -660,8 +767,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud
while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
cons_index & size)) {
mb_list = ring->mbuf + index;
- rx_desc = (struct mlx4_en_rx_desc *)
- (ring->buf + (index << ring->log_stride));
+ rx_desc = ((struct mlx4_en_rx_desc *)ring->buf) + index;
/*
* make sure we read the CQE after we read the ownership bit
@@ -830,7 +936,7 @@ static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn,
qp->event = mlx4_en_sqp_event;
memset(context, 0, sizeof *context);
- mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0,
+ mlx4_en_fill_qp_context(priv, ring->actual_size, sizeof(struct mlx4_en_rx_desc), 0, 0,
qpn, ring->cqn, -1, context);
context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma);
diff --git a/sys/dev/mlx4/mlx4_en/mlx4_en_tx.c b/sys/dev/mlx4/mlx4_en/mlx4_en_tx.c
index a3692712767a..e07964f3225e 100644
--- a/sys/dev/mlx4/mlx4_en/mlx4_en_tx.c
+++ b/sys/dev/mlx4/mlx4_en/mlx4_en_tx.c
@@ -94,15 +94,6 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
mtx_init(&ring->tx_lock.m, "mlx4 tx", NULL, MTX_DEF);
mtx_init(&ring->comp_lock.m, "mlx4 comp", NULL, MTX_DEF);
- /* Allocate the buf ring */
- ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF,
- M_WAITOK, &ring->tx_lock.m);
- if (ring->br == NULL) {
- en_err(priv, "Failed allocating tx_info ring\n");
- err = -ENOMEM;
- goto err_free_dma_tag;
- }
-
tmp = size * sizeof(struct mlx4_en_tx_info);
ring->tx_info = kzalloc_node(tmp, GFP_KERNEL, node);
if (!ring->tx_info) {
@@ -190,8 +181,6 @@ err_dma_map:
err_info:
vfree(ring->tx_info);
err_ring:
- buf_ring_free(ring->br, M_DEVBUF);
-err_free_dma_tag:
bus_dma_tag_destroy(ring->dma_tag);
done:
kfree(ring);
@@ -206,7 +195,6 @@ void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
uint32_t x;
en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn);
- buf_ring_free(ring->br, M_DEVBUF);
if (ring->bf_enabled)
mlx4_bf_free(mdev->dev, &ring->bf);
mlx4_qp_remove(mdev->dev, &ring->qp);
@@ -236,8 +224,8 @@ int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv,
ring->cons = 0xffffffff;
ring->last_nr_txbb = 1;
ring->poll_cnt = 0;
- ring->blocked = 0;
memset(ring->buf, 0, ring->buf_size);
+ ring->watchdog_time = 0;
ring->qp_state = MLX4_QP_STATE_RST;
ring->doorbell_qpn = ring->qp.qpn << 8;
@@ -429,14 +417,6 @@ static int mlx4_en_process_tx_cq(struct net_device *dev,
wmb();
ring->cons += txbbs_skipped;
- /* Wakeup Tx queue if it was stopped and ring is not full */
- if (unlikely(ring->blocked) && !mlx4_en_tx_ring_is_full(ring)) {
- ring->blocked = 0;
- if (atomic_fetchadd_int(&priv->blocked, -1) == 1)
- atomic_clear_int(&dev->if_drv_flags ,IFF_DRV_OACTIVE);
- priv->port_stats.wake_queue++;
- ring->wake_queue++;
- }
return (0);
}
@@ -648,7 +628,7 @@ static void mlx4_bf_copy(void __iomem *dst, volatile unsigned long *src, unsigne
__iowrite64_copy(dst, __DEVOLATILE(void *, src), bytecnt / 8);
}
-static int mlx4_en_xmit(struct mlx4_en_priv *priv, int tx_ind, struct mbuf **mbp)
+int mlx4_en_xmit(struct mlx4_en_priv *priv, int tx_ind, struct mbuf **mbp)
{
enum {
DS_FACT = TXBB_SIZE / DS_SIZE_ALIGNMENT,
@@ -682,15 +662,6 @@ static int mlx4_en_xmit(struct mlx4_en_priv *priv, int tx_ind, struct mbuf **mbp
/* check if TX ring is full */
if (unlikely(mlx4_en_tx_ring_is_full(ring))) {
- /* every full native Tx ring stops queue */
- if (ring->blocked == 0)
- atomic_add_int(&priv->blocked, 1);
- /* Set HW-queue-is-full flag */
- atomic_set_int(&ifp->if_drv_flags, IFF_DRV_OACTIVE);
- priv->port_stats.queue_stopped++;
- ring->blocked = 1;
- ring->queue_stopped++;
-
/* Use interrupts to find out when queue opened */
mlx4_en_arm_cq(priv, priv->tx_cq[tx_ind]);
return (ENOBUFS);
@@ -956,73 +927,28 @@ tx_drop:
}
static int
-mlx4_en_transmit_locked(struct ifnet *dev, int tx_ind, struct mbuf *m)
+mlx4_en_transmit_locked(struct ifnet *ifp, int tx_ind, struct mbuf *mb)
{
- struct mlx4_en_priv *priv = netdev_priv(dev);
- struct mlx4_en_tx_ring *ring;
- struct mbuf *next;
- int enqueued, err = 0;
-
- ring = priv->tx_ring[tx_ind];
- if ((dev->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
- IFF_DRV_RUNNING || priv->port_up == 0) {
- if (m != NULL)
- err = drbr_enqueue(dev, ring->br, m);
- return (err);
- }
+ struct mlx4_en_priv *priv = netdev_priv(ifp);
+ struct mlx4_en_tx_ring *ring = priv->tx_ring[tx_ind];
+ int err = 0;
- enqueued = 0;
- if (m != NULL)
- /*
- * If we can't insert mbuf into drbr, try to xmit anyway.
- * We keep the error we got so we could return that after xmit.
- */
- err = drbr_enqueue(dev, ring->br, m);
-
- /* Process the queue */
- while ((next = drbr_peek(dev, ring->br)) != NULL) {
- if (mlx4_en_xmit(priv, tx_ind, &next) != 0) {
- if (next == NULL) {
- drbr_advance(dev, ring->br);
- } else {
- drbr_putback(dev, ring->br, next);
- }
- break;
- }
- drbr_advance(dev, ring->br);
- enqueued++;
- if ((dev->if_drv_flags & IFF_DRV_RUNNING) == 0)
- break;
+ if (unlikely((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
+ READ_ONCE(priv->port_up) == 0)) {
+ m_freem(mb);
+ return (ENETDOWN);
}
- if (enqueued > 0)
- ring->watchdog_time = ticks;
-
- return (err);
-}
-
-void
-mlx4_en_tx_que(void *context, int pending)
-{
- struct mlx4_en_tx_ring *ring;
- struct mlx4_en_priv *priv;
- struct net_device *dev;
- struct mlx4_en_cq *cq;
- int tx_ind;
- cq = context;
- dev = cq->dev;
- priv = dev->if_softc;
- tx_ind = cq->ring;
- ring = priv->tx_ring[tx_ind];
-
- if (priv->port_up != 0 &&
- (dev->if_drv_flags & IFF_DRV_RUNNING) != 0) {
- mlx4_en_xmit_poll(priv, tx_ind);
- spin_lock(&ring->tx_lock);
- if (!drbr_empty(dev, ring->br))
- mlx4_en_transmit_locked(dev, tx_ind, NULL);
- spin_unlock(&ring->tx_lock);
+ if (mlx4_en_xmit(priv, tx_ind, &mb) != 0) {
+ /* NOTE: m_freem() is NULL safe */
+ m_freem(mb);
+ err = ENOBUFS;
+ if (ring->watchdog_time == 0)
+ ring->watchdog_time = ticks + MLX4_EN_WATCHDOG_TIMEOUT;
+ } else {
+ ring->watchdog_time = 0;
}
+ return (err);
}
int
@@ -1030,7 +956,6 @@ mlx4_en_transmit(struct ifnet *dev, struct mbuf *m)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_en_tx_ring *ring;
- struct mlx4_en_cq *cq;
int i, err = 0;
if (priv->port_up == 0) {
@@ -1047,16 +972,14 @@ mlx4_en_transmit(struct ifnet *dev, struct mbuf *m)
}
ring = priv->tx_ring[i];
- if (spin_trylock(&ring->tx_lock)) {
- err = mlx4_en_transmit_locked(dev, i, m);
- spin_unlock(&ring->tx_lock);
- /* Poll CQ here */
- mlx4_en_xmit_poll(priv, i);
- } else {
- err = drbr_enqueue(dev, ring->br, m);
- cq = priv->tx_cq[i];
- taskqueue_enqueue(cq->tq, &cq->cq_task);
- }
+
+ spin_lock(&ring->tx_lock);
+
+ err = mlx4_en_transmit_locked(dev, i, m);
+ spin_unlock(&ring->tx_lock);
+
+ /* Poll CQ here */
+ mlx4_en_xmit_poll(priv, i);
#if __FreeBSD_version >= 1100000
if (unlikely(err != 0))
@@ -1072,18 +995,9 @@ void
mlx4_en_qflush(struct ifnet *dev)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
- struct mlx4_en_tx_ring *ring;
- struct mbuf *m;
if (priv->port_up == 0)
return;
- for (int i = 0; i < priv->tx_ring_num; i++) {
- ring = priv->tx_ring[i];
- spin_lock(&ring->tx_lock);
- while ((m = buf_ring_dequeue_sc(ring->br)) != NULL)
- m_freem(m);
- spin_unlock(&ring->tx_lock);
- }
if_qflush(dev);
}