mlx5en: Patch to inhibit transmit doorbell writes during packet reception.

During packet reception the network stack frequently transmit data in
response to TCP window updates. To reduce the number of transmit doorbells
needed, inhibit all transmit doorbells designated for the same channel until
after the reception of packets for the given channel is completed.

While at it slightly refactor the mlx5e_tx_notify_hw() function:

1) The doorbell information is always stored into sq->doorbell.d64 .
No need to pass a separate pointer to this variable.

2) Move checks for skipping doorbell writes inside this function.

MFC after:	1 week
Sponsored by:	NVIDIA Networking

4 files changed, 32 insertions, 20 deletions

diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h
index af6cd8371506..4eb771d5c5c6 100644
--- a/sys/dev/mlx5/mlx5_en/en.h
+++ b/sys/dev/mlx5/mlx5_en/en.h
@@ -797,6 +797,7 @@ struct mlx5e_sq {
 	struct	mtx comp_lock;
 	struct	mlx5e_sq_stats stats;
 	struct	callout cev_callout;
+	int	db_inhibit;
 
 	/* data path */
 #define	mlx5e_sq_zero_start dma_tag
@@ -1154,8 +1155,13 @@ int	mlx5e_add_all_vxlan_rules(struct mlx5e_priv *priv);
 void	mlx5e_del_all_vxlan_rules(struct mlx5e_priv *priv);
 
 static inline void
-mlx5e_tx_notify_hw(struct mlx5e_sq *sq, u32 *wqe)
+mlx5e_tx_notify_hw(struct mlx5e_sq *sq, bool force)
 {
+	if (unlikely((force == false && sq->db_inhibit != 0) || sq->doorbell.d64 == 0)) {
+		/* skip writing the doorbell record */
+		return;
+	}
+
 	/* ensure wqe is visible to device before updating doorbell record */
 	wmb();
 
@@ -1167,8 +1173,10 @@ mlx5e_tx_notify_hw(struct mlx5e_sq *sq, u32 *wqe)
 	 */
 	wmb();
 
-	mlx5_write64(wqe, sq->uar_map,
+	mlx5_write64(sq->doorbell.d32, sq->uar_map,
 	    MLX5_GET_DOORBELL_LOCK(&sq->priv->doorbell_lock));
+
+	sq->doorbell.d64 = 0;
 }
 
 static inline void
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
index e8d9d26efc6f..6eb266f37761 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
@@ -1807,11 +1807,7 @@ mlx5e_sq_send_nops_locked(struct mlx5e_sq *sq, int can_sleep)
 		atomic_thread_fence_rel();
 	}
 done:
-	/* Check if we need to write the doorbell */
-	if (likely(sq->doorbell.d64 != 0)) {
-		mlx5e_tx_notify_hw(sq, sq->doorbell.d32);
-		sq->doorbell.d64 = 0;
-	}
+	mlx5e_tx_notify_hw(sq, false);
 }
 
 void
@@ -3867,8 +3863,7 @@ mlx5e_reset_sq_doorbell_record(struct mlx5e_sq *sq)
 
 	sq->doorbell.d32[0] = cpu_to_be32(MLX5_OPCODE_NOP);
 	sq->doorbell.d32[1] = cpu_to_be32(sq->sqn << 8);
-	mlx5e_tx_notify_hw(sq, sq->doorbell.d32);
-	sq->doorbell.d64 = 0;
+	mlx5e_tx_notify_hw(sq, true);
 }
 
 void
@@ -4785,10 +4780,8 @@ mlx5_en_debugnet_transmit(struct ifnet *dev, struct mbuf *m)
 		err = 0;
 	}
 
-	if (likely(sq->doorbell.d64 != 0)) {
-		mlx5e_tx_notify_hw(sq, sq->doorbell.d32);
-		sq->doorbell.d64 = 0;
-	}
+	mlx5e_tx_notify_hw(sq, true);
+
 	return (err);
 }
 
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
index e943a65709cf..ec6b027b324e 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -566,6 +566,7 @@ wq_ll_pop:
 void
 mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused)
 {
+	struct mlx5e_channel *c = container_of(mcq, struct mlx5e_channel, rq.cq.mcq);
 	struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq);
 	int i = 0;
 
@@ -584,6 +585,11 @@ mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused)
 		rq->ifp->if_input(rq->ifp, mb);
 	}
 #endif
+	for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) {
+		mtx_lock(&c->sq[j].lock);
+		c->sq[j].db_inhibit++;
+		mtx_unlock(&c->sq[j].lock);
+	}
 
 	mtx_lock(&rq->mtx);
 
@@ -607,4 +613,12 @@ mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused)
 	mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK(&rq->channel->priv->doorbell_lock));
 	tcp_lro_flush_all(&rq->lro);
 	mtx_unlock(&rq->mtx);
+
+	for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) {
+		mtx_lock(&c->sq[j].lock);
+		c->sq[j].db_inhibit--;
+		/* Update the doorbell record, if any. */
+		mlx5e_tx_notify_hw(c->sq + j, true);
+		mtx_unlock(&c->sq[j].lock);
+	}
 }
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
index 7bb12d7b5b8a..1bf3a857d1e8 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015-2019 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -1122,11 +1122,8 @@ mlx5e_xmit_locked(struct ifnet *ifp, struct mlx5e_sq *sq, struct mbuf *mb)
 		err = ENOBUFS;
 	}
 
-	/* Check if we need to write the doorbell */
-	if (likely(sq->doorbell.d64 != 0)) {
-		mlx5e_tx_notify_hw(sq, sq->doorbell.d32);
-		sq->doorbell.d64 = 0;
-	}
+	/* Write the doorbell record, if any. */
+	mlx5e_tx_notify_hw(sq, false);
 
 	/*
 	 * Check if we need to start the event timer which flushes the