diff options
| -rw-r--r-- | sys/dev/mlx5/mlx5_en/en.h | 12 | ||||
| -rw-r--r-- | sys/dev/mlx5/mlx5_en/mlx5_en_main.c | 15 | ||||
| -rw-r--r-- | sys/dev/mlx5/mlx5_en/mlx5_en_rx.c | 16 | ||||
| -rw-r--r-- | sys/dev/mlx5/mlx5_en/mlx5_en_tx.c | 9 |
4 files changed, 32 insertions, 20 deletions
diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h index af6cd8371506..4eb771d5c5c6 100644 --- a/sys/dev/mlx5/mlx5_en/en.h +++ b/sys/dev/mlx5/mlx5_en/en.h @@ -797,6 +797,7 @@ struct mlx5e_sq { struct mtx comp_lock; struct mlx5e_sq_stats stats; struct callout cev_callout; + int db_inhibit; /* data path */ #define mlx5e_sq_zero_start dma_tag @@ -1154,8 +1155,13 @@ int mlx5e_add_all_vxlan_rules(struct mlx5e_priv *priv); void mlx5e_del_all_vxlan_rules(struct mlx5e_priv *priv); static inline void -mlx5e_tx_notify_hw(struct mlx5e_sq *sq, u32 *wqe) +mlx5e_tx_notify_hw(struct mlx5e_sq *sq, bool force) { + if (unlikely((force == false && sq->db_inhibit != 0) || sq->doorbell.d64 == 0)) { + /* skip writing the doorbell record */ + return; + } + /* ensure wqe is visible to device before updating doorbell record */ wmb(); @@ -1167,8 +1173,10 @@ mlx5e_tx_notify_hw(struct mlx5e_sq *sq, u32 *wqe) */ wmb(); - mlx5_write64(wqe, sq->uar_map, + mlx5_write64(sq->doorbell.d32, sq->uar_map, MLX5_GET_DOORBELL_LOCK(&sq->priv->doorbell_lock)); + + sq->doorbell.d64 = 0; } static inline void diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c index e8d9d26efc6f..6eb266f37761 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c @@ -1807,11 +1807,7 @@ mlx5e_sq_send_nops_locked(struct mlx5e_sq *sq, int can_sleep) atomic_thread_fence_rel(); } done: - /* Check if we need to write the doorbell */ - if (likely(sq->doorbell.d64 != 0)) { - mlx5e_tx_notify_hw(sq, sq->doorbell.d32); - sq->doorbell.d64 = 0; - } + mlx5e_tx_notify_hw(sq, false); } void @@ -3867,8 +3863,7 @@ mlx5e_reset_sq_doorbell_record(struct mlx5e_sq *sq) sq->doorbell.d32[0] = cpu_to_be32(MLX5_OPCODE_NOP); sq->doorbell.d32[1] = cpu_to_be32(sq->sqn << 8); - mlx5e_tx_notify_hw(sq, sq->doorbell.d32); - sq->doorbell.d64 = 0; + mlx5e_tx_notify_hw(sq, true); } void @@ -4785,10 +4780,8 @@ mlx5_en_debugnet_transmit(struct ifnet *dev, struct mbuf *m) err = 0; } - if (likely(sq->doorbell.d64 != 0)) { - mlx5e_tx_notify_hw(sq, sq->doorbell.d32); - sq->doorbell.d64 = 0; - } + mlx5e_tx_notify_hw(sq, true); + return (err); } diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c index e943a65709cf..ec6b027b324e 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015 Mellanox Technologies. All rights reserved. + * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -566,6 +566,7 @@ wq_ll_pop: void mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused) { + struct mlx5e_channel *c = container_of(mcq, struct mlx5e_channel, rq.cq.mcq); struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq); int i = 0; @@ -584,6 +585,11 @@ mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused) rq->ifp->if_input(rq->ifp, mb); } #endif + for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) { + mtx_lock(&c->sq[j].lock); + c->sq[j].db_inhibit++; + mtx_unlock(&c->sq[j].lock); + } mtx_lock(&rq->mtx); @@ -607,4 +613,12 @@ mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused) mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK(&rq->channel->priv->doorbell_lock)); tcp_lro_flush_all(&rq->lro); mtx_unlock(&rq->mtx); + + for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) { + mtx_lock(&c->sq[j].lock); + c->sq[j].db_inhibit--; + /* Update the doorbell record, if any. */ + mlx5e_tx_notify_hw(c->sq + j, true); + mtx_unlock(&c->sq[j].lock); + } } diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c index 7bb12d7b5b8a..1bf3a857d1e8 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015-2019 Mellanox Technologies. All rights reserved. + * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -1122,11 +1122,8 @@ mlx5e_xmit_locked(struct ifnet *ifp, struct mlx5e_sq *sq, struct mbuf *mb) err = ENOBUFS; } - /* Check if we need to write the doorbell */ - if (likely(sq->doorbell.d64 != 0)) { - mlx5e_tx_notify_hw(sq, sq->doorbell.d32); - sq->doorbell.d64 = 0; - } + /* Write the doorbell record, if any. */ + mlx5e_tx_notify_hw(sq, false); /* * Check if we need to start the event timer which flushes the |