mlx5en: Implement TLS RX support.

TLS RX support is modeled after TLS TX support. The basic structures and layouts
are almost identical, except that the send tag created filters RX traffic and
not TX traffic.

The TLS RX tag keeps track of past TLS records up to a certain limit,
approximately 1 Gbyte of TCP data. TLS records of same length are joined
into a single database record.

Regularly the HW is queried for TLS RX progress information. The TCP sequence
number gotten from the HW is then matches against the database of TLS TCP
sequence number records and lengths. If a match is found a static params WQE
is queued on the IQ and the hardware should immediately resume decrypting TLS
data until the next non-sequential TCP packet arrives.

Offloading TLS RX data is supported for untagged, prio-tagged, and
regular VLAN traffic.

MFC after:	1 week
Sponsored by:	NVIDIA Networking

9 files changed, 1228 insertions, 3 deletions

diff --git a/sys/conf/files b/sys/conf/files
index 5f452f851ea6..0278d5fa4083 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -4929,6 +4929,8 @@ dev/mlx5/mlx5_en/mlx5_en_flow_table.c		optional mlx5en pci inet inet6	\
 	compile-with "${OFED_C}"
 dev/mlx5/mlx5_en/mlx5_en_hw_tls.c		optional mlx5en pci inet inet6	\
 	compile-with "${OFED_C}"
+dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c		optional mlx5en pci inet inet6	\
+	compile-with "${OFED_C}"
 dev/mlx5/mlx5_en/mlx5_en_iq.c			optional mlx5en pci inet inet6	\
 	compile-with "${OFED_C}"
 dev/mlx5/mlx5_en/mlx5_en_rx.c			optional mlx5en pci inet inet6	\
diff --git a/sys/dev/mlx5/device.h b/sys/dev/mlx5/device.h
index f183ca74c2d7..e2868c62337f 100644
--- a/sys/dev/mlx5/device.h
+++ b/sys/dev/mlx5/device.h
@@ -408,6 +408,14 @@ enum {
 	MLX5_OPCODE_MOD_PSV_TLS_TIR_PROGRESS_PARAMS = 0x2,
 };
 
+struct mlx5_wqe_tls_static_params_seg {
+	u8     ctx[MLX5_ST_SZ_BYTES(tls_static_params)];
+};
+
+struct mlx5_wqe_tls_progress_params_seg {
+	u8     ctx[MLX5_ST_SZ_BYTES(tls_progress_params)];
+} __aligned(64);
+
 enum {
 	MLX5_SET_PORT_RESET_QKEY	= 0,
 	MLX5_SET_PORT_GUID0		= 16,
@@ -750,6 +758,11 @@ static inline bool cqe_is_tunneled(struct mlx5_cqe64 *cqe)
 	return cqe->tls_outer_l3_tunneled & 0x1;
 }
 
+static inline u8 get_cqe_tls_offload(struct mlx5_cqe64 *cqe)
+{
+	return (cqe->tls_outer_l3_tunneled >> 3) & 0x3;
+}
+
 enum {
 	CQE_L4_HDR_TYPE_NONE			= 0x0,
 	CQE_L4_HDR_TYPE_TCP_NO_ACK		= 0x1,
@@ -794,6 +807,13 @@ enum {
 	CQE_L4_OK	= 1 << 2,
 };
 
+enum {
+	CQE_TLS_OFFLOAD_NOT_DECRYPTED		= 0x0,
+	CQE_TLS_OFFLOAD_DECRYPTED		= 0x1,
+	CQE_TLS_OFFLOAD_RESYNC			= 0x2,
+	CQE_TLS_OFFLOAD_ERROR			= 0x3,
+};
+
 struct mlx5_sig_err_cqe {
 	u8		rsvd0[16];
 	__be32		expected_trans_sig;
diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h
index 50215defdf60..1c8a53b1ba4d 100644
--- a/sys/dev/mlx5/mlx5_en/en.h
+++ b/sys/dev/mlx5/mlx5_en/en.h
@@ -207,7 +207,9 @@ typedef void (mlx5e_cq_comp_t)(struct mlx5_core_cq *, struct mlx5_eqe *);
   m(+1, u64, tx_defragged, "tx_defragged", "Transmit queue defragged") \
   m(+1, u64, rx_wqe_err, "rx_wqe_err", "Receive WQE errors") \
   m(+1, u64, tx_jumbo_packets, "tx_jumbo_packets", "TX packets greater than 1518 octets") \
-  m(+1, u64, rx_steer_missed_packets, "rx_steer_missed_packets", "RX packets dropped by steering rule(s)")
+  m(+1, u64, rx_steer_missed_packets, "rx_steer_missed_packets", "RX packets dropped by steering rule(s)") \
+  m(+1, u64, rx_decrypted_ok_packets, "rx_decrypted_ok_packets", "RX packets successfully decrypted by steering rule(s)") \
+  m(+1, u64, rx_decrypted_error_packets, "rx_decrypted_error_packets", "RX packets not decrypted by steering rule(s)")
 
 #define	MLX5E_VPORT_STATS_NUM (0 MLX5E_VPORT_STATS(MLX5E_STATS_COUNT))
 
@@ -608,7 +610,9 @@ struct mlx5e_port_stats_debug {
   m(+1, u64, lro_bytes, "lro_bytes", "Received LRO bytes")	\
   m(+1, u64, sw_lro_queued, "sw_lro_queued", "Packets queued for SW LRO")	\
   m(+1, u64, sw_lro_flushed, "sw_lro_flushed", "Packets flushed from SW LRO")	\
-  m(+1, u64, wqe_err, "wqe_err", "Received packets")
+  m(+1, u64, wqe_err, "wqe_err", "Received packets") \
+  m(+1, u64, decrypted_ok_packets, "decrypted_ok_packets", "Received packets successfully decrypted by steering rule(s)") \
+  m(+1, u64, decrypted_error_packets, "decrypted_error_packets", "Received packets not decrypted by steering rule(s)")
 
 #define	MLX5E_RQ_STATS_NUM (0 MLX5E_RQ_STATS(MLX5E_STATS_COUNT))
 
@@ -1050,6 +1054,7 @@ struct mlx5e_xmit_args {
 
 #include <dev/mlx5/mlx5_en/en_rl.h>
 #include <dev/mlx5/mlx5_en/en_hw_tls.h>
+#include <dev/mlx5/mlx5_en/en_hw_tls_rx.h>
 
 #define	MLX5E_TSTMP_PREC 10
 
@@ -1131,6 +1136,7 @@ struct mlx5e_priv {
 	struct mlx5e_rl_priv_data rl;
 
 	struct mlx5e_tls tls;
+	struct mlx5e_tls_rx tls_rx;
 
 	struct callout tstmp_clbr;
 	int	clbr_done;
diff --git a/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h b/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h
new file mode 100644
index 000000000000..6954b97b827b
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h
@@ -0,0 +1,149 @@
+/*-
+ * Copyright (c) 2021 NVIDIA corporation & affiliates.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MLX5_TLS_RX_H_
+#define	_MLX5_TLS_RX_H_
+
+#include <linux/completion.h>
+
+#define	MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE 128
+
+#define	MLX5E_TLS_RX_RESYNC_MAX 32	/* units */
+#define	MLX5E_TLS_RX_NUM_MAX (1U << 11)	/* packets */
+
+#define	MLX5E_TLS_RX_TAG_LOCK(tag)	mtx_lock(&(tag)->mtx)
+#define	MLX5E_TLS_RX_TAG_UNLOCK(tag)	mtx_unlock(&(tag)->mtx)
+
+#define	MLX5E_TLS_RX_STAT_INC(tag, field, num) \
+	counter_u64_add((tag)->tls_rx->stats.field, num)
+
+#if ((MLX5E_TLS_RX_RESYNC_MAX * MLX5E_TLS_RX_NUM_MAX) << 14) > (1U << 30)
+#error "Please lower the limits of the TLS record length database."
+#endif
+
+enum {
+	MLX5E_TLS_RX_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD = 0,
+	MLX5E_TLS_RX_PROGRESS_PARAMS_AUTH_STATE_OFFLOAD = 1,
+	MLX5E_TLS_RX_PROGRESS_PARAMS_AUTH_STATE_AUTHENTICATION = 2,
+};
+
+enum {
+	MLX5E_TLS_RX_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START = 0,
+	MLX5E_TLS_RX_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING = 1,
+	MLX5E_TLS_RX_PROGRESS_PARAMS_RECORD_TRACKER_STATE_SEARCHING = 2,
+};
+
+struct mlx5e_tls_rx;
+struct mlx5e_tls_rx_tag {
+	struct m_snd_tag tag;
+	volatile s32 refs;	/* number of pending mbufs */
+	uint32_t tirn;		/* HW TIR context number */
+	uint32_t dek_index;	/* HW TLS context number */
+	struct mlx5e_tls_rx *tls_rx; /* parent pointer */
+	struct mlx5_flow_rule *flow_rule;
+	struct mtx mtx;
+	struct completion progress_complete;
+	uint32_t state;	/* see MLX5E_TLS_RX_ST_XXX */
+#define	MLX5E_TLS_RX_ST_INIT 0
+#define	MLX5E_TLS_RX_ST_SETUP 1
+#define	MLX5E_TLS_RX_ST_READY 2
+#define	MLX5E_TLS_RX_ST_FREED 3
+
+	/*
+	 * The following fields are used to store the TCP starting
+	 * point of TLS records in the past. When TLS records of same
+	 * length are back to back the tcp_resync_num[] is incremented
+	 * instead of creating new entries. This way up to
+	 * "MLX5E_TLS_RX_RESYNC_MAX" * "MLX5E_TLS_RX_NUM_MAX" * 16
+	 * KBytes, around 1GByte worth of TCP data, may be remembered
+	 * in the good case. The amount of history should not exceed
+	 * 2GBytes of TCP data, because then the TCP sequence numbers
+	 * may wrap around.
+	 *
+	 * This information is used to tell if a given TCP sequence
+	 * number is a valid TLS record or not.
+	 */
+	uint64_t rcd_resync_start;	/* starting TLS record number */
+	uint32_t tcp_resync_start;	/* starting TCP sequence number */
+	uint32_t tcp_resync_next;	/* next expected TCP sequence number */
+	uint32_t tcp_resync_len[MLX5E_TLS_RX_RESYNC_MAX];
+	uint32_t tcp_resync_num[MLX5E_TLS_RX_RESYNC_MAX];
+	uint16_t tcp_resync_pc;		/* producer counter for arrays above */
+	uint16_t tcp_resync_cc;		/* consumer counter for arrays above */
+
+	struct work_struct work;
+
+	uint32_t flowid;
+	uint32_t flowtype;
+	uint32_t dek_index_ok:1;
+	uint32_t tcp_resync_active:1;
+	uint32_t tcp_resync_pending:1;
+
+	/* parameters needed */
+	uint8_t crypto_params[128] __aligned(4);
+	uint8_t rx_progress[MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE * 2];
+} __aligned(MLX5E_CACHELINE_SIZE);
+
+static inline void *
+mlx5e_tls_rx_get_progress_buffer(struct mlx5e_tls_rx_tag *ptag)
+{
+	/* return properly aligned RX buffer */
+	return (ptag->rx_progress +
+	    ((-(uintptr_t)ptag->rx_progress) &
+	    (MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE - 1)));
+}
+
+#define	MLX5E_TLS_RX_STATS(m) \
+  m(+1, u64, rx_resync_ok, "rx_resync_ok", "Successful resync requests")\
+  m(+1, u64, rx_resync_err, "rx_resync_err", "Failed resync requests")\
+  m(+1, u64, rx_error, "rx_error", "Other errors")
+
+#define	MLX5E_TLS_RX_STATS_NUM (0 MLX5E_TLS_RX_STATS(MLX5E_STATS_COUNT))
+
+struct mlx5e_tls_rx_stats {
+	struct	sysctl_ctx_list ctx;
+	counter_u64_t	arg[0];
+	MLX5E_TLS_RX_STATS(MLX5E_STATS_COUNTER)
+};
+
+struct mlx5e_tls_rx {
+	struct sysctl_ctx_list ctx;
+	struct mlx5e_tls_rx_stats stats;
+	struct workqueue_struct *wq;
+	uma_zone_t zone;
+	uint32_t max_resources;		/* max number of resources */
+	volatile uint32_t num_resources;	/* current number of resources */
+	int init;			/* set when ready */
+	char zname[32];
+};
+
+int mlx5e_tls_rx_init(struct mlx5e_priv *);
+void mlx5e_tls_rx_cleanup(struct mlx5e_priv *);
+
+if_snd_tag_alloc_t mlx5e_tls_rx_snd_tag_alloc;
+
+#endif		/* _MLX5_TLS_RX_H_ */
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c
new file mode 100644
index 000000000000..5ac60c51a1db
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c
@@ -0,0 +1,1008 @@
+/*-
+ * Copyright (c) 2021 NVIDIA corporation & affiliates.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "opt_kern_tls.h"
+#include "opt_rss.h"
+#include "opt_ratelimit.h"
+
+#include <dev/mlx5/mlx5_en/en.h>
+
+#include <dev/mlx5/tls.h>
+
+#include <dev/mlx5/fs.h>
+#include <dev/mlx5/mlx5_core/fs_tcp.h>
+
+#include <sys/ktls.h>
+#include <opencrypto/cryptodev.h>
+
+#ifdef KERN_TLS
+
+static if_snd_tag_free_t mlx5e_tls_rx_snd_tag_free;
+static if_snd_tag_modify_t mlx5e_tls_rx_snd_tag_modify;
+
+static const struct if_snd_tag_sw mlx5e_tls_rx_snd_tag_sw = {
+	.snd_tag_modify = mlx5e_tls_rx_snd_tag_modify,
+	.snd_tag_free = mlx5e_tls_rx_snd_tag_free,
+	.type = IF_SND_TAG_TYPE_TLS_RX
+};
+
+MALLOC_DEFINE(M_MLX5E_TLS_RX, "MLX5E_TLS_RX", "MLX5 ethernet HW TLS RX");
+
+/* software TLS RX context */
+struct mlx5_ifc_sw_tls_rx_cntx_bits {
+	struct mlx5_ifc_tls_static_params_bits param;
+	struct mlx5_ifc_tls_progress_params_bits progress;
+	struct {
+		uint8_t key_data[8][0x20];
+		uint8_t key_len[0x20];
+	} key;
+};
+
+CTASSERT(MLX5_ST_SZ_BYTES(sw_tls_rx_cntx) <= sizeof(((struct mlx5e_tls_rx_tag *)NULL)->crypto_params));
+CTASSERT(MLX5_ST_SZ_BYTES(mkc) == sizeof(((struct mlx5e_tx_umr_wqe *)NULL)->mkc));
+
+static const char *mlx5e_tls_rx_stats_desc[] = {
+	MLX5E_TLS_RX_STATS(MLX5E_STATS_DESC)
+};
+
+static void mlx5e_tls_rx_work(struct work_struct *);
+static bool mlx5e_tls_rx_snd_tag_find_tcp_sn_and_tls_rcd(struct mlx5e_tls_rx_tag *,
+    uint32_t, uint32_t *, uint64_t *);
+
+CTASSERT((MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param) % 16) == 0);
+
+static uint32_t
+mlx5e_tls_rx_get_ch(struct mlx5e_priv *priv, uint32_t flowid, uint32_t flowtype)
+{
+	u32 ch;
+#ifdef RSS
+	u32 temp;
+#endif
+
+	/* keep this code synced with mlx5e_select_queue() */
+	ch = priv->params.num_channels;
+#ifdef RSS
+	if (rss_hash2bucket(flowid, flowtype, &temp) == 0)
+		ch = temp % ch;
+	else
+#endif
+		ch = (flowid % 128) % ch;
+	return (ch);
+}
+
+/*
+ * This function gets a pointer to an internal queue, IQ, based on the
+ * provided "flowid" and "flowtype". The IQ returned may in some rare
+ * cases not be activated or running, but this is all handled by the
+ * "mlx5e_iq_get_producer_index()" function.
+ *
+ * The idea behind this function is to spread the IQ traffic as much
+ * as possible and to avoid congestion on the same IQ when processing
+ * RX traffic.
+ */
+static struct mlx5e_iq *
+mlx5e_tls_rx_get_iq(struct mlx5e_priv *priv, uint32_t flowid, uint32_t flowtype)
+{
+	/*
+	 * NOTE: The channels array is only freed at detach
+	 * and it safe to return a pointer to the send tag
+	 * inside the channels structure as long as we
+	 * reference the priv.
+	 */
+	return (&priv->channel[mlx5e_tls_rx_get_ch(priv, flowid, flowtype)].iq);
+}
+
+/*
+ * This function sends the so-called TLS RX static parameters to the
+ * hardware. These parameters are temporarily stored in the
+ * "crypto_params" field of the TLS RX tag.  Most importantly this
+ * function sets the TCP sequence number (32-bit) and TLS record
+ * number (64-bit) where the decryption can resume.
+ *
+ * Zero is returned upon success. Else some error happend.
+ */
+static int
+mlx5e_tls_rx_send_static_parameters(struct mlx5e_iq *iq, struct mlx5e_tls_rx_tag *ptag)
+{
+	const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_umr_wqe) +
+	    MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param), MLX5_SEND_WQE_DS);
+	struct mlx5e_tx_umr_wqe *wqe;
+	int pi;
+
+	mtx_lock(&iq->lock);
+	pi = mlx5e_iq_get_producer_index(iq);
+	if (pi < 0) {
+		mtx_unlock(&iq->lock);
+		return (-ENOMEM);
+	}
+	wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi);
+
+	memset(wqe, 0, sizeof(*wqe));
+
+	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) |
+	    MLX5_OPCODE_UMR | (MLX5_OPCODE_MOD_UMR_TLS_TIR_STATIC_PARAMS << 24));
+	wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt);
+	wqe->ctrl.imm = cpu_to_be32(ptag->tirn << 8);
+	wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
+
+	/* fill out UMR control segment */
+	wqe->umr.flags = 0x80;	/* inline data */
+	wqe->umr.bsf_octowords =
+	    cpu_to_be16(MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param) / 16);
+
+	/* copy in the static crypto parameters */
+	memcpy(wqe + 1, MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, param),
+	    MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param));
+
+	/* copy data for doorbell */
+	memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32));
+
+	iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
+
+	iq->data[pi].p_refcount = &ptag->refs;
+	atomic_add_int(&ptag->refs, 1);
+	iq->pc += iq->data[pi].num_wqebbs;
+
+	mlx5e_iq_notify_hw(iq);
+
+	mtx_unlock(&iq->lock);
+
+	return (0);	/* success */
+}
+
+static void
+mlx5e_tls_rx_send_progress_parameters_cb(void *arg)
+{
+	struct mlx5e_tls_rx_tag *ptag;
+
+	ptag = (struct mlx5e_tls_rx_tag *)arg;
+
+	complete(&ptag->progress_complete);
+}
+
+CTASSERT(MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, progress) ==
+    sizeof(((struct mlx5e_tx_psv_wqe *)NULL)->psv));
+
+/*
+ * This function resets the state of the TIR context to start
+ * searching for a valid TLS header and is used only when allocating
+ * the TLS RX tag.
+ *
+ * Zero is returned upon success, else some error happened.
+ */
+static int
+mlx5e_tls_rx_send_progress_parameters_sync(struct mlx5e_iq *iq,
+    struct mlx5e_tls_rx_tag *ptag)
+{
+	const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_psv_wqe),
+	    MLX5_SEND_WQE_DS);
+	struct mlx5e_priv *priv;
+	struct mlx5e_tx_psv_wqe *wqe;
+	int pi;
+
+	mtx_lock(&iq->lock);
+	pi = mlx5e_iq_get_producer_index(iq);
+	if (pi < 0) {
+		mtx_unlock(&iq->lock);
+		return (-ENOMEM);
+	}
+	wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi);
+
+	memset(wqe, 0, sizeof(*wqe));
+
+	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) |
+	    MLX5_OPCODE_SET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIR_PROGRESS_PARAMS << 24));
+	wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt);
+	wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
+
+	/* copy in the PSV control segment */
+	memcpy(&wqe->psv, MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, progress),
+	    sizeof(wqe->psv));
+
+	/* copy data for doorbell */
+	memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32));
+
+	iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
+	iq->data[pi].callback = &mlx5e_tls_rx_send_progress_parameters_cb;
+	iq->data[pi].arg = ptag;
+	iq->data[pi].p_refcount = &ptag->refs;
+	atomic_add_int(&ptag->refs, 1);
+	iq->pc += iq->data[pi].num_wqebbs;
+
+	init_completion(&ptag->progress_complete);
+
+	mlx5e_iq_notify_hw(iq);
+
+	mtx_unlock(&iq->lock);
+
+	while (1) {
+		if (wait_for_completion_timeout(&ptag->progress_complete, hz) != 0)
+			break;
+		priv = container_of(iq, struct mlx5e_channel, iq)->priv;
+		if (priv->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR ||
+		    pci_channel_offline(priv->mdev->pdev) != 0)
+			return (-EWOULDBLOCK);
+	}
+
+	return (0);	/* success */
+}
+
+CTASSERT(MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE >= MLX5_ST_SZ_BYTES(tls_progress_params));
+CTASSERT(MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE <= PAGE_SIZE);
+
+struct mlx5e_get_tls_progress_params_wqe {
+	struct mlx5_wqe_ctrl_seg ctrl;
+	struct mlx5_seg_get_psv	 psv;
+};
+
+static void
+mlx5e_tls_rx_receive_progress_parameters_cb(void *arg)
+{
+	struct mlx5e_tls_rx_tag *ptag;
+	struct mlx5e_iq *iq;
+	uint32_t tcp_curr_sn_he;
+	uint32_t tcp_next_sn_he;
+	uint64_t tls_rcd_num;
+	void *buffer;
+
+	ptag = (struct mlx5e_tls_rx_tag *)arg;
+	buffer = mlx5e_tls_rx_get_progress_buffer(ptag);
+
+	MLX5E_TLS_RX_TAG_LOCK(ptag);
+
+	ptag->tcp_resync_pending = 0;
+
+	switch (MLX5_GET(tls_progress_params, buffer, record_tracker_state)) {
+	case MLX5E_TLS_RX_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING:
+		break;
+	default:
+		goto done;
+	}
+
+	switch (MLX5_GET(tls_progress_params, buffer, auth_state)) {
+	case MLX5E_TLS_RX_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD:
+		break;
+	default:
+		goto done;
+	}
+
+	tcp_curr_sn_he = MLX5_GET(tls_progress_params, buffer, hw_resync_tcp_sn);
+
+	if (mlx5e_tls_rx_snd_tag_find_tcp_sn_and_tls_rcd(ptag, tcp_curr_sn_he,
+	    &tcp_next_sn_he, &tls_rcd_num)) {
+
+		MLX5_SET64(sw_tls_rx_cntx, ptag->crypto_params,
+		    param.initial_record_number, tls_rcd_num);
+		MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params,
+		    param.resync_tcp_sn, tcp_curr_sn_he);
+
+		iq = mlx5e_tls_rx_get_iq(
+		    container_of(ptag->tls_rx, struct mlx5e_priv, tls_rx),
+		    ptag->flowid, ptag->flowtype);
+
+		if (mlx5e_tls_rx_send_static_parameters(iq, ptag) != 0)
+			MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
+	}
+done:
+	MLX5E_TLS_RX_TAG_UNLOCK(ptag);
+}
+
+/*
+ * This function queries the hardware for the current state of the TIR
+ * in question. It is typically called when encrypted data is received
+ * to re-establish hardware decryption of received TLS data.
+ *
+ * Zero is returned upon success, else some error happened.
+ */
+static int
+mlx5e_tls_rx_receive_progress_parameters(struct mlx5e_iq *iq, struct mlx5e_tls_rx_tag *ptag)
+{
+	struct mlx5e_get_tls_progress_params_wqe *wqe;
+	const u32 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
+	u64 dma_address;
+	int pi;
+
+	mtx_lock(&iq->lock);
+	pi = mlx5e_iq_get_producer_index(iq);
+	if (pi < 0) {
+		mtx_unlock(&iq->lock);
+		return (-ENOMEM);
+	}
+
+	mlx5e_iq_load_memory_single(iq, pi,
+	    mlx5e_tls_rx_get_progress_buffer(ptag),
+	    MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE,
+	    &dma_address, BUS_DMASYNC_PREREAD);
+
+	wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi);
+
+	memset(wqe, 0, sizeof(*wqe));
+
+	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) |
+	    MLX5_OPCODE_GET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIR_PROGRESS_PARAMS << 24));
+	wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt);
+	wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
+	wqe->psv.num_psv = 1 << 4;
+	wqe->psv.l_key = iq->mkey_be;
+	wqe->psv.psv_index[0] = cpu_to_be32(ptag->tirn);
+	wqe->psv.va = cpu_to_be64(dma_address);
+
+	/* copy data for doorbell */
+	memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32));
+
+	iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
+	iq->data[pi].p_refcount = &ptag->refs;
+	iq->data[pi].callback = &mlx5e_tls_rx_receive_progress_parameters_cb;
+	iq->data[pi].arg = ptag;
+	atomic_add_int(&ptag->refs, 1);
+	iq->pc += iq->data[pi].num_wqebbs;
+
+	mlx5e_iq_notify_hw(iq);
+
+	mtx_unlock(&iq->lock);
+
+	return (0);	/* success */
+}
+
+/*
+ * This is the import function for TLS RX tags.
+ */
+static int
+mlx5e_tls_rx_tag_import(void *arg, void **store, int cnt, int domain, int flags)
+{
+	struct mlx5e_tls_rx_tag *ptag;
+	int i;
+
+	for (i = 0; i != cnt; i++) {
+		ptag = malloc_domainset(sizeof(*ptag), M_MLX5E_TLS_RX,
+		    mlx5_dev_domainset(arg), flags | M_ZERO);
+		mtx_init(&ptag->mtx, "mlx5-tls-rx-tag-mtx", NULL, MTX_DEF);
+		INIT_WORK(&ptag->work, mlx5e_tls_rx_work);
+		store[i] = ptag;
+	}
+	return (i);
+}
+
+/*
+ * This is the release function for TLS RX tags.
+ */
+static void
+mlx5e_tls_rx_tag_release(void *arg, void **store, int cnt)
+{
+	struct mlx5e_tls_rx_tag *ptag;
+	int i;
+
+	for (i = 0; i != cnt; i++) {
+		ptag = store[i];
+
+		flush_work(&ptag->work);
+		mtx_destroy(&ptag->mtx);
+		free(ptag, M_MLX5E_TLS_RX);
+	}
+}
+
+/*
+ * This is a convenience function to free TLS RX tags. It resets some
+ * selected fields, updates the number of resources and returns the
+ * TLS RX tag to the UMA pool of free tags.
+ */
+static void
+mlx5e_tls_rx_tag_zfree(struct mlx5e_tls_rx_tag *ptag)
+{
+	/* reset some variables */
+	ptag->state = MLX5E_TLS_RX_ST_INIT;
+	ptag->dek_index = 0;
+	ptag->dek_index_ok = 0;
+	ptag->tirn = 0;
+	ptag->flow_rule = NULL;
+	ptag->tcp_resync_active = 0;
+	ptag->tcp_resync_pending = 0;
+
+	/* avoid leaking keys */
+	memset(ptag->crypto_params, 0, sizeof(ptag->crypto_params));
+
+	/* update number of resources in use */
+	atomic_add_32(&ptag->tls_rx->num_resources, -1U);
+
+	/* return tag to UMA */
+	uma_zfree(ptag->tls_rx->zone, ptag);
+}
+
+/*
+ * This function enables TLS RX support for the given NIC, if all
+ * needed firmware capabilites are present.
+ */
+int
+mlx5e_tls_rx_init(struct mlx5e_priv *priv)
+{
+	struct mlx5e_tls_rx *ptls = &priv->tls_rx;
+	struct sysctl_oid *node;
+	uint32_t x;
+
+	if (MLX5_CAP_GEN(priv->mdev, tls_rx) == 0 ||
+	    MLX5_CAP_GEN(priv->mdev, log_max_dek) == 0 ||
+	    MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version) == 0)
+		return (0);
+
+	ptls->wq = create_singlethread_workqueue("mlx5-tls-rx-wq");
+	if (ptls->wq == NULL)
+		return (ENOMEM);
+
+	sysctl_ctx_init(&ptls->ctx);
+
+	snprintf(ptls->zname, sizeof(ptls->zname),
+	    "mlx5_%u_tls_rx", device_get_unit(priv->mdev->pdev->dev.bsddev));
+
+	ptls->zone = uma_zcache_create(ptls->zname,
+	    sizeof(struct mlx5e_tls_rx_tag), NULL, NULL, NULL, NULL,
+	    mlx5e_tls_rx_tag_import, mlx5e_tls_rx_tag_release, priv->mdev, 0);
+
+	/* shared between RX and TX TLS */
+	ptls->max_resources = 1U << (MLX5_CAP_GEN(priv->mdev, log_max_dek) - 1);
+
+	for (x = 0; x != MLX5E_TLS_RX_STATS_NUM; x++)
+		ptls->stats.arg[x] = counter_u64_alloc(M_WAITOK);
+
+	ptls->init = 1;
+
+	node = SYSCTL_ADD_NODE(&priv->sysctl_ctx,
+	    SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO,
+	    "tls_rx", CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "Hardware TLS receive offload");
+	if (node == NULL)
+		return (0);
+
+	mlx5e_create_counter_stats(&ptls->ctx,
+	    SYSCTL_CHILDREN(node), "stats",
+	    mlx5e_tls_rx_stats_desc, MLX5E_TLS_RX_STATS_NUM,
+	    ptls->stats.arg);
+
+	return (0);
+}
+
+/*
+ * This function disables TLS RX support for the given NIC.
+ */
+void
+mlx5e_tls_rx_cleanup(struct mlx5e_priv *priv)
+{
+	struct mlx5e_tls_rx *ptls = &priv->tls_rx;
+	uint32_t x;
+
+	if (ptls->init == 0)
+		return;
+
+	ptls->init = 0;
+	flush_workqueue(ptls->wq);
+	sysctl_ctx_free(&ptls->ctx);
+	uma_zdestroy(ptls->zone);
+	destroy_workqueue(ptls->wq);
+
+	/* check if all resources are freed */
+	MPASS(priv->tls_rx.num_resources == 0);
+
+	for (x = 0; x != MLX5E_TLS_RX_STATS_NUM; x++)
+		counter_u64_free(ptls->stats.arg[x]);
+}
+
+/*
+ * This function is used to serialize sleeping firmware operations
+ * needed in order to establish and destroy a TLS RX tag.
+ */
+static void
+mlx5e_tls_rx_work(struct work_struct *work)
+{
+	struct mlx5e_tls_rx_tag *ptag;
+	struct mlx5e_priv *priv;
+	int err;
+
+	ptag = container_of(work, struct mlx5e_tls_rx_tag, work);
+	priv = container_of(ptag->tls_rx, struct mlx5e_priv, tls_rx);
+
+	switch (ptag->state) {
+	case MLX5E_TLS_RX_ST_INIT:
+		/* try to allocate new TIR context */
+		err = mlx5_tls_open_tir(priv->mdev, priv->tdn,
+		    priv->channel[mlx5e_tls_rx_get_ch(priv, ptag->flowid, ptag->flowtype)].rqtn,
+		    &ptag->tirn);
+		if (err) {
+			MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
+			break;
+		}
+		MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, progress.pd, ptag->tirn);
+
+		/* try to allocate a DEK context ID */
+		err = mlx5_encryption_key_create(priv->mdev, priv->pdn,
+		    MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, key.key_data),
+		    MLX5_GET(sw_tls_rx_cntx, ptag->crypto_params, key.key_len),
+		    &ptag->dek_index);
+		if (err) {
+			MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
+			break;
+		}
+
+		MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, param.dek_index, ptag->dek_index);
+
+		ptag->dek_index_ok = 1;
+
+		MLX5E_TLS_RX_TAG_LOCK(ptag);
+		if (ptag->state == MLX5E_TLS_RX_ST_INIT)
+			ptag->state = MLX5E_TLS_RX_ST_SETUP;
+		MLX5E_TLS_RX_TAG_UNLOCK(ptag);
+		break;
+
+	case MLX5E_TLS_RX_ST_FREED:
+		/* remove flow rule for incoming traffic, if any */
+		if (ptag->flow_rule != NULL)
+			mlx5e_accel_fs_del_inpcb(ptag->flow_rule);
+
+		/* wait for all refs to go away */
+		while (ptag->refs != 0)
+			msleep(1);
+
+		/* try to destroy DEK context by ID */
+		if (ptag->dek_index_ok)
+			mlx5_encryption_key_destroy(priv->mdev, ptag->dek_index);
+
+		/* try to destroy TIR context by ID */
+		if (ptag->tirn != 0)
+			mlx5_tls_close_tir(priv->mdev, ptag->tirn);
+
+		/* free tag */
+		mlx5e_tls_rx_tag_zfree(ptag);
+		break;
+
+	default:
+		break;
+	}
+}
+
+/*
+ * This function translates the crypto parameters into the format used
+ * by the firmware and hardware. Currently only AES-128 and AES-256 is
+ * supported for TLS v1.2 and TLS v1.3.
+ *
+ * Returns zero on success, else an error happened.
+ */
+static int
+mlx5e_tls_rx_set_params(void *ctx, struct inpcb *inp, const struct tls_session_params *en)
+{
+	uint32_t tcp_sn_he;
+	uint64_t tls_sn_he;
+
+	MLX5_SET(sw_tls_rx_cntx, ctx, param.const_2, 2);
+	if (en->tls_vminor == TLS_MINOR_VER_TWO)
+		MLX5_SET(sw_tls_rx_cntx, ctx, param.tls_version, 2); /* v1.2 */
+	else
+		MLX5_SET(sw_tls_rx_cntx, ctx, param.tls_version, 3); /* v1.3 */
+	MLX5_SET(sw_tls_rx_cntx, ctx, param.const_1, 1);
+	MLX5_SET(sw_tls_rx_cntx, ctx, param.encryption_standard, 1); /* TLS */
+
+	/* copy the initial vector in place */
+	switch (en->iv_len) {
+	case MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param.gcm_iv):
+	case MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param.gcm_iv) +
+	     MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param.implicit_iv):
+		memcpy(MLX5_ADDR_OF(sw_tls_rx_cntx, ctx, param.gcm_iv),
+		    en->iv, en->iv_len);
+		break;
+	default:
+		return (EINVAL);
+	}
+
+	if (en->cipher_key_len <= MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, key.key_data)) {
+		memcpy(MLX5_ADDR_OF(sw_tls_rx_cntx, ctx, key.key_data),
+		    en->cipher_key, en->cipher_key_len);
+		MLX5_SET(sw_tls_rx_cntx, ctx, key.key_len, en->cipher_key_len);
+	} else {
+		return (EINVAL);
+	}
+
+	if (__predict_false(inp == NULL ||
+	    ktls_get_rx_sequence(inp, &tcp_sn_he, &tls_sn_he) != 0))
+		return (EINVAL);
+
+	MLX5_SET64(sw_tls_rx_cntx, ctx, param.initial_record_number, tls_sn_he);
+	MLX5_SET(sw_tls_rx_cntx, ctx, param.resync_tcp_sn, tcp_sn_he);
+
+	return (0);
+}
+
+/* Verify zero default */
+CTASSERT(MLX5E_TLS_RX_ST_INIT == 0);
+
+/*
+ * This functino is responsible for allocating a TLS RX tag. It is a
+ * callback function invoked by the network stack.
+ *
+ * Returns zero on success else an error happened.
+ */
+int
+mlx5e_tls_rx_snd_tag_alloc(struct ifnet *ifp,
+    union if_snd_tag_alloc_params *params,
+    struct m_snd_tag **ppmt)
+{
+	struct mlx5e_iq *iq;
+	struct mlx5e_priv *priv;
+	struct mlx5e_tls_rx_tag *ptag;
+	struct mlx5_flow_rule *flow_rule;
+	const struct tls_session_params *en;
+	uint32_t value;
+	int error;
+
+	priv = ifp->if_softc;
+
+	if (unlikely(priv->gone != 0 || priv->tls_rx.init == 0 ||
+	    params->hdr.flowtype == M_HASHTYPE_NONE))
+		return (EOPNOTSUPP);
+
+	/* allocate new tag from zone, if any */
+	ptag = uma_zalloc(priv->tls_rx.zone, M_NOWAIT);
+	if (ptag == NULL)
+		return (ENOMEM);
+
+	/* sanity check default values */
+	MPASS(ptag->state == MLX5E_TLS_RX_ST_INIT);
+	MPASS(ptag->dek_index == 0);
+	MPASS(ptag->dek_index_ok == 0);
+
+	/* setup TLS RX tag */
+	ptag->tls_rx = &priv->tls_rx;
+	ptag->flowtype = params->hdr.flowtype;
+	ptag->flowid = params->hdr.flowid;
+
+	value = atomic_fetchadd_32(&priv->tls_rx.num_resources, 1U);
+
+	/* check resource limits */
+	if (value >= priv->tls_rx.max_resources) {
+		error = ENOMEM;
+		goto failure;
+	}
+
+	en = &params->tls_rx.tls->params;
+
+	/* only TLS v1.2 and v1.3 is currently supported */
+	if (en->tls_vmajor != TLS_MAJOR_VER_ONE ||
+	    (en->tls_vminor != TLS_MINOR_VER_TWO
+#ifdef TLS_MINOR_VER_THREE
+	     && en->tls_vminor != TLS_MINOR_VER_THREE
+#endif
+	     )) {
+		error = EPROTONOSUPPORT;
+		goto failure;
+	}
+
+	switch (en->cipher_algorithm) {
+	case CRYPTO_AES_NIST_GCM_16:
+		switch (en->cipher_key_len) {
+		case 128 / 8:
+			if (en->tls_vminor == TLS_MINOR_VER_TWO) {
+				if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_128) == 0) {
+					error = EPROTONOSUPPORT;
+					goto failure;
+				}
+			} else {
+				if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_128) == 0) {
+					error = EPROTONOSUPPORT;
+					goto failure;
+				}
+			}
+			error = mlx5e_tls_rx_set_params(
+			    ptag->crypto_params, params->tls_rx.inp, en);
+			if (error)
+				goto failure;
+			break;
+
+		case 256 / 8:
+			if (en->tls_vminor == TLS_MINOR_VER_TWO) {
+				if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_256) == 0) {
+					error = EPROTONOSUPPORT;
+					goto failure;
+				}
+			} else {
+				if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_256) == 0) {
+					error = EPROTONOSUPPORT;
+					goto failure;
+				}
+			}
+			error = mlx5e_tls_rx_set_params(
+			    ptag->crypto_params, params->tls_rx.inp, en);
+			if (error)
+				goto failure;
+			break;
+
+		default:
+			error = EINVAL;
+			goto failure;
+		}
+		break;
+	default:
+		error = EPROTONOSUPPORT;
+		goto failure;
+	}
+
+	/* store pointer to mbuf tag */
+	MPASS(ptag->tag.refcount == 0);
+	m_snd_tag_init(&ptag->tag, ifp, &mlx5e_tls_rx_snd_tag_sw);
+	*ppmt = &ptag->tag;
+
+	queue_work(priv->tls_rx.wq, &ptag->work);
+	flush_work(&ptag->work);
+
+	/* check that worker task completed successfully */
+	MLX5E_TLS_RX_TAG_LOCK(ptag);
+	if (ptag->state == MLX5E_TLS_RX_ST_SETUP) {
+		ptag->state = MLX5E_TLS_RX_ST_READY;
+		error = 0;
+	} else {
+		error = ENOMEM;
+	}
+	MLX5E_TLS_RX_TAG_UNLOCK(ptag);
+
+	if (unlikely(error))
+		goto cleanup;
+
+	iq = mlx5e_tls_rx_get_iq(priv, ptag->flowid, ptag->flowtype);
+
+	/* establish connection between DEK and TIR */
+	if (mlx5e_tls_rx_send_static_parameters(iq, ptag) != 0) {
+		MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
+		error = ENOMEM;
+		goto cleanup;
+	}
+
+	MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, progress.auth_state,
+	    MLX5E_TLS_RX_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD);
+	MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, progress.record_tracker_state,
+	    MLX5E_TLS_RX_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START);
+
+	/* reset state to all zeros */
+	if (mlx5e_tls_rx_send_progress_parameters_sync(iq, ptag) != 0) {
+		MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
+		error = ENOMEM;
+		goto cleanup;
+	}
+
+	if (ifp->if_pcp != IFNET_PCP_NONE || params->tls_rx.vlan_id != 0) {
+		/* create flow rule for TLS RX traffic (tagged) */
+		flow_rule = mlx5e_accel_fs_add_inpcb(priv, params->tls_rx.inp,
+		    ptag->tirn, MLX5_FS_DEFAULT_FLOW_TAG, params->tls_rx.vlan_id);
+	} else {
+		/* create flow rule for TLS RX traffic (untagged) */
+		flow_rule = mlx5e_accel_fs_add_inpcb(priv, params->tls_rx.inp,
+		    ptag->tirn, MLX5_FS_DEFAULT_FLOW_TAG, MLX5E_ACCEL_FS_ADD_INPCB_NO_VLAN);
+	}
+
+	if (IS_ERR_OR_NULL(flow_rule)) {
+		MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
+		error = ENOMEM;
+		goto cleanup;
+	}
+
+	ptag->flow_rule = flow_rule;
+
+	return (0);
+
+cleanup:
+	MLX5E_TLS_RX_TAG_LOCK(ptag);
+	ptag->state = MLX5E_TLS_RX_ST_FREED;
+	MLX5E_TLS_RX_TAG_UNLOCK(ptag);
+
+	queue_work(priv->tls_rx.wq, &ptag->work);
+	flush_work(&ptag->work);
+	return (error);
+
+failure:
+	mlx5e_tls_rx_tag_zfree(ptag);
+	return (error);
+}
+
+
+/*
+ * This function adds the TCP sequence number and TLS record number in
+ * host endian format to a small database. When TLS records have the
+ * same length, they are simply accumulated by counting instead of
+ * separated entries in the TLS database. The dimension of the
+ * database is such that it cannot store more than 1GByte of
+ * continuous TCP data to avoid issues with TCP sequence number wrap
+ * around. A record length of zero bytes has special meaning and means
+ * that resync completed and all data in the database can be
+ * discarded. This function is called after the TCP stack has
+ * re-assembled all TCP fragments due to out of order packet reception
+ * and all TCP sequence numbers should be sequential.
+ *
+ * This function returns true if a so-called TLS RX resync operation
+ * is in progress. Else no such operation is in progress.
+ */
+static bool
+mlx5e_tls_rx_snd_tag_add_tcp_sequence(struct mlx5e_tls_rx_tag *ptag,
+    uint32_t tcp_sn_he, uint32_t len, uint64_t tls_rcd)
+{
+	uint16_t i, j, n;
+
+	if (ptag->tcp_resync_active == 0 ||
+	    ptag->tcp_resync_next != tcp_sn_he ||
+	    len == 0) {
+		/* start over again or terminate */
+		ptag->tcp_resync_active = (len != 0);
+		ptag->tcp_resync_len[0] = len;
+		ptag->tcp_resync_num[0] = 1;
+		ptag->tcp_resync_pc = (len != 0);
+		ptag->tcp_resync_cc = 0;
+		ptag->tcp_resync_start = tcp_sn_he;
+		ptag->rcd_resync_start = tls_rcd;
+	} else {
+		i = (ptag->tcp_resync_pc - 1) & (MLX5E_TLS_RX_RESYNC_MAX - 1);
+		n = ptag->tcp_resync_pc - ptag->tcp_resync_cc;
+
+		/* check if same length like last time */
+		if (ptag->tcp_resync_len[i] == len &&
+		    ptag->tcp_resync_num[i] != MLX5E_TLS_RX_NUM_MAX) {
+			/* use existing entry */
+			ptag->tcp_resync_num[i]++;
+		} else if (n == MLX5E_TLS_RX_RESYNC_MAX) {
+			j = ptag->tcp_resync_cc++ & (MLX5E_TLS_RX_RESYNC_MAX - 1);
+			/* adjust starting TCP sequence number */
+			ptag->rcd_resync_start += ptag->tcp_resync_num[j];
+			ptag->tcp_resync_start += ptag->tcp_resync_len[j] * ptag->tcp_resync_num[j];
+			i = ptag->tcp_resync_pc++ & (MLX5E_TLS_RX_RESYNC_MAX - 1);
+			/* store new entry */
+			ptag->tcp_resync_len[i] = len;
+			ptag->tcp_resync_num[i] = 1;
+		} else {
+			i = ptag->tcp_resync_pc++ & (MLX5E_TLS_RX_RESYNC_MAX - 1);
+			/* add new entry */
+			ptag->tcp_resync_len[i] = len;
+			ptag->tcp_resync_num[i] = 1;
+		}
+	}
+
+	/* store next TCP SN in host endian format */
+	ptag->tcp_resync_next = tcp_sn_he + len;
+
+	return (ptag->tcp_resync_active);
+}
+
+/*
+ * This function checks if the given TCP sequence number points to the
+ * beginning of a valid TLS header.
+ *
+ * Returns true if a match is found. Else false.
+ */
+static bool
+mlx5e_tls_rx_snd_tag_find_tcp_sn_and_tls_rcd(struct mlx5e_tls_rx_tag *ptag,
+    uint32_t tcp_sn_he, uint32_t *p_next_tcp_sn_he, uint64_t *p_tls_rcd)
+{
+	uint16_t i, j;
+	uint32_t off = 0;
+	uint32_t rcd = 0;
+	uint32_t delta;
+	uint32_t leap;
+
+	for (i = ptag->tcp_resync_cc; i != ptag->tcp_resync_pc; i++) {
+		delta = tcp_sn_he - off - ptag->tcp_resync_start;
+
+		/* check if subtraction went negative */
+		if ((int32_t)delta < 0)
+			break;
+
+		j = i & (MLX5E_TLS_RX_RESYNC_MAX - 1);
+		leap = ptag->tcp_resync_len[j] * ptag->tcp_resync_num[j];
+		if (delta < leap) {
+			if ((delta % ptag->tcp_resync_len[j]) == 0) {
+				*p_next_tcp_sn_he = tcp_sn_he +
+				    ptag->tcp_resync_len[j];
+				*p_tls_rcd = ptag->rcd_resync_start +
+				    (uint64_t)rcd +
+				    (uint64_t)(delta / ptag->tcp_resync_len[j]);
+				return (true);		/* success */
+			}
+			break;	/* invalid offset */
+		}
+		rcd += ptag->tcp_resync_num[j];
+		off += leap;
+	}
+	return (false);	/* not found */
+}
+
+/*
+ * This is a callback function from the network stack to keep track of
+ * TLS RX TCP sequence numbers.
+ *
+ * Returns zero on success else an error happened.
+ */
+static int
+mlx5e_tls_rx_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
+{
+	struct mlx5e_tls_rx_tag *ptag;
+	struct mlx5e_priv *priv;
+	struct mlx5e_iq *iq;
+	int err;
+
+	ptag = container_of(pmt, struct mlx5e_tls_rx_tag, tag);
+	priv = container_of(ptag->tls_rx, struct mlx5e_priv, tls_rx);
+
+	if (unlikely(priv->gone != 0))
+		return (ENXIO);
+
+	iq = mlx5e_tls_rx_get_iq(priv, ptag->flowid, ptag->flowtype);
+
+	MLX5E_TLS_RX_TAG_LOCK(ptag);
+
+	if (mlx5e_tls_rx_snd_tag_add_tcp_sequence(ptag,
+	    params->tls_rx.tls_hdr_tcp_sn,
+	    params->tls_rx.tls_rec_length,
+	    params->tls_rx.tls_seq_number) &&
+	    ptag->tcp_resync_pending == 0) {
+		err = mlx5e_tls_rx_receive_progress_parameters(iq, ptag);
+		if (err != 0) {
+			MLX5E_TLS_RX_STAT_INC(ptag, rx_resync_err, 1);
+		} else {
+			ptag->tcp_resync_pending = 1;
+			MLX5E_TLS_RX_STAT_INC(ptag, rx_resync_ok, 1);
+		}
+	} else {
+		err = 0;
+	}
+	MLX5E_TLS_RX_TAG_UNLOCK(ptag);
+
+	return (-err);
+}
+
+/*
+ * This function frees a TLS RX tag in a non-blocking way.
+ */
+static void
+mlx5e_tls_rx_snd_tag_free(struct m_snd_tag *pmt)
+{
+	struct mlx5e_tls_rx_tag *ptag =
+	    container_of(pmt, struct mlx5e_tls_rx_tag, tag);
+	struct mlx5e_priv *priv;
+
+	MLX5E_TLS_RX_TAG_LOCK(ptag);
+	ptag->state = MLX5E_TLS_RX_ST_FREED;
+	MLX5E_TLS_RX_TAG_UNLOCK(ptag);
+
+	priv = ptag->tag.ifp->if_softc;
+	queue_work(priv->tls_rx.wq, &ptag->work);
+}
+
+#else
+
+int
+mlx5e_tls_rx_init(struct mlx5e_priv *priv)
+{
+
+	return (0);
+}
+
+void
+mlx5e_tls_rx_cleanup(struct mlx5e_priv *priv)
+{
+	/* NOP */
+}
+
+#endif
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
index ed0b72084dc7..16488ace5a74 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
@@ -827,6 +827,8 @@ mlx5e_update_stats_locked(struct mlx5e_priv *priv)
 	u64 rx_wqe_err = 0;
 	u64 rx_packets = 0;
 	u64 rx_bytes = 0;
+	u64 rx_decrypted_error = 0;
+	u64 rx_decrypted_ok = 0;
 	u32 rx_out_of_buffer = 0;
 	int error;
 	int i;
@@ -853,6 +855,8 @@ mlx5e_update_stats_locked(struct mlx5e_priv *priv)
 		rx_wqe_err += rq_stats->wqe_err;
 		rx_packets += rq_stats->packets;
 		rx_bytes += rq_stats->bytes;
+		rx_decrypted_error += rq_stats->decrypted_error_packets;
+		rx_decrypted_ok += rq_stats->decrypted_ok_packets;
 
 		for (j = 0; j < priv->num_tc; j++) {
 			sq_stats = &pch->sq[j].stats;
@@ -903,6 +907,8 @@ mlx5e_update_stats_locked(struct mlx5e_priv *priv)
 	s->rx_wqe_err = rx_wqe_err;
 	s->rx_packets = rx_packets;
 	s->rx_bytes = rx_bytes;
+	s->rx_decrypted_error_packets = rx_decrypted_error;
+	s->rx_decrypted_ok_packets = rx_decrypted_ok;
 
 	mlx5e_grp_vnic_env_update_stats(priv);
 
@@ -4388,6 +4394,8 @@ mlx5e_snd_tag_alloc(struct ifnet *ifp,
 #ifdef KERN_TLS
 	case IF_SND_TAG_TYPE_TLS:
 		return (mlx5e_tls_snd_tag_alloc(ifp, params, ppmt));
+	case IF_SND_TAG_TYPE_TLS_RX:
+		return (mlx5e_tls_rx_snd_tag_alloc(ifp, params, ppmt));
 #endif
 	default:
 		return (EOPNOTSUPP);
@@ -4644,6 +4652,12 @@ mlx5e_create_ifp(struct mlx5_core_dev *mdev)
 		goto err_open_tirs;
 	}
 
+	err = mlx5e_tls_rx_init(priv);
+	if (err) {
+		if_printf(ifp, "%s: mlx5e_tls_rx_init() failed, %d\n", __func__, err);
+		goto err_open_flow_tables;
+	}
+
 	/* set default MTU */
 	mlx5e_set_dev_port_mtu(ifp, ifp->if_mtu);
 
@@ -4779,6 +4793,9 @@ mlx5e_create_ifp(struct mlx5_core_dev *mdev)
 
 	return (priv);
 
+err_open_flow_tables:
+	mlx5e_close_flow_tables(priv);
+
 err_open_tirs:
 	mlx5e_close_tirs(priv);
 
@@ -4853,6 +4870,14 @@ mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
 		    "Waiting for all TLS connections to terminate\n");
 		pause("W", hz);
 	}
+
+	/* wait for all TLS RX tags to get freed */
+	while (priv->tls_rx.init != 0 &&
+	    uma_zone_get_cur(priv->tls_rx.zone) != 0)  {
+		mlx5_en_err(priv->ifp,
+		    "Waiting for all TLS RX connections to terminate\n");
+		pause("W", hz);
+	}
 #endif
 	/* wait for all unlimited send tags to complete */
 	mlx5e_priv_wait_for_completion(priv, mdev->priv.eq_table.num_comp_vectors);
@@ -4887,6 +4912,7 @@ mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
 	ifmedia_removeall(&priv->media);
 	ether_ifdetach(ifp);
 
+	mlx5e_tls_rx_cleanup(priv);
 	mlx5e_close_flow_tables(priv);
 	mlx5e_close_tirs(priv);
 	mlx5e_close_rqts(priv);
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
index 9e58a9bd7904..0e3a3b3917f4 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
@@ -372,6 +372,19 @@ mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe,
 		mb->m_pkthdr.rcv_tstmp = tstmp;
 		mb->m_flags |= M_TSTMP;
 	}
+
+	switch (get_cqe_tls_offload(cqe)) {
+	case CQE_TLS_OFFLOAD_DECRYPTED:
+		/* set proper checksum flag for decrypted packets */
+		mb->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED;
+		rq->stats.decrypted_ok_packets++;
+		break;
+	case CQE_TLS_OFFLOAD_ERROR:
+		rq->stats.decrypted_error_packets++;
+		break;
+	default:
+		break;
+	}
 }
 
 static inline void
diff --git a/sys/dev/mlx5/mlx5_ifc.h b/sys/dev/mlx5/mlx5_ifc.h
index e9c7a86e862d..52bd65ccdb45 100644
--- a/sys/dev/mlx5/mlx5_ifc.h
+++ b/sys/dev/mlx5/mlx5_ifc.h
@@ -1366,7 +1366,7 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 
 	u8         reserved_at_480[0x1];
 	u8         tls_tx[0x1];
-	u8         reserved_at_482[0x1];
+	u8         tls_rx[0x1];
 	u8         log_max_l2_table[0x5];
 	u8         reserved_64[0x8];
 	u8         log_uar_page_sz[0x10];
diff --git a/sys/modules/mlx5en/Makefile b/sys/modules/mlx5en/Makefile
index 6e15fe3ac236..cc2cc264b435 100644
--- a/sys/modules/mlx5en/Makefile
+++ b/sys/modules/mlx5en/Makefile
@@ -9,6 +9,7 @@ mlx5_en_main.c \
 mlx5_en_tx.c \
 mlx5_en_flow_table.c \
 mlx5_en_hw_tls.c \
+mlx5_en_hw_tls_rx.c \
 mlx5_en_iq.c \
 mlx5_en_rx.c \
 mlx5_en_rl.c \