kTLS support for TLS 1.3

TLS 1.3 requires a few changes because 1.3 pretends to be 1.2
with a record type of application data. The "real" record type is
then included at the end of the user-supplied plaintext
data. This required adding a field to the mbuf_ext_pgs struct to
save the record type, and passing the real record type to the
sw_encrypt() ktls backend functions.

Reviewed by:	jhb, hselasky
Sponsored by:	Netflix
Differential Revision:	D21801

5 files changed, 50 insertions, 15 deletions

diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c
index 5736be83b6a6..5dfb6b3d873c 100644
--- a/sys/kern/uipc_ktls.c
+++ b/sys/kern/uipc_ktls.c
@@ -389,14 +389,14 @@ ktls_create_session(struct socket *so, struct tls_enable *en,
 	if (en->tls_vmajor != TLS_MAJOR_VER_ONE)
 		return (EINVAL);
 	if (en->tls_vminor < TLS_MINOR_VER_ZERO ||
-	    en->tls_vminor > TLS_MINOR_VER_TWO)
+	    en->tls_vminor > TLS_MINOR_VER_THREE)
 		return (EINVAL);
 
 	if (en->auth_key_len < 0 || en->auth_key_len > TLS_MAX_PARAM_SIZE)
 		return (EINVAL);
 	if (en->cipher_key_len < 0 || en->cipher_key_len > TLS_MAX_PARAM_SIZE)
 		return (EINVAL);
-	if (en->iv_len < 0 || en->iv_len > TLS_MAX_PARAM_SIZE)
+	if (en->iv_len < 0 || en->iv_len > sizeof(tls->params.iv))
 		return (EINVAL);
 
 	/* All supported algorithms require a cipher key. */
@@ -425,7 +425,10 @@ ktls_create_session(struct socket *so, struct tls_enable *en,
 		}
 		if (en->auth_key_len != 0)
 			return (EINVAL);
-		if (en->iv_len != TLS_AEAD_GCM_LEN)
+		if ((en->tls_vminor == TLS_MINOR_VER_TWO &&
+			en->iv_len != TLS_AEAD_GCM_LEN) ||
+		    (en->tls_vminor == TLS_MINOR_VER_THREE &&
+			en->iv_len != TLS_1_3_GCM_IV_LEN))
 			return (EINVAL);
 		break;
 	case CRYPTO_AES_CBC:
@@ -477,8 +480,22 @@ ktls_create_session(struct socket *so, struct tls_enable *en,
 	tls->params.tls_hlen = sizeof(struct tls_record_layer);
 	switch (en->cipher_algorithm) {
 	case CRYPTO_AES_NIST_GCM_16:
-		tls->params.tls_hlen += 8;
+		/*
+		 * TLS 1.2 uses a 4 byte implicit IV with an explicit 8 byte
+		 * nonce.  TLS 1.3 uses a 12 byte implicit IV.
+		 */
+		if (en->tls_vminor < TLS_MINOR_VER_THREE)
+			tls->params.tls_hlen += sizeof(uint64_t);
 		tls->params.tls_tlen = AES_GMAC_HASH_LEN;
+
+		/*
+		 * TLS 1.3 includes optional padding which we
+		 * do not support, and also puts the "real" record
+		 * type at the end of the encrypted data.
+		 */
+		if (en->tls_vminor == TLS_MINOR_VER_THREE)
+			tls->params.tls_tlen += sizeof(uint8_t);
+
 		tls->params.tls_bs = 1;
 		break;
 	case CRYPTO_AES_CBC:
@@ -539,7 +556,6 @@ ktls_create_session(struct socket *so, struct tls_enable *en,
 	 * of the IV are generated in ktls_frame() and ktls_seq().
 	 */
 	if (en->iv_len != 0) {
-		MPASS(en->iv_len <= sizeof(tls->params.iv));
 		tls->params.iv_len = en->iv_len;
 		error = copyin(en->iv, tls->params.iv, en->iv_len);
 		if (error)
@@ -1188,8 +1204,21 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
 		/* Populate the TLS header. */
 		tlshdr = (void *)pgs->hdr;
 		tlshdr->tls_vmajor = tls->params.tls_vmajor;
-		tlshdr->tls_vminor = tls->params.tls_vminor;
-		tlshdr->tls_type = record_type;
+
+		/*
+		 * TLS 1.3 masquarades as TLS 1.2 with a record type
+		 * of TLS_RLTYPE_APP.
+		 */
+		if (tls->params.tls_vminor == TLS_MINOR_VER_THREE &&
+		    tls->params.tls_vmajor == TLS_MAJOR_VER_ONE) {
+			tlshdr->tls_vminor = TLS_MINOR_VER_TWO;
+			tlshdr->tls_type = TLS_RLTYPE_APP;
+			/* save the real record type for later */
+			pgs->record_type = record_type;
+		} else {
+			tlshdr->tls_vminor = tls->params.tls_vminor;
+			tlshdr->tls_type = record_type;
+		}
 		tlshdr->tls_length = htons(m->m_len - sizeof(*tlshdr));
 
 		/*
@@ -1365,7 +1394,8 @@ retry_page:
 
 		error = (*tls->sw_encrypt)(tls,
 		    (const struct tls_record_layer *)pgs->hdr,
-		    pgs->trail, src_iov, dst_iov, i, pgs->seqno);
+		    pgs->trail, src_iov, dst_iov, i, pgs->seqno,
+		    pgs->record_type);
 		if (error) {
 			counter_u64_add(ktls_offload_failed_crypto, 1);
 			break;
diff --git a/sys/net/iflib.c b/sys/net/iflib.c
index 148a8b3e6d90..c0d76b5b94a3 100644
--- a/sys/net/iflib.c
+++ b/sys/net/iflib.c
@@ -4076,7 +4076,7 @@ iflib_if_qflush(if_t ifp)
 #define IFCAP_FLAGS (IFCAP_HWCSUM_IPV6 | IFCAP_HWCSUM | IFCAP_LRO | \
 		     IFCAP_TSO | IFCAP_VLAN_HWTAGGING | IFCAP_HWSTATS | \
 		     IFCAP_VLAN_MTU | IFCAP_VLAN_HWFILTER | \
-		     IFCAP_VLAN_HWTSO | IFCAP_VLAN_HWCSUM)
+		     IFCAP_VLAN_HWTSO | IFCAP_VLAN_HWCSUM | IFCAP_NOMAP)
 
 static int
 iflib_if_ioctl(if_t ifp, u_long command, caddr_t data)
@@ -4201,7 +4201,7 @@ iflib_if_ioctl(if_t ifp, u_long command, caddr_t data)
 
 		oldmask = if_getcapenable(ifp);
 		mask = ifr->ifr_reqcap ^ oldmask;
-		mask &= ctx->ifc_softc_ctx.isc_capabilities;
+		mask &= ctx->ifc_softc_ctx.isc_capabilities | IFCAP_NOMAP;
 		setmask = 0;
 #ifdef TCP_OFFLOAD
 		setmask |= mask & (IFCAP_TOE4|IFCAP_TOE6);
@@ -4596,8 +4596,10 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct
 		MPASS(scctx->isc_tx_csum_flags);
 #endif
 
-	if_setcapabilities(ifp, scctx->isc_capabilities | IFCAP_HWSTATS);
-	if_setcapenable(ifp, scctx->isc_capenable | IFCAP_HWSTATS);
+	if_setcapabilities(ifp,
+	    scctx->isc_capabilities | IFCAP_HWSTATS | IFCAP_NOMAP);
+	if_setcapenable(ifp,
+	    scctx->isc_capenable | IFCAP_HWSTATS | IFCAP_NOMAP);
 
 	if (scctx->isc_ntxqsets == 0 || (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_ntxqsets))
 		scctx->isc_ntxqsets = scctx->isc_ntxqsets_max;
diff --git a/sys/opencrypto/ktls_ocf.c b/sys/opencrypto/ktls_ocf.c
index 953fc1c9b6e3..d9e76e1d0ffd 100644
--- a/sys/opencrypto/ktls_ocf.c
+++ b/sys/opencrypto/ktls_ocf.c
@@ -86,7 +86,7 @@ ktls_ocf_callback(struct cryptop *crp)
 static int
 ktls_ocf_encrypt(struct ktls_session *tls, const struct tls_record_layer *hdr,
     uint8_t *trailer, struct iovec *iniov, struct iovec *outiov, int iovcnt,
-    uint64_t seqno)
+    uint64_t seqno, uint8_t record_type __unused)
 {
 	struct uio uio;
 	struct tls_aead_data ad;
diff --git a/sys/sys/ktls.h b/sys/sys/ktls.h
index 079d4448bd8d..62f694b3daee 100644
--- a/sys/sys/ktls.h
+++ b/sys/sys/ktls.h
@@ -43,6 +43,7 @@ struct tls_record_layer {
 #define	TLS_MAX_MSG_SIZE_V10_2	16384
 #define	TLS_MAX_PARAM_SIZE	1024	/* Max key/mac/iv in sockopt */
 #define	TLS_AEAD_GCM_LEN	4
+#define	TLS_1_3_GCM_IV_LEN	12
 #define	TLS_CBC_IMPLICIT_IV_LEN	16
 
 /* Type values for the record layer */
@@ -85,6 +86,7 @@ struct tls_mac_data {
 #define	TLS_MINOR_VER_ZERO	1	/* 3, 1 */
 #define	TLS_MINOR_VER_ONE	2	/* 3, 2 */
 #define	TLS_MINOR_VER_TWO	3	/* 3, 3 */
+#define	TLS_MINOR_VER_THREE	4	/* 3, 4 */
 
 /* For TCP_TXTLS_ENABLE */
 struct tls_enable {
@@ -121,7 +123,7 @@ struct tls_session_params {
 
 #ifdef _KERNEL
 
-#define	KTLS_API_VERSION 5
+#define	KTLS_API_VERSION 6
 
 struct iovec;
 struct ktls_session;
@@ -144,7 +146,7 @@ struct ktls_session {
 	int	(*sw_encrypt)(struct ktls_session *tls,
 	    const struct tls_record_layer *hdr, uint8_t *trailer,
 	    struct iovec *src, struct iovec *dst, int iovcnt,
-	    uint64_t seqno);
+	    uint64_t seqno, uint8_t record_type);
 	union {
 		void *cipher;
 		struct m_snd_tag *snd_tag;
diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
index c20e02d258fe..8eb01e6c46b7 100644
--- a/sys/sys/mbuf.h
+++ b/sys/sys/mbuf.h
@@ -359,6 +359,7 @@ struct mbuf_ext_pgs {
 	union {
 		char	trail[MBUF_PEXT_TRAIL_LEN]; /* TLS trailer */
 		struct {
+			uint8_t record_type;	/* Must be first */
 			struct socket *so;
 			struct mbuf *mbuf;
 			uint64_t seqno;