ktls: Support for TLS 1.3 receive offload.

Note that support for TLS 1.3 receive offload in OpenSSL is still an
open pull request in active development.  However, potential changes
to that pull request should not affect the kernel interface.

Reviewed by:	hselasky
Sponsored by:	Netflix
Differential Revision:	https://reviews.freebsd.org/D33007

1 files changed, 66 insertions, 11 deletions

diff --git a/sys/opencrypto/ktls_ocf.c b/sys/opencrypto/ktls_ocf.c
index dc8b6f76257b..a2bb94dc717a 100644
--- a/sys/opencrypto/ktls_ocf.c
+++ b/sys/opencrypto/ktls_ocf.c
@@ -106,11 +106,21 @@ SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_encrypts,
     CTLFLAG_RD, &ocf_tls12_chacha20_encrypts,
     "Total number of OCF TLS 1.2 Chacha20-Poly1305 encryption operations");
 
+static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_decrypts);
+SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_decrypts,
+    CTLFLAG_RD, &ocf_tls13_gcm_decrypts,
+    "Total number of OCF TLS 1.3 GCM decryption operations");
+
 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_encrypts);
 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_encrypts,
     CTLFLAG_RD, &ocf_tls13_gcm_encrypts,
     "Total number of OCF TLS 1.3 GCM encryption operations");
 
+static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_decrypts);
+SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_decrypts,
+    CTLFLAG_RD, &ocf_tls13_chacha20_decrypts,
+    "Total number of OCF TLS 1.3 Chacha20-Poly1305 decryption operations");
+
 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_encrypts);
 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_encrypts,
     CTLFLAG_RD, &ocf_tls13_chacha20_encrypts,
@@ -600,6 +610,58 @@ ktls_ocf_tls13_aead_encrypt(struct ktls_ocf_encrypt_state *state,
 	return (error);
 }
 
+static int
+ktls_ocf_tls13_aead_decrypt(struct ktls_session *tls,
+    const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno,
+    int *trailer_len)
+{
+	struct tls_aead_data_13 ad;
+	struct cryptop crp;
+	struct ktls_ocf_session *os;
+	int error;
+	u_int tag_len;
+
+	os = tls->ocf_session;
+
+	tag_len = tls->params.tls_tlen - 1;
+
+	/* Payload must contain at least one byte for the record type. */
+	if (ntohs(hdr->tls_length) < tag_len + 1)
+		return (EBADMSG);
+
+	crypto_initreq(&crp, os->sid);
+
+	/* Setup the nonce. */
+	memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len);
+	*(uint64_t *)(crp.crp_iv + 4) ^= htobe64(seqno);
+
+	/* Setup the AAD. */
+	ad.type = hdr->tls_type;
+	ad.tls_vmajor = hdr->tls_vmajor;
+	ad.tls_vminor = hdr->tls_vminor;
+	ad.tls_length = hdr->tls_length;
+	crp.crp_aad = &ad;
+	crp.crp_aad_length = sizeof(ad);
+
+	crp.crp_payload_start = tls->params.tls_hlen;
+	crp.crp_payload_length = ntohs(hdr->tls_length) - tag_len;
+	crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length;
+
+	crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST;
+	crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
+	crypto_use_mbuf(&crp, m);
+
+	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
+		counter_u64_add(ocf_tls13_gcm_decrypts, 1);
+	else
+		counter_u64_add(ocf_tls13_chacha20_decrypts, 1);
+	error = ktls_ocf_dispatch(os, &crp);
+
+	crypto_destroyreq(&crp);
+	*trailer_len = tag_len;
+	return (error);
+}
+
 void
 ktls_ocf_free(struct ktls_session *tls)
 {
@@ -639,11 +701,6 @@ ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction)
 		    tls->params.tls_vminor > TLS_MINOR_VER_THREE)
 			return (EPROTONOSUPPORT);
 
-		/* TLS 1.3 is not yet supported for receive. */
-		if (direction == KTLS_RX &&
-		    tls->params.tls_vminor == TLS_MINOR_VER_THREE)
-			return (EPROTONOSUPPORT);
-
 		csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD;
 		csp.csp_mode = CSP_MODE_AEAD;
 		csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
@@ -711,11 +768,6 @@ ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction)
 		    tls->params.tls_vminor > TLS_MINOR_VER_THREE)
 			return (EPROTONOSUPPORT);
 
-		/* TLS 1.3 is not yet supported for receive. */
-		if (direction == KTLS_RX &&
-		    tls->params.tls_vminor == TLS_MINOR_VER_THREE)
-			return (EPROTONOSUPPORT);
-
 		csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD;
 		csp.csp_mode = CSP_MODE_AEAD;
 		csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305;
@@ -759,7 +811,10 @@ ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction)
 			else
 				tls->sw_encrypt = ktls_ocf_tls12_aead_encrypt;
 		} else {
-			tls->sw_decrypt = ktls_ocf_tls12_aead_decrypt;
+			if (tls->params.tls_vminor == TLS_MINOR_VER_THREE)
+				tls->sw_decrypt = ktls_ocf_tls13_aead_decrypt;
+			else
+				tls->sw_decrypt = ktls_ocf_tls12_aead_decrypt;
 		}
 	} else {
 		tls->sw_encrypt = ktls_ocf_tls_cbc_encrypt;