Initial support for kernel offload of TLS receive.

- Add a new TCP_RXTLS_ENABLE socket option to set the encryption and
  authentication algorithms and keys as well as the initial sequence
  number.

- When reading from a socket using KTLS receive, applications must use
  recvmsg().  Each successful call to recvmsg() will return a single
  TLS record.  A new TCP control message, TLS_GET_RECORD, will contain
  the TLS record header of the decrypted record.  The regular message
  buffer passed to recvmsg() will receive the decrypted payload.  This
  is similar to the interface used by Linux's KTLS RX except that
  Linux does not return the full TLS header in the control message.

- Add plumbing to the TOE KTLS interface to request either transmit
  or receive KTLS sessions.

- When a socket is using receive KTLS, redirect reads from
  soreceive_stream() into soreceive_generic().

- Note that this interface is currently only defined for TLS 1.1 and
  1.2, though I believe we will be able to reuse the same interface
  and structures for 1.3.

11 files changed, 175 insertions, 18 deletions

diff --git a/share/man/man4/tcp.4 b/share/man/man4/tcp.4
index 7fc1ccb29928..915b8d0b6bf5 100644
--- a/share/man/man4/tcp.4
+++ b/share/man/man4/tcp.4
@@ -34,7 +34,7 @@
 .\"     From: @(#)tcp.4	8.1 (Berkeley) 6/5/93
 .\" $FreeBSD$
 .\"
-.Dd April 16, 2020
+.Dd April 27, 2020
 .Dt TCP 4
 .Os
 .Sh NAME
@@ -319,14 +319,11 @@ control message.
 The payload of this control message is a single byte holding the desired
 TLS record type.
 .Pp
-Data read from this socket will still be encrypted and must be parsed by
-a TLS-aware consumer.
-.Pp
-At present, only a single key may be set on a socket.
+At present, only a single transmit key may be set on a socket.
 As such, users of this option must disable rekeying.
 .It Dv TCP_TXTLS_MODE
-The integer argument can be used to get or set the current TLS mode of a
-socket.
+The integer argument can be used to get or set the current TLS transmit mode
+of a socket.
 Setting the mode can only used to toggle between software and NIC TLS after
 TLS has been initially enabled via the
 .Dv TCP_TXTLS_ENABLE
@@ -344,6 +341,33 @@ TLS records are encrypted by the network interface card (NIC).
 .It Dv TCP_TLS_MODE_TOE
 TLS records are encrypted by the NIC using a TCP offload engine (TOE).
 .El
+.It Dv TCP_RXTLS_ENABLE
+Enable in-kernel TLS for data read from this socket.
+The
+.Vt struct tls_so_enable
+argument defines the encryption and authentication algorithms and keys
+used to decrypt the socket data.
+.Pp
+Each received TLS record must be read from the socket using
+.Xr recvmsg 2 .
+Each received TLS record will contain a
+.Dv TLS_GET_RECORD
+control message along with the decrypted payload.
+The control message contains a
+.Vt struct tls_get_record
+which includes fields from the TLS record header.
+If a corrupted TLS record is received,
+recvmsg 2
+will fail with
+.Dv EBADMSG .
+.Pp
+At present, only a single receive key may be set on a socket.
+As such, users of this option must disable rekeying.
+.It Dv TCP_RXTLS_MODE
+The integer argument can be used to get the current TLS receive mode
+of a socket.
+The available modes are the same as for
+.Dv TCP_TXTLS_MODE .
 .El
 .Pp
 The option level for the
diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c
index b253bf3c786e..71770c549218 100644
--- a/sys/dev/cxgbe/tom/t4_tom.c
+++ b/sys/dev/cxgbe/tom/t4_tom.c
@@ -40,6 +40,9 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/ktr.h>
+#ifdef KERN_TLS
+#include <sys/ktls.h>
+#endif
 #include <sys/lock.h>
 #include <sys/limits.h>
 #include <sys/module.h>
@@ -814,13 +817,16 @@ t4_tcp_info(struct toedev *tod, struct tcpcb *tp, struct tcp_info *ti)
 #ifdef KERN_TLS
 static int
 t4_alloc_tls_session(struct toedev *tod, struct tcpcb *tp,
-    struct ktls_session *tls)
+    struct ktls_session *tls, int direction)
 {
 	struct toepcb *toep = tp->t_toe;
 
 	INP_WLOCK_ASSERT(tp->t_inpcb);
 	MPASS(tls != NULL);
 
+	if (direction != KTLS_TX)
+		return (EOPNOTSUPP);
+
 	return (tls_alloc_ktls(toep, tls));
 }
 #endif
diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c
index 3d14d67e9f10..6b6bf6df831f 100644
--- a/sys/kern/uipc_ktls.c
+++ b/sys/kern/uipc_ktls.c
@@ -702,7 +702,7 @@ ktls_cleanup(struct ktls_session *tls)
 
 #ifdef TCP_OFFLOAD
 static int
-ktls_try_toe(struct socket *so, struct ktls_session *tls)
+ktls_try_toe(struct socket *so, struct ktls_session *tls, int direction)
 {
 	struct inpcb *inp;
 	struct tcpcb *tp;
@@ -728,7 +728,7 @@ ktls_try_toe(struct socket *so, struct ktls_session *tls)
 		return (EOPNOTSUPP);
 	}
 
-	error = tcp_offload_alloc_tls_session(tp, tls);
+	error = tcp_offload_alloc_tls_session(tp, tls, direction);
 	INP_WUNLOCK(inp);
 	if (error == 0) {
 		tls->mode = TCP_TLS_MODE_TOE;
@@ -901,6 +901,60 @@ ktls_try_sw(struct socket *so, struct ktls_session *tls)
 }
 
 int
+ktls_enable_rx(struct socket *so, struct tls_enable *en)
+{
+	struct ktls_session *tls;
+	int error;
+
+	if (!ktls_offload_enable)
+		return (ENOTSUP);
+
+	counter_u64_add(ktls_offload_enable_calls, 1);
+
+	/*
+	 * This should always be true since only the TCP socket option
+	 * invokes this function.
+	 */
+	if (so->so_proto->pr_protocol != IPPROTO_TCP)
+		return (EINVAL);
+
+	/*
+	 * XXX: Don't overwrite existing sessions.  We should permit
+	 * this to support rekeying in the future.
+	 */
+	if (so->so_rcv.sb_tls_info != NULL)
+		return (EALREADY);
+
+	if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable)
+		return (ENOTSUP);
+
+	error = ktls_create_session(so, en, &tls);
+	if (error)
+		return (error);
+
+	/* TLS RX offload is only supported on TOE currently. */
+#ifdef TCP_OFFLOAD
+	error = ktls_try_toe(so, tls, KTLS_RX);
+#else
+	error = EOPNOTSUPP;
+#endif
+
+	if (error) {
+		ktls_cleanup(tls);
+		return (error);
+	}
+
+	/* Mark the socket as using TLS offload. */
+	SOCKBUF_LOCK(&so->so_rcv);
+	so->so_rcv.sb_tls_info = tls;
+	SOCKBUF_UNLOCK(&so->so_rcv);
+
+	counter_u64_add(ktls_offload_total, 1);
+
+	return (0);
+}
+
+int
 ktls_enable_tx(struct socket *so, struct tls_enable *en)
 {
 	struct ktls_session *tls;
@@ -938,7 +992,7 @@ ktls_enable_tx(struct socket *so, struct tls_enable *en)
 
 	/* Prefer TOE -> ifnet TLS -> software TLS. */
 #ifdef TCP_OFFLOAD
-	error = ktls_try_toe(so, tls);
+	error = ktls_try_toe(so, tls, KTLS_TX);
 	if (error)
 #endif
 		error = ktls_try_ifnet(so, tls, false);
@@ -970,6 +1024,25 @@ ktls_enable_tx(struct socket *so, struct tls_enable *en)
 }
 
 int
+ktls_get_rx_mode(struct socket *so)
+{
+	struct ktls_session *tls;
+	struct inpcb *inp;
+	int mode;
+
+	inp = so->so_pcb;
+	INP_WLOCK_ASSERT(inp);
+	SOCKBUF_LOCK(&so->so_rcv);
+	tls = so->so_rcv.sb_tls_info;
+	if (tls == NULL)
+		mode = TCP_TLS_MODE_NONE;
+	else
+		mode = tls->mode;
+	SOCKBUF_UNLOCK(&so->so_rcv);
+	return (mode);
+}
+
+int
 ktls_get_tx_mode(struct socket *so)
 {
 	struct ktls_session *tls;
diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index ada337426be8..dd86c9050924 100644
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -2372,12 +2372,34 @@ soreceive_stream(struct socket *so, struct sockaddr **psa, struct uio *uio,
 
 	sb = &so->so_rcv;
 
+#ifdef KERN_TLS
+	/*
+	 * KTLS store TLS records as records with a control message to
+	 * describe the framing.
+	 *
+	 * We check once here before acquiring locks to optimize the
+	 * common case.
+	 */
+	if (sb->sb_tls_info != NULL)
+		return (soreceive_generic(so, psa, uio, mp0, controlp,
+		    flagsp));
+#endif
+
 	/* Prevent other readers from entering the socket. */
 	error = sblock(sb, SBLOCKWAIT(flags));
 	if (error)
 		return (error);
 	SOCKBUF_LOCK(sb);
 
+#ifdef KERN_TLS
+	if (sb->sb_tls_info != NULL) {
+		SOCKBUF_UNLOCK(sb);
+		sbunlock(sb);
+		return (soreceive_generic(so, psa, uio, mp0, controlp,
+		    flagsp));
+	}
+#endif
+
 	/* Easy one, no space to copyout anything. */
 	if (uio->uio_resid == 0) {
 		error = EINVAL;
diff --git a/sys/netinet/tcp.h b/sys/netinet/tcp.h
index b5f01b3bcf21..5dc13eca217d 100644
--- a/sys/netinet/tcp.h
+++ b/sys/netinet/tcp.h
@@ -178,6 +178,8 @@ struct tcphdr {
 				   device */
 #define	TCP_TXTLS_ENABLE 39	/* TLS framing and encryption for transmit */
 #define	TCP_TXTLS_MODE	40	/* Transmit TLS mode */
+#define	TCP_RXTLS_ENABLE 41	/* TLS framing and encryption for receive */
+#define	TCP_RXTLS_MODE	42	/* Receive TLS mode */
 #define	TCP_CONGESTION	64	/* get/set congestion control algorithm */
 #define	TCP_CCALGOOPT	65	/* get/set cc algorithm specific options */
 #define TCP_DELACK  	72	/* socket option for delayed ack */
@@ -388,6 +390,7 @@ struct tcp_function_set {
  * TCP Control message types
  */
 #define	TLS_SET_RECORD_TYPE	1
+#define	TLS_GET_RECORD		2
 
 /*
  * TCP specific variables of interest for tp->t_stats stats(9) accounting.
diff --git a/sys/netinet/tcp_offload.c b/sys/netinet/tcp_offload.c
index 651a2ab6a845..ba190f0303f1 100644
--- a/sys/netinet/tcp_offload.c
+++ b/sys/netinet/tcp_offload.c
@@ -198,14 +198,15 @@ tcp_offload_tcp_info(struct tcpcb *tp, struct tcp_info *ti)
 }
 
 int
-tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls)
+tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls,
+    int direction)
 {
 	struct toedev *tod = tp->tod;
 
 	KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp));
 	INP_WLOCK_ASSERT(tp->t_inpcb);
 
-	return (tod->tod_alloc_tls_session(tod, tp, tls));
+	return (tod->tod_alloc_tls_session(tod, tp, tls, direction));
 }
 
 void
diff --git a/sys/netinet/tcp_offload.h b/sys/netinet/tcp_offload.h
index b89a367cd62e..19c120ccdd7d 100644
--- a/sys/netinet/tcp_offload.h
+++ b/sys/netinet/tcp_offload.h
@@ -46,7 +46,7 @@ int  tcp_offload_output(struct tcpcb *);
 void tcp_offload_rcvd(struct tcpcb *);
 void tcp_offload_ctloutput(struct tcpcb *, int, int);
 void tcp_offload_tcp_info(struct tcpcb *, struct tcp_info *);
-int  tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *);
+int  tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *, int);
 void tcp_offload_detach(struct tcpcb *);
 
 #endif
diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c
index 670345d62346..d1eb34e49ec4 100644
--- a/sys/netinet/tcp_usrreq.c
+++ b/sys/netinet/tcp_usrreq.c
@@ -2080,6 +2080,14 @@ unlock_and_done:
 			error = ktls_set_tx_mode(so, ui);
 			INP_WUNLOCK(inp);
 			break;
+		case TCP_RXTLS_ENABLE:
+			INP_WUNLOCK(inp);
+			error = sooptcopyin(sopt, &tls, sizeof(tls),
+			    sizeof(tls));
+			if (error)
+				break;
+			error = ktls_enable_rx(so, &tls);
+			break;
 #endif
 
 		case TCP_KEEPIDLE:
@@ -2418,6 +2426,11 @@ unhold:
 			INP_WUNLOCK(inp);
 			error = sooptcopyout(sopt, &optval, sizeof(optval));
 			break;
+		case TCP_RXTLS_MODE:
+			optval = ktls_get_rx_mode(so);
+			INP_WUNLOCK(inp);
+			error = sooptcopyout(sopt, &optval, sizeof(optval));
+			break;
 #endif
 		default:
 			INP_WUNLOCK(inp);
diff --git a/sys/netinet/toecore.c b/sys/netinet/toecore.c
index c3c40a8436a3..5b4e85cca0a3 100644
--- a/sys/netinet/toecore.c
+++ b/sys/netinet/toecore.c
@@ -193,7 +193,7 @@ toedev_tcp_info(struct toedev *tod __unused, struct tcpcb *tp __unused,
 
 static int
 toedev_alloc_tls_session(struct toedev *tod __unused, struct tcpcb *tp __unused,
-    struct ktls_session *tls __unused)
+    struct ktls_session *tls __unused, int direction __unused)
 {
 
 	return (EINVAL);
diff --git a/sys/netinet/toecore.h b/sys/netinet/toecore.h
index 6df89663fe7d..36493abf7149 100644
--- a/sys/netinet/toecore.h
+++ b/sys/netinet/toecore.h
@@ -113,7 +113,7 @@ struct toedev {
 
 	/* Create a TLS session */
 	int (*tod_alloc_tls_session)(struct toedev *, struct tcpcb *,
-	    struct ktls_session *);
+	    struct ktls_session *, int);
 };
 
 typedef	void (*tcp_offload_listen_start_fn)(void *, struct tcpcb *);
diff --git a/sys/sys/ktls.h b/sys/sys/ktls.h
index 94d5a976274a..bb7d41a7fa5c 100644
--- a/sys/sys/ktls.h
+++ b/sys/sys/ktls.h
@@ -98,7 +98,7 @@ struct tls_mac_data {
 #define	TLS_MINOR_VER_TWO	3	/* 3, 3 */
 #define	TLS_MINOR_VER_THREE	4	/* 3, 4 */
 
-/* For TCP_TXTLS_ENABLE */
+/* For TCP_TXTLS_ENABLE and TCP_RXTLS_ENABLE. */
 #ifdef _KERNEL
 struct tls_enable_v0 {
 	const uint8_t *cipher_key;
@@ -130,6 +130,17 @@ struct tls_enable {
 	uint8_t rec_seq[8];
 };
 
+/* Structure for TLS_GET_RECORD. */
+struct tls_get_record {
+	/* TLS record header. */
+	uint8_t  tls_type;
+	uint8_t  tls_vmajor;
+	uint8_t  tls_vminor;
+	uint16_t tls_length;
+};
+
+#ifdef _KERNEL
+
 struct tls_session_params {
 	uint8_t *cipher_key;
 	uint8_t *auth_key;
@@ -148,7 +159,9 @@ struct tls_session_params {
 	uint8_t flags;
 };
 
-#ifdef _KERNEL
+/* Used in APIs to request RX vs TX sessions. */
+#define	KTLS_TX		1
+#define	KTLS_RX		2
 
 #define	KTLS_API_VERSION 6
 
@@ -192,6 +205,7 @@ struct ktls_session {
 
 int ktls_crypto_backend_register(struct ktls_crypto_backend *be);
 int ktls_crypto_backend_deregister(struct ktls_crypto_backend *be);
+int ktls_enable_rx(struct socket *so, struct tls_enable *en);
 int ktls_enable_tx(struct socket *so, struct tls_enable *en);
 void ktls_destroy(struct ktls_session *tls);
 void ktls_frame(struct mbuf *m, struct ktls_session *tls, int *enqueue_cnt,
@@ -199,6 +213,7 @@ void ktls_frame(struct mbuf *m, struct ktls_session *tls, int *enqueue_cnt,
 void ktls_seq(struct sockbuf *sb, struct mbuf *m);
 void ktls_enqueue(struct mbuf *m, struct socket *so, int page_count);
 void ktls_enqueue_to_free(struct mbuf_ext_pgs *pgs);
+int ktls_get_rx_mode(struct socket *so);
 int ktls_set_tx_mode(struct socket *so, int mode);
 int ktls_get_tx_mode(struct socket *so);
 int ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls);