armv8crypto: add AES-GCM support

Add support for AES-GCM using OpenSSL's accelerated routines.

Reviewed by:	jhb
Differential Revision:	https://reviews.freebsd.org/D27454
Sponsored by:	Ampere Computing
Submitted by:	Klara, Inc.
Approved by:	re (kib)

(cherry picked from commit ed9b7f4414663703e0e9581a730c4bdfaca5687f)

5 files changed, 445 insertions, 75 deletions

diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
index fb001f95a672..f7003b1048c8 100644
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -279,6 +279,16 @@ armv8_crypto_wrap.o		optional	armv8crypto		\
 	compile-with	"${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8/ ${WERROR} ${NO_WCAST_QUAL} ${PROF} -march=armv8-a+crypto ${.IMPSRC}" \
 	no-implicit-rule						\
 	clean		"armv8_crypto_wrap.o"
+aesv8-armx.o		optional	armv8crypto			\
+	dependency	"$S/crypto/openssl/aarch64/aesv8-armx.S"	\
+	compile-with	"${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8/ -I$S/crypto/openssl/crypto ${WERROR} ${NO_WCAST_QUAL} ${PROF} -march=armv8-a+crypto ${.IMPSRC}" \
+	no-implicit-rule						\
+	clean		"aesv8-armx.o"
+ghashv8-armx.o		optional	armv8crypto			\
+	dependency	"$S/crypto/openssl/aarch64/ghashv8-armx.S"	\
+	compile-with	"${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8/ -I$S/crypto/openssl/crypto ${WERROR} ${NO_WCAST_QUAL} ${PROF} -march=armv8-a+crypto ${.IMPSRC}" \
+	no-implicit-rule						\
+	clean		"ghashv8-armx.o"
 crypto/des/des_enc.c		optional	netsmb
 crypto/openssl/ossl_aarch64.c	optional	ossl
 crypto/openssl/aarch64/sha1-armv8.S	optional ossl		\
diff --git a/sys/crypto/armv8/armv8_crypto.c b/sys/crypto/armv8/armv8_crypto.c
index f296337e6f33..be39168d50f3 100644
--- a/sys/crypto/armv8/armv8_crypto.c
+++ b/sys/crypto/armv8/armv8_crypto.c
@@ -2,6 +2,7 @@
  * Copyright (c) 2005-2008 Pawel Jakub Dawidek <[email protected]>
  * Copyright (c) 2010 Konstantin Belousov <[email protected]>
  * Copyright (c) 2014,2016 The FreeBSD Foundation
+ * Copyright (c) 2020 Ampere Computing
  * All rights reserved.
  *
  * Portions of this software were developed by John-Mark Gurney
@@ -58,6 +59,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/vfp.h>
 
 #include <opencrypto/cryptodev.h>
+#include <opencrypto/gmac.h>
 #include <cryptodev_if.h>
 #include <crypto/armv8/armv8_crypto.h>
 #include <crypto/rijndael/rijndael.h>
@@ -66,6 +68,7 @@ struct armv8_crypto_softc {
 	int		dieing;
 	int32_t		cid;
 	struct rwlock	lock;
+	bool		has_pmul;
 };
 
 static struct mtx *ctx_mtx;
@@ -109,16 +112,20 @@ armv8_crypto_probe(device_t dev)
 
 	switch (ID_AA64ISAR0_AES_VAL(reg)) {
 	case ID_AA64ISAR0_AES_BASE:
+		ret = 0;
+		device_set_desc(dev, "AES-CBC,AES-XTS");
+		break;
 	case ID_AA64ISAR0_AES_PMULL:
 		ret = 0;
+		device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM");
+		break;
+	default:
 		break;
 	case ID_AA64ISAR0_AES_NONE:
 		device_printf(dev, "CPU lacks AES instructions");
 		break;
 	}
 
-	device_set_desc_copy(dev, "AES-CBC,AES-XTS");
-
 	/* TODO: Check more fields as we support more features */
 
 	return (ret);
@@ -128,11 +135,17 @@ static int
 armv8_crypto_attach(device_t dev)
 {
 	struct armv8_crypto_softc *sc;
+	uint64_t reg;
 	int i;
 
 	sc = device_get_softc(dev);
 	sc->dieing = 0;
 
+	reg = READ_SPECIALREG(id_aa64isar0_el1);
+
+	if (ID_AA64ISAR0_AES_VAL(reg) == ID_AA64ISAR0_AES_PMULL)
+		sc->has_pmul = true;
+
 	sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session),
 	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE);
 	if (sc->cid < 0) {
@@ -185,14 +198,43 @@ armv8_crypto_detach(device_t dev)
 	return (0);
 }
 
+#define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)
+
 static int
 armv8_crypto_probesession(device_t dev,
     const struct crypto_session_params *csp)
 {
+	struct armv8_crypto_softc *sc;
+
+	sc = device_get_softc(dev);
 
-	if (csp->csp_flags != 0)
+	if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
 		return (EINVAL);
+
 	switch (csp->csp_mode) {
+	case CSP_MODE_AEAD:
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_AES_NIST_GCM_16:
+			if (!sc->has_pmul)
+				return (EINVAL);
+			if (csp->csp_ivlen != AES_GCM_IV_LEN)
+				return (EINVAL);
+			if (csp->csp_auth_mlen != 0 &&
+			    csp->csp_auth_mlen != GMAC_DIGEST_LEN)
+				return (EINVAL);
+			switch (csp->csp_cipher_klen * 8) {
+			case 128:
+			case 192:
+			case 256:
+				break;
+			default:
+				return (EINVAL);
+			}
+			break;
+		default:
+			return (EINVAL);
+		}
+		break;
 	case CSP_MODE_CIPHER:
 		switch (csp->csp_cipher_alg) {
 		case CRYPTO_AES_CBC:
@@ -228,40 +270,55 @@ armv8_crypto_probesession(device_t dev,
 	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
 }
 
-static void
+static int
 armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
     const struct crypto_session_params *csp, const uint8_t *key, int keylen)
 {
-	int i;
+	__uint128_val_t H;
+	struct fpu_kern_ctx *ctx;
+	int kt, i;
 
 	if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
 		keylen /= 2;
 
 	switch (keylen * 8) {
 	case 128:
-		ses->rounds = AES128_ROUNDS;
-		break;
 	case 192:
-		ses->rounds = AES192_ROUNDS;
-		break;
 	case 256:
-		ses->rounds = AES256_ROUNDS;
 		break;
 	default:
-		panic("invalid AES key length");
+		return (EINVAL);
 	}
 
-	rijndaelKeySetupEnc(ses->enc_schedule, key, keylen * 8);
-	rijndaelKeySetupDec(ses->dec_schedule, key, keylen * 8);
-	if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
-		rijndaelKeySetupEnc(ses->xts_schedule, key + keylen, keylen * 8);
+	kt = is_fpu_kern_thread(0);
+	if (!kt) {
+		AQUIRE_CTX(i, ctx);
+		fpu_kern_enter(curthread, ctx,
+		    FPU_KERN_NORMAL | FPU_KERN_KTHR);
+	}
+
+	aes_v8_set_encrypt_key(key,
+	    keylen * 8, &ses->enc_schedule);
+
+	if ((csp->csp_cipher_alg == CRYPTO_AES_XTS) ||
+	    (csp->csp_cipher_alg == CRYPTO_AES_CBC))
+		aes_v8_set_decrypt_key(key,
+		    keylen * 8, &ses->dec_schedule);
+
+	if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
+		memset(H.c, 0, sizeof(H.c));
+		aes_v8_encrypt(H.c, H.c, &ses->enc_schedule);
+		H.u[0] = bswap64(H.u[0]);
+		H.u[1] = bswap64(H.u[1]);
+		gcm_init_v8(ses->Htable, H.u);
+	}
 
-	for (i = 0; i < nitems(ses->enc_schedule); i++) {
-		ses->enc_schedule[i] = bswap32(ses->enc_schedule[i]);
-		ses->dec_schedule[i] = bswap32(ses->dec_schedule[i]);
-		if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
-			ses->xts_schedule[i] = bswap32(ses->xts_schedule[i]);
+	if (!kt) {
+		fpu_kern_leave(curthread, ctx);
+		RELEASE_CTX(i, ctx);
 	}
+
+	return (0);
 }
 
 static int
@@ -270,6 +327,7 @@ armv8_crypto_newsession(device_t dev, crypto_session_t cses,
 {
 	struct armv8_crypto_softc *sc;
 	struct armv8_crypto_session *ses;
+	int error;
 
 	sc = device_get_softc(dev);
 	rw_wlock(&sc->lock);
@@ -279,40 +337,29 @@ armv8_crypto_newsession(device_t dev, crypto_session_t cses,
 	}
 
 	ses = crypto_get_driver_session(cses);
-	armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key,
+	error = armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key,
 	    csp->csp_cipher_klen);
 	rw_wunlock(&sc->lock);
-	return (0);
+	return (error);
 }
 
 static int
 armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused)
 {
 	struct armv8_crypto_session *ses;
-	int error;
-
-	/* We can only handle full blocks for now */
-	if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
-		error = EINVAL;
-		goto out;
-	}
 
 	ses = crypto_get_driver_session(crp->crp_session);
-	error = armv8_crypto_cipher_process(ses, crp);
-
-out:
-	crp->crp_etype = error;
+	crp->crp_etype = armv8_crypto_cipher_process(ses, crp);
 	crypto_done(crp);
 	return (0);
 }
 
 static uint8_t *
-armv8_crypto_cipher_alloc(struct cryptop *crp, int *allocated)
+armv8_crypto_cipher_alloc(struct cryptop *crp, int start, int length, int *allocated)
 {
 	uint8_t *addr;
 
-	addr = crypto_contiguous_subsegment(crp, crp->crp_payload_start,
-	    crp->crp_payload_length);
+	addr = crypto_contiguous_subsegment(crp, start, length);
 	if (addr != NULL) {
 		*allocated = 0;
 		return (addr);
@@ -320,8 +367,7 @@ armv8_crypto_cipher_alloc(struct cryptop *crp, int *allocated)
 	addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT);
 	if (addr != NULL) {
 		*allocated = 1;
-		crypto_copydata(crp, crp->crp_payload_start,
-		    crp->crp_payload_length, addr);
+		crypto_copydata(crp, start, length, addr);
 	} else
 		*allocated = 0;
 	return (addr);
@@ -333,19 +379,63 @@ armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
 {
 	const struct crypto_session_params *csp;
 	struct fpu_kern_ctx *ctx;
-	uint8_t *buf;
-	uint8_t iv[AES_BLOCK_LEN];
-	int allocated, i;
+	uint8_t *buf, *authbuf, *outbuf;
+	uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN];
+	int allocated, authallocated, outallocated, i;
 	int encflag;
 	int kt;
+	int error;
+	bool outcopy;
 
 	csp = crypto_get_params(crp->crp_session);
 	encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
 
-	buf = armv8_crypto_cipher_alloc(crp, &allocated);
+	allocated = 0;
+	outallocated = 0;
+	authallocated = 0;
+	authbuf = NULL;
+	kt = 1;
+
+	buf = armv8_crypto_cipher_alloc(crp, crp->crp_payload_start,
+	    crp->crp_payload_length, &allocated);
 	if (buf == NULL)
 		return (ENOMEM);
 
+	if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
+		if (crp->crp_aad != NULL)
+			authbuf = crp->crp_aad;
+		else
+			authbuf = armv8_crypto_cipher_alloc(crp, crp->crp_aad_start,
+			    crp->crp_aad_length, &authallocated);
+		if (authbuf == NULL) {
+			error = ENOMEM;
+			goto out;
+		}
+	}
+
+	if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
+		outbuf = crypto_buffer_contiguous_subsegment(&crp->crp_obuf,
+		    crp->crp_payload_output_start, crp->crp_payload_length);
+		if (outbuf == NULL) {
+			outcopy = true;
+			if (allocated)
+				outbuf = buf;
+			else {
+				outbuf = malloc(crp->crp_payload_length,
+				    M_ARMV8_CRYPTO, M_NOWAIT);
+				if (outbuf == NULL) {
+					error = ENOMEM;
+					goto out;
+				}
+				outallocated = true;
+			}
+		} else
+			outcopy = false;
+	} else {
+		outbuf = buf;
+		outcopy = allocated;
+	}
+
 	kt = is_fpu_kern_thread(0);
 	if (!kt) {
 		AQUIRE_CTX(i, ctx);
@@ -363,36 +453,74 @@ armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
 	/* Do work */
 	switch (csp->csp_cipher_alg) {
 	case CRYPTO_AES_CBC:
+		if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
+			error = EINVAL;
+			goto out;
+		}
 		if (encflag)
-			armv8_aes_encrypt_cbc(ses->rounds, ses->enc_schedule,
+			armv8_aes_encrypt_cbc(&ses->enc_schedule,
 			    crp->crp_payload_length, buf, buf, iv);
 		else
-			armv8_aes_decrypt_cbc(ses->rounds, ses->dec_schedule,
+			armv8_aes_decrypt_cbc(&ses->dec_schedule,
 			    crp->crp_payload_length, buf, iv);
 		break;
 	case CRYPTO_AES_XTS:
 		if (encflag)
-			armv8_aes_encrypt_xts(ses->rounds, ses->enc_schedule,
-			    ses->xts_schedule, crp->crp_payload_length, buf,
+			armv8_aes_encrypt_xts(&ses->enc_schedule,
+			    &ses->xts_schedule.aes_key, crp->crp_payload_length, buf,
 			    buf, iv);
 		else
-			armv8_aes_decrypt_xts(ses->rounds, ses->dec_schedule,
-			    ses->xts_schedule, crp->crp_payload_length, buf,
+			armv8_aes_decrypt_xts(&ses->dec_schedule,
+			    &ses->xts_schedule.aes_key, crp->crp_payload_length, buf,
 			    buf, iv);
 		break;
+	case CRYPTO_AES_NIST_GCM_16:
+		if (encflag) {
+			memset(tag, 0, sizeof(tag));
+			armv8_aes_encrypt_gcm(&ses->enc_schedule,
+			    crp->crp_payload_length,
+			    buf, outbuf,
+			    crp->crp_aad_length, authbuf,
+			    tag, iv, ses->Htable);
+			crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
+			    tag);
+		} else {
+			crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
+			    tag);
+			if (armv8_aes_decrypt_gcm(&ses->enc_schedule,
+			    crp->crp_payload_length,
+			    buf, outbuf,
+			    crp->crp_aad_length, authbuf,
+			    tag, iv, ses->Htable) != 0) {
+				error = EBADMSG;
+				goto out;
+			}
+		}
+		break;
 	}
 
-	if (allocated)
-		crypto_copyback(crp, crp->crp_payload_start,
-		    crp->crp_payload_length, buf);
+	if (outcopy)
+		crypto_copyback(crp, CRYPTO_HAS_OUTPUT_BUFFER(crp) ?
+		    crp->crp_payload_output_start : crp->crp_payload_start,
+		    crp->crp_payload_length, outbuf);
 
+	error = 0;
+out:
 	if (!kt) {
 		fpu_kern_leave(curthread, ctx);
 		RELEASE_CTX(i, ctx);
 	}
+
 	if (allocated)
 		zfree(buf, M_ARMV8_CRYPTO);
-	return (0);
+	if (authallocated)
+		zfree(authbuf, M_ARMV8_CRYPTO);
+	if (outallocated)
+		zfree(outbuf, M_ARMV8_CRYPTO);
+	explicit_bzero(iv, sizeof(iv));
+	explicit_bzero(tag, sizeof(tag));
+
+	return (error);
 }
 
 static device_method_t armv8_crypto_methods[] = {
diff --git a/sys/crypto/armv8/armv8_crypto.h b/sys/crypto/armv8/armv8_crypto.h
index 2d0be163b072..0e4cd91e7062 100644
--- a/sys/crypto/armv8/armv8_crypto.h
+++ b/sys/crypto/armv8/armv8_crypto.h
@@ -32,27 +32,56 @@
 #ifndef _ARMV8_CRYPTO_H_
 #define _ARMV8_CRYPTO_H_
 
-#define	AES128_ROUNDS	10
-#define	AES192_ROUNDS	12
 #define	AES256_ROUNDS	14
 #define	AES_SCHED_LEN	((AES256_ROUNDS + 1) * AES_BLOCK_LEN)
 
+typedef struct {
+	uint32_t		aes_key[AES_SCHED_LEN/4];
+	int			aes_rounds;
+} AES_key_t;
+
+typedef union {
+		uint64_t u[2];
+		uint32_t d[4];
+		uint8_t c[16];
+		size_t t[16 / sizeof(size_t)];
+} __uint128_val_t;
+
 struct armv8_crypto_session {
-	uint32_t enc_schedule[AES_SCHED_LEN/4];
-	uint32_t dec_schedule[AES_SCHED_LEN/4];
-	uint32_t xts_schedule[AES_SCHED_LEN/4];
-	int algo;
-	int rounds;
+	AES_key_t enc_schedule;
+	AES_key_t dec_schedule;
+	AES_key_t xts_schedule;
+	__uint128_val_t Htable[16];
 };
 
-void armv8_aes_encrypt_cbc(int, const void *, size_t, const uint8_t *,
+/* Prototypes for aesv8-armx.S */
+void aes_v8_encrypt(uint8_t *in, uint8_t *out, const AES_key_t *key);
+int aes_v8_set_encrypt_key(const unsigned char *userKey, const int bits, const AES_key_t *key);
+int aes_v8_set_decrypt_key(const unsigned char *userKey, const int bits, const AES_key_t *key);
+
+/* Prototypes for ghashv8-armx.S */
+void gcm_init_v8(__uint128_val_t Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_v8(uint64_t Xi[2], const __uint128_val_t Htable[16]);
+void gcm_ghash_v8(uint64_t Xi[2], const __uint128_val_t Htable[16], const uint8_t *inp, size_t len);
+
+void armv8_aes_encrypt_cbc(const AES_key_t *, size_t, const uint8_t *,
     uint8_t *, const uint8_t[static AES_BLOCK_LEN]);
-void armv8_aes_decrypt_cbc(int, const void *, size_t, uint8_t *,
+void armv8_aes_decrypt_cbc(const AES_key_t *, size_t, uint8_t *,
     const uint8_t[static AES_BLOCK_LEN]);
+void armv8_aes_encrypt_gcm(AES_key_t *, size_t, const uint8_t *,
+    uint8_t *, size_t, const uint8_t*,
+    uint8_t tag[static GMAC_DIGEST_LEN],
+    const uint8_t[static AES_BLOCK_LEN],
+    const __uint128_val_t *);
+int armv8_aes_decrypt_gcm(AES_key_t *, size_t, const uint8_t *,
+    uint8_t *, size_t, const uint8_t*,
+    const uint8_t tag[static GMAC_DIGEST_LEN],
+    const uint8_t[static AES_BLOCK_LEN],
+    const __uint128_val_t *);
 
-void armv8_aes_encrypt_xts(int, const void *, const void *, size_t,
+void armv8_aes_encrypt_xts(AES_key_t *, const void *, size_t,
     const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]);
-void armv8_aes_decrypt_xts(int, const void *, const void *, size_t,
+void armv8_aes_decrypt_xts(AES_key_t *, const void *, size_t,
     const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]);
 
 #endif /* _ARMV8_CRYPTO_H_ */
diff --git a/sys/crypto/armv8/armv8_crypto_wrap.c b/sys/crypto/armv8/armv8_crypto_wrap.c
index 83af3fad40ef..ea93f1b9a176 100644
--- a/sys/crypto/armv8/armv8_crypto_wrap.c
+++ b/sys/crypto/armv8/armv8_crypto_wrap.c
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2016 The FreeBSD Foundation
+ * Copyright (c) 2020 Ampere Computing
  * All rights reserved.
  *
  * This software was developed by Andrew Turner under
@@ -41,6 +42,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/queue.h>
 
 #include <opencrypto/cryptodev.h>
+#include <opencrypto/gmac.h>
+#include <crypto/rijndael/rijndael.h>
 #include <crypto/armv8/armv8_crypto.h>
 
 #include <arm_neon.h>
@@ -90,7 +93,7 @@ armv8_aes_dec(int rounds, const uint8x16_t *keysched, const uint8x16_t from)
 }
 
 void
-armv8_aes_encrypt_cbc(int rounds, const void *key_schedule, size_t len,
+armv8_aes_encrypt_cbc(const AES_key_t *key, size_t len,
     const uint8_t *from, uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN])
 {
 	uint8x16_t tot, ivreg, tmp;
@@ -100,8 +103,8 @@ armv8_aes_encrypt_cbc(int rounds, const void *key_schedule, size_t len,
 	ivreg = vld1q_u8(iv);
 	for (i = 0; i < len; i++) {
 		tmp = vld1q_u8(from);
-		tot = armv8_aes_enc(rounds - 1, key_schedule,
-		    veorq_u8(tmp, ivreg));
+		tot = armv8_aes_enc(key->aes_rounds - 1,
+		    (const void*)key->aes_key, veorq_u8(tmp, ivreg));
 		ivreg = tot;
 		vst1q_u8(to, tot);
 		from += AES_BLOCK_LEN;
@@ -110,7 +113,7 @@ armv8_aes_encrypt_cbc(int rounds, const void *key_schedule, size_t len,
 }
 
 void
-armv8_aes_decrypt_cbc(int rounds, const void *key_schedule, size_t len,
+armv8_aes_decrypt_cbc(const AES_key_t *key, size_t len,
     uint8_t *buf, const uint8_t iv[static AES_BLOCK_LEN])
 {
 	uint8x16_t ivreg, nextiv, tmp;
@@ -120,7 +123,8 @@ armv8_aes_decrypt_cbc(int rounds, const void *key_schedule, size_t len,
 	ivreg = vld1q_u8(iv);
 	for (i = 0; i < len; i++) {
 		nextiv = vld1q_u8(buf);
-		tmp = armv8_aes_dec(rounds - 1, key_schedule, nextiv);
+		tmp = armv8_aes_dec(key->aes_rounds - 1,
+		    (const void*)key->aes_key, nextiv);
 		vst1q_u8(buf, veorq_u8(tmp, ivreg));
 		ivreg = nextiv;
 		buf += AES_BLOCK_LEN;
@@ -200,21 +204,203 @@ armv8_aes_crypt_xts(int rounds, const uint8x16_t *data_schedule,
 }
 
 void
-armv8_aes_encrypt_xts(int rounds, const void *data_schedule,
+armv8_aes_encrypt_xts(AES_key_t *data_schedule,
     const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
     const uint8_t iv[static AES_BLOCK_LEN])
 {
 
-	armv8_aes_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to,
-	    iv, 1);
+	armv8_aes_crypt_xts(data_schedule->aes_rounds,
+	    (const void *)&data_schedule->aes_key, tweak_schedule, len, from,
+	    to, iv, 1);
 }
 
 void
-armv8_aes_decrypt_xts(int rounds, const void *data_schedule,
+armv8_aes_decrypt_xts(AES_key_t *data_schedule,
     const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
     const uint8_t iv[static AES_BLOCK_LEN])
 {
 
-	armv8_aes_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to,
-	    iv, 0);
+	armv8_aes_crypt_xts(data_schedule->aes_rounds,
+	    (const void *)&data_schedule->aes_key, tweak_schedule, len, from,
+	    to,iv, 0);
+
+}
+
+#define	AES_INC_COUNTER(counter)				\
+	do {							\
+		for (int pos = AES_BLOCK_LEN - 1;		\
+		     pos >= 0; pos--)				\
+			if (++(counter)[pos])			\
+				break;				\
+	} while (0)
+
+void
+armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len,
+    const uint8_t *from, uint8_t *to,
+    size_t authdatalen, const uint8_t *authdata,
+    uint8_t tag[static GMAC_DIGEST_LEN],
+    const uint8_t iv[static AES_GCM_IV_LEN],
+    const __uint128_val_t *Htable)
+{
+	size_t i;
+	const uint64_t *from64;
+	uint64_t *to64;
+	uint8_t aes_counter[AES_BLOCK_LEN];
+	uint8_t block[AES_BLOCK_LEN];
+	size_t trailer;
+	__uint128_val_t EK0, EKi, Xi, lenblock;
+
+	bzero(&aes_counter, AES_BLOCK_LEN);
+	memcpy(aes_counter, iv, AES_GCM_IV_LEN);
+
+	/* Setup the counter */
+	aes_counter[AES_BLOCK_LEN - 1] = 1;
+
+	/* EK0 for a final GMAC round */
+	aes_v8_encrypt(aes_counter, EK0.c, aes_key);
+
+	/* GCM starts with 2 as counter, 1 is used for final xor of tag. */
+	aes_counter[AES_BLOCK_LEN - 1] = 2;
+
+	memset(Xi.c, 0, sizeof(Xi.c));
+	memset(block, 0, sizeof(block));
+	memcpy(block, authdata, min(authdatalen, sizeof(block)));
+	gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN);
+
+	from64 = (const uint64_t*)from;
+	to64 = (uint64_t*)to;
+	trailer = len % AES_BLOCK_LEN;
+
+	for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) {
+		aes_v8_encrypt(aes_counter, EKi.c, aes_key);
+		AES_INC_COUNTER(aes_counter);
+		to64[0] = from64[0] ^ EKi.u[0];
+		to64[1] = from64[1] ^ EKi.u[1];
+		gcm_ghash_v8(Xi.u, Htable, (uint8_t*)to64, AES_BLOCK_LEN);
+
+		to64 += 2;
+		from64 += 2;
+	}
+
+	to += (len - trailer);
+	from += (len - trailer);
+
+	if (trailer) {
+		aes_v8_encrypt(aes_counter, EKi.c, aes_key);
+		AES_INC_COUNTER(aes_counter);
+		for (i = 0; i < trailer; i++) {
+			block[i] = to[i] = from[i] ^ EKi.c[i % AES_BLOCK_LEN];
+		}
+
+		for (; i < AES_BLOCK_LEN; i++)
+			block[i] = 0;
+
+		gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN);
+	}
+
+	/* Lengths block */
+	lenblock.u[0] = lenblock.u[1] = 0;
+	lenblock.d[1] = htobe32(authdatalen * 8);
+	lenblock.d[3] = htobe32(len * 8);
+	gcm_ghash_v8(Xi.u, Htable, lenblock.c, AES_BLOCK_LEN);
+
+	Xi.u[0] ^= EK0.u[0];
+	Xi.u[1] ^= EK0.u[1];
+	memcpy(tag, Xi.c, GMAC_DIGEST_LEN);
+
+	explicit_bzero(aes_counter, sizeof(aes_counter));
+	explicit_bzero(Xi.c, sizeof(Xi.c));
+	explicit_bzero(EK0.c, sizeof(EK0.c));
+	explicit_bzero(EKi.c, sizeof(EKi.c));
+	explicit_bzero(lenblock.c, sizeof(lenblock.c));
+}
+
+int
+armv8_aes_decrypt_gcm(AES_key_t *aes_key, size_t len,
+    const uint8_t *from, uint8_t *to,
+    size_t authdatalen, const uint8_t *authdata,
+    const uint8_t tag[static GMAC_DIGEST_LEN],
+    const uint8_t iv[static AES_GCM_IV_LEN],
+    const __uint128_val_t *Htable)
+{
+	size_t i;
+	const uint64_t *from64;
+	uint64_t *to64;
+	uint8_t aes_counter[AES_BLOCK_LEN];
+	uint8_t block[AES_BLOCK_LEN];
+	size_t trailer;
+	__uint128_val_t EK0, EKi, Xi, lenblock;
+	int error;
+
+	error = 0;
+	bzero(&aes_counter, AES_BLOCK_LEN);
+	memcpy(aes_counter, iv, AES_GCM_IV_LEN);
+
+	/* Setup the counter */
+	aes_counter[AES_BLOCK_LEN - 1] = 1;
+
+	/* EK0 for a final GMAC round */
+	aes_v8_encrypt(aes_counter, EK0.c, aes_key);
+
+	memset(Xi.c, 0, sizeof(Xi.c));
+	memset(block, 0, sizeof(block));
+	memcpy(block, authdata, min(authdatalen, sizeof(block)));
+	gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN);
+	trailer = len % AES_BLOCK_LEN;
+	gcm_ghash_v8(Xi.u, Htable, from, len - trailer);
+
+	if (trailer) {
+		for (i = 0; i < trailer; i++)
+			block[i] = from[len - trailer + i];
+		for (; i < AES_BLOCK_LEN; i++)
+			block[i] = 0;
+		gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN);
+	}
+
+	/* Lengths block */
+	lenblock.u[0] = lenblock.u[1] = 0;
+	lenblock.d[1] = htobe32(authdatalen * 8);
+	lenblock.d[3] = htobe32(len * 8);
+	gcm_ghash_v8(Xi.u, Htable, lenblock.c, AES_BLOCK_LEN);
+
+	Xi.u[0] ^= EK0.u[0];
+	Xi.u[1] ^= EK0.u[1];
+	if (timingsafe_bcmp(tag, Xi.c, GMAC_DIGEST_LEN) != 0) {
+		error = EBADMSG;
+		goto out;
+	}
+
+	/* GCM starts with 2 as counter, 1 is used for final xor of tag. */
+	aes_counter[AES_BLOCK_LEN - 1] = 2;
+
+	from64 = (const uint64_t*)from;
+	to64 = (uint64_t*)to;
+
+	for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) {
+		aes_v8_encrypt(aes_counter, EKi.c, aes_key);
+		AES_INC_COUNTER(aes_counter);
+		to64[0] = from64[0] ^ EKi.u[0];
+		to64[1] = from64[1] ^ EKi.u[1];
+		to64 += 2;
+		from64 += 2;
+	}
+
+	to += (len - trailer);
+	from += (len - trailer);
+
+	if (trailer) {
+		aes_v8_encrypt(aes_counter, EKi.c, aes_key);
+		AES_INC_COUNTER(aes_counter);
+		for (i = 0; i < trailer; i++)
+			to[i] = from[i] ^ EKi.c[i % AES_BLOCK_LEN];
+	}
+
+out:
+	explicit_bzero(aes_counter, sizeof(aes_counter));
+	explicit_bzero(Xi.c, sizeof(Xi.c));
+	explicit_bzero(EK0.c, sizeof(EK0.c));
+	explicit_bzero(EKi.c, sizeof(EKi.c));
+	explicit_bzero(lenblock.c, sizeof(lenblock.c));
+
+	return (error);
 }
diff --git a/sys/modules/armv8crypto/Makefile b/sys/modules/armv8crypto/Makefile
index 4b9a84c953d2..bd38fe039fc9 100644
--- a/sys/modules/armv8crypto/Makefile
+++ b/sys/modules/armv8crypto/Makefile
@@ -1,12 +1,13 @@
 # $FreeBSD$
 
 .PATH: ${SRCTOP}/sys/crypto/armv8
+.PATH: ${SRCTOP}/sys/crypto/openssl/aarch64
 
 KMOD=	armv8crypto
 SRCS=	armv8_crypto.c
 SRCS+=	device_if.h bus_if.h opt_bus.h cryptodev_if.h
 
-OBJS+=	armv8_crypto_wrap.o
+OBJS+=	armv8_crypto_wrap.o aesv8-armx.o ghashv8-armx.o
 
 # Remove -nostdinc so we can get the intrinsics.
 armv8_crypto_wrap.o: armv8_crypto_wrap.c
@@ -16,6 +17,22 @@ armv8_crypto_wrap.o: armv8_crypto_wrap.c
 	     -march=armv8-a+crypto ${.IMPSRC}
 	${CTFCONVERT_CMD}
 
+aesv8-armx.o: aesv8-armx.S
+	${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} \
+	    -I${SRCTOP}/sys/crypto/armv8 \
+	    -I${SRCTOP}/sys/crypto/openssl/crypto \
+	    ${WERROR} ${PROF} \
+	    -march=armv8-a+crypto ${.IMPSRC}
+	${CTFCONVERT_CMD}
+
+ghashv8-armx.o: ghashv8-armx.S
+	${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} \
+	    -I${SRCTOP}/sys/crypto/armv8 \
+	    -I${SRCTOP}/sys/crypto/openssl/crypto \
+	    ${WERROR} ${PROF} \
+	    -march=armv8-a+crypto ${.IMPSRC}
+	${CTFCONVERT_CMD}
+
 armv8_crypto_wrap.o: armv8_crypto.h
 
 .include <bsd.kmod.mk>