66 lines
2.7 KiB
Diff
66 lines
2.7 KiB
Diff
From 5e5a835c50afc3b9bb2b8b9175d0924abb5a7f3c Mon Sep 17 00:00:00 2001
|
|
From: Eric Biggers <ebiggers@google.com>
|
|
Date: Mon, 27 Jan 2025 13:16:09 -0800
|
|
Subject: crypto: x86/aes-xts - make the fast path 64-bit specific
|
|
|
|
Remove 32-bit support from the fast path in xts_crypt(). Then optimize
|
|
it for 64-bit, and simplify the code, by switching to sg_virt() and
|
|
removing the now-unnecessary checks for crossing a page boundary.
|
|
|
|
The result is simpler code that is slightly smaller and faster in the
|
|
case that actually matters (64-bit).
|
|
|
|
Signed-off-by: Eric Biggers <ebiggers@google.com>
|
|
---
|
|
arch/x86/crypto/aesni-intel_glue.c | 30 ++++++++++--------------------
|
|
1 file changed, 10 insertions(+), 20 deletions(-)
|
|
|
|
--- a/arch/x86/crypto/aesni-intel_glue.c
|
|
+++ b/arch/x86/crypto/aesni-intel_glue.c
|
|
@@ -581,11 +581,8 @@ xts_crypt(struct skcipher_request *req,
|
|
{
|
|
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
|
|
const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
|
|
- const unsigned int cryptlen = req->cryptlen;
|
|
- struct scatterlist *src = req->src;
|
|
- struct scatterlist *dst = req->dst;
|
|
|
|
- if (unlikely(cryptlen < AES_BLOCK_SIZE))
|
|
+ if (unlikely(req->cryptlen < AES_BLOCK_SIZE))
|
|
return -EINVAL;
|
|
|
|
kernel_fpu_begin();
|
|
@@ -593,23 +590,16 @@ xts_crypt(struct skcipher_request *req,
|
|
|
|
/*
|
|
* In practice, virtually all XTS plaintexts and ciphertexts are either
|
|
- * 512 or 4096 bytes, aligned such that they don't span page boundaries.
|
|
- * To optimize the performance of these cases, and also any other case
|
|
- * where no page boundary is spanned, the below fast-path handles
|
|
- * single-page sources and destinations as efficiently as possible.
|
|
+ * 512 or 4096 bytes and do not use multiple scatterlist elements. To
|
|
+ * optimize the performance of these cases, the below fast-path handles
|
|
+ * single-scatterlist-element messages as efficiently as possible. The
|
|
+ * code is 64-bit specific, as it assumes no page mapping is needed.
|
|
*/
|
|
- if (likely(src->length >= cryptlen && dst->length >= cryptlen &&
|
|
- src->offset + cryptlen <= PAGE_SIZE &&
|
|
- dst->offset + cryptlen <= PAGE_SIZE)) {
|
|
- struct page *src_page = sg_page(src);
|
|
- struct page *dst_page = sg_page(dst);
|
|
- void *src_virt = kmap_local_page(src_page) + src->offset;
|
|
- void *dst_virt = kmap_local_page(dst_page) + dst->offset;
|
|
-
|
|
- (*crypt_func)(&ctx->crypt_ctx, src_virt, dst_virt, cryptlen,
|
|
- req->iv);
|
|
- kunmap_local(dst_virt);
|
|
- kunmap_local(src_virt);
|
|
+ if (IS_ENABLED(CONFIG_X86_64) &&
|
|
+ likely(req->src->length >= req->cryptlen &&
|
|
+ req->dst->length >= req->cryptlen)) {
|
|
+ (*crypt_func)(&ctx->crypt_ctx, sg_virt(req->src),
|
|
+ sg_virt(req->dst), req->cryptlen, req->iv);
|
|
kernel_fpu_end();
|
|
return 0;
|
|
}
|