From patchwork Sun Nov 17 00:22:38 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Eric Biggers X-Patchwork-Id: 13877759 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1241117579; Sun, 17 Nov 2024 00:23:55 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1731803036; cv=none; b=jTMrz/2euc1z2OdUexHccHG+Z8rFIi0ifukcAs7yYH8etMyPhxPV1MSuGVXWNSwB1ljniFypmh7NQ4zqaXg7dbpNoQ4zY8wBoKKZLujoxyP+K24gW33AVO8cENUYxoC/fXsfAXvubvZRfkqXwZSZrckOHkJKNxS8gmSfVPR8Yzg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1731803036; c=relaxed/simple; bh=HgR1LEY3VEIzOlaLOqyuvwljcBVjiL79uYLCzgDKDDY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=hkU8Yd7JgyhF54DX4sq8cmIiYX36fufccps0B/7xwUO3xpPAsy7CsbvGadcucPzdCFZaxdti6xnSw0FHOXAbw7fA72rfL6KfAZdXg6Ve+S0oLimquDtX2Jv8o+4whYsqKEiEKLVeE2an5ANG/Nz6ZhCTmyY9jr4pSUN6DqasE3I= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=m17zfdcU; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="m17zfdcU" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 702A8C4CEC3; Sun, 17 Nov 2024 00:23:55 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1731803035; bh=HgR1LEY3VEIzOlaLOqyuvwljcBVjiL79uYLCzgDKDDY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=m17zfdcUmnOLe8DQbOyMMXD3CE0kIL6HKxvxjId6ZFnU5PJfVjkVnxBFz1Wwrb4b1 mA4o87HzsC0Gs3CxAXgAGmSv8ASzgPT3F8pvz0GQ+3lSqegRpXWzwSCN/Ay+nsrEOG Vk2vzTn2Z+wI6fwh/Ds3+g46FyHJlPImUj+ZAD/7gXPfZN5oSQ0wbxrqgzZ3edp1oB ucPc/xBWdsnOYCcwH0cwEIb02rutjadO2VRpIW2BdVxrPwN28zxN3Jx0+5oY8GNTqT rO4kR78EKtwAyV/3YXp4M/fDJ7L6HsVb0DsWP5iDaPLuhORb2+ryZZvrzxH96Eb4Zl FCFLLc9bGXmnA== From: Eric Biggers To: linux-kernel@vger.kernel.org Cc: linux-arch@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-crypto@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, x86@kernel.org, Zhihang Shao , Ard Biesheuvel Subject: [PATCH 05/11] arm/crc-t10dif: expose CRC-T10DIF function through lib Date: Sat, 16 Nov 2024 16:22:38 -0800 Message-ID: <20241117002244.105200-6-ebiggers@kernel.org> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20241117002244.105200-1-ebiggers@kernel.org> References: <20241117002244.105200-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-crypto@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: Eric Biggers Move the arm CRC-T10DIF assembly code into the lib directory and wire it up to the library interface. This allows it to be used without going through the crypto API. It remains usable via the crypto API too via the shash algorithms that use the library interface. Thus all the arch-specific "shash" code becomes unnecessary and is removed. Note: to see the diff from arch/arm/crypto/crct10dif-ce-glue.c to arch/arm/lib/crc-t10dif-glue.c, view this commit with 'git show -M10'. Signed-off-by: Eric Biggers --- arch/arm/Kconfig | 1 + arch/arm/crypto/Kconfig | 11 -- arch/arm/crypto/Makefile | 2 - arch/arm/crypto/crct10dif-ce-glue.c | 124 ------------------ arch/arm/lib/Makefile | 3 + .../crc-t10dif-core.S} | 0 arch/arm/lib/crc-t10dif-glue.c | 77 +++++++++++ 7 files changed, 81 insertions(+), 137 deletions(-) delete mode 100644 arch/arm/crypto/crct10dif-ce-glue.c rename arch/arm/{crypto/crct10dif-ce-core.S => lib/crc-t10dif-core.S} (100%) create mode 100644 arch/arm/lib/crc-t10dif-glue.c diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 851260303234..b9081b1a0c06 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -6,10 +6,11 @@ config ARM select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE if HAVE_KRETPROBES && FRAME_POINTER && !ARM_UNWIND select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_HAS_CPU_FINALIZE_INIT if MMU select ARCH_HAS_CRC32 if KERNEL_MODE_NEON + select ARCH_HAS_CRC_T10DIF if KERNEL_MODE_NEON select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL if MMU select ARCH_HAS_DMA_ALLOC if MMU select ARCH_HAS_DMA_OPS select ARCH_HAS_DMA_WRITE_COMBINE if !ARM_DMA_MEM_BUFFERABLE diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index ea0ebf336d0d..32650c8431d9 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -220,18 +220,7 @@ config CRYPTO_CHACHA20_NEON stream cipher algorithms Architecture: arm using: - NEON (Advanced SIMD) extensions -config CRYPTO_CRCT10DIF_ARM_CE - tristate "CRCT10DIF" - depends on KERNEL_MODE_NEON - depends on CRC_T10DIF - select CRYPTO_HASH - help - CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF) - - Architecture: arm using: - - PMULL (Polynomial Multiply Long) instructions - endmenu diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index 38ec5cc1e844..3d0e23ff9e74 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -18,11 +18,10 @@ obj-$(CONFIG_CRYPTO_CURVE25519_NEON) += curve25519-neon.o obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o obj-$(CONFIG_CRYPTO_SHA2_ARM_CE) += sha2-arm-ce.o obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o -obj-$(CONFIG_CRYPTO_CRCT10DIF_ARM_CE) += crct10dif-arm-ce.o aes-arm-y := aes-cipher-core.o aes-cipher-glue.o aes-arm-bs-y := aes-neonbs-core.o aes-neonbs-glue.o sha1-arm-y := sha1-armv4-large.o sha1_glue.o sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o @@ -34,11 +33,10 @@ libblake2s-arm-y:= blake2s-core.o blake2s-glue.o blake2b-neon-y := blake2b-neon-core.o blake2b-neon-glue.o sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o sha2-arm-ce-y := sha2-ce-core.o sha2-ce-glue.o aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o -crct10dif-arm-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o chacha-neon-y := chacha-scalar-core.o chacha-glue.o chacha-neon-$(CONFIG_KERNEL_MODE_NEON) += chacha-neon-core.o poly1305-arm-y := poly1305-core.o poly1305-glue.o nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o curve25519-neon-y := curve25519-core.o curve25519-glue.o diff --git a/arch/arm/crypto/crct10dif-ce-glue.c b/arch/arm/crypto/crct10dif-ce-glue.c deleted file mode 100644 index a8b74523729e..000000000000 --- a/arch/arm/crypto/crct10dif-ce-glue.c +++ /dev/null @@ -1,124 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions - * - * Copyright (C) 2016 Linaro Ltd - */ - -#include -#include -#include -#include -#include - -#include -#include - -#include -#include - -#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U - -asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len); -asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len, - u8 out[16]); - -static int crct10dif_init(struct shash_desc *desc) -{ - u16 *crc = shash_desc_ctx(desc); - - *crc = 0; - return 0; -} - -static int crct10dif_update_ce(struct shash_desc *desc, const u8 *data, - unsigned int length) -{ - u16 *crc = shash_desc_ctx(desc); - - if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { - kernel_neon_begin(); - *crc = crc_t10dif_pmull64(*crc, data, length); - kernel_neon_end(); - } else { - *crc = crc_t10dif_generic(*crc, data, length); - } - - return 0; -} - -static int crct10dif_update_neon(struct shash_desc *desc, const u8 *data, - unsigned int length) -{ - u16 *crcp = shash_desc_ctx(desc); - u8 buf[16] __aligned(16); - u16 crc = *crcp; - - if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { - kernel_neon_begin(); - crc_t10dif_pmull8(crc, data, length, buf); - kernel_neon_end(); - - crc = 0; - data = buf; - length = sizeof(buf); - } - - *crcp = crc_t10dif_generic(crc, data, length); - return 0; -} - -static int crct10dif_final(struct shash_desc *desc, u8 *out) -{ - u16 *crc = shash_desc_ctx(desc); - - *(u16 *)out = *crc; - return 0; -} - -static struct shash_alg algs[] = {{ - .digestsize = CRC_T10DIF_DIGEST_SIZE, - .init = crct10dif_init, - .update = crct10dif_update_neon, - .final = crct10dif_final, - .descsize = CRC_T10DIF_DIGEST_SIZE, - - .base.cra_name = "crct10dif", - .base.cra_driver_name = "crct10dif-arm-neon", - .base.cra_priority = 150, - .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -}, { - .digestsize = CRC_T10DIF_DIGEST_SIZE, - .init = crct10dif_init, - .update = crct10dif_update_ce, - .final = crct10dif_final, - .descsize = CRC_T10DIF_DIGEST_SIZE, - - .base.cra_name = "crct10dif", - .base.cra_driver_name = "crct10dif-arm-ce", - .base.cra_priority = 200, - .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -}}; - -static int __init crc_t10dif_mod_init(void) -{ - if (!(elf_hwcap & HWCAP_NEON)) - return -ENODEV; - - return crypto_register_shashes(algs, 1 + !!(elf_hwcap2 & HWCAP2_PMULL)); -} - -static void __exit crc_t10dif_mod_exit(void) -{ - crypto_unregister_shashes(algs, 1 + !!(elf_hwcap2 & HWCAP2_PMULL)); -} - -module_init(crc_t10dif_mod_init); -module_exit(crc_t10dif_mod_exit); - -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_DESCRIPTION("Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions"); -MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("crct10dif"); diff --git a/arch/arm/lib/Makefile b/arch/arm/lib/Makefile index 01cd4db2ed47..007874320937 100644 --- a/arch/arm/lib/Makefile +++ b/arch/arm/lib/Makefile @@ -46,5 +46,8 @@ endif obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o obj-$(CONFIG_CRC32_ARCH) += crc32-arm.o crc32-arm-y := crc32-glue.o crc32-core.o + +obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-arm.o +crc-t10dif-arm-y := crc-t10dif-glue.o crc-t10dif-core.o diff --git a/arch/arm/crypto/crct10dif-ce-core.S b/arch/arm/lib/crc-t10dif-core.S similarity index 100% rename from arch/arm/crypto/crct10dif-ce-core.S rename to arch/arm/lib/crc-t10dif-core.S diff --git a/arch/arm/lib/crc-t10dif-glue.c b/arch/arm/lib/crc-t10dif-glue.c new file mode 100644 index 000000000000..70d391d1b697 --- /dev/null +++ b/arch/arm/lib/crc-t10dif-glue.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions + * + * Copyright (C) 2016 Linaro Ltd + */ + +#include +#include +#include +#include +#include + +#include + +#include +#include + +static DEFINE_STATIC_KEY_FALSE(have_neon); +static DEFINE_STATIC_KEY_FALSE(have_pmull); + +#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U + +asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len); +asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len, + u8 out[16]); + +u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length) +{ + if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && + static_branch_likely(&have_pmull) && crypto_simd_usable()) { + kernel_neon_begin(); + crc = crc_t10dif_pmull64(crc, data, length); + kernel_neon_end(); + return crc; + } + if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && + static_branch_likely(&have_neon) && crypto_simd_usable()) { + u8 buf[16] __aligned(16); + + kernel_neon_begin(); + crc_t10dif_pmull8(crc, data, length, buf); + kernel_neon_end(); + + crc = 0; + data = buf; + length = sizeof(buf); + } + return crc_t10dif_generic(crc, data, length); +} +EXPORT_SYMBOL(crc_t10dif_arch); + +static int __init crc_t10dif_arm_init(void) +{ + if (elf_hwcap & HWCAP_NEON) { + static_branch_enable(&have_neon); + if (elf_hwcap2 & HWCAP2_PMULL) + static_branch_enable(&have_pmull); + } + return 0; +} +arch_initcall(crc_t10dif_arm_init); + +static void __exit crc_t10dif_arm_exit(void) +{ +} +module_exit(crc_t10dif_arm_exit); + +bool crc_t10dif_is_optimized(void) +{ + return static_key_enabled(&have_neon); +} +EXPORT_SYMBOL(crc_t10dif_is_optimized); + +MODULE_AUTHOR("Ard Biesheuvel "); +MODULE_DESCRIPTION("Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions"); +MODULE_LICENSE("GPL v2");