@@ -16,6 +16,14 @@ typedef unsigned int csum_t;
typedef unsigned long csum_t;
#endif
+/* Default version is sufficient for 32 bit */
+#ifdef CONFIG_64BIT
+#define _HAVE_ARCH_IPV6_CSUM
+__sum16 csum_ipv6_magic(const struct in6_addr *saddr,
+ const struct in6_addr *daddr,
+ __u32 len, __u8 proto, __wsum sum);
+#endif
+
/*
* Fold a partial checksum without adding pseudo headers
*/
@@ -91,6 +99,9 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
#define ip_fast_csum ip_fast_csum
+extern unsigned int do_csum(const unsigned char *buff, int len);
+#define do_csum do_csum
+
#include <asm-generic/checksum.h>
#endif // __ASM_RISCV_CHECKSUM_H
@@ -6,6 +6,7 @@ lib-y += memmove.o
lib-y += strcmp.o
lib-y += strlen.o
lib-y += strncmp.o
+lib-y += csum.o
lib-$(CONFIG_MMU) += uaccess.o
lib-$(CONFIG_64BIT) += tishift.o
lib-$(CONFIG_RISCV_ISA_ZICBOZ) += clear_page.o
new file mode 100644
@@ -0,0 +1,227 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * IP checksum library
+ *
+ * Influenced by arch/arm64/lib/csum.c
+ * Copyright (C) 2023 Rivos Inc.
+ */
+#include <linux/bitops.h>
+#include <linux/compiler.h>
+#include <linux/kasan-checks.h>
+#include <linux/kernel.h>
+
+#include <net/checksum.h>
+
+/* Default version is sufficient for 32 bit */
+#ifndef CONFIG_32BIT
+__sum16 csum_ipv6_magic(const struct in6_addr *saddr,
+ const struct in6_addr *daddr,
+ __u32 len, __u8 proto, __wsum csum)
+{
+ /*
+ * Inform the compiler/processor that the operation we are performing is
+ * "Commutative and Associative" by summing parts of the checksum in a
+ * tree-like structure (Section 2(A) of "Computing the Internet
+ * Checksum"). Furthermore, defer the overflow until the end of the
+ * computation which is shown to be valid in Section 2(C)(1) of the
+ * same handbook.
+ */
+ unsigned long sum, sum1, sum2, sum3, sum4, ulen, uproto;
+
+ uproto = htonl(proto);
+ ulen = htonl(len);
+
+ sum = saddr->s6_addr32[0];
+ sum += saddr->s6_addr32[1];
+ sum1 = saddr->s6_addr32[2];
+ sum1 += saddr->s6_addr32[3];
+
+ sum2 = daddr->s6_addr32[0];
+ sum2 += daddr->s6_addr32[1];
+ sum3 = daddr->s6_addr32[2];
+ sum3 += daddr->s6_addr32[3];
+
+ sum4 = csum;
+ sum4 += ulen;
+ sum4 += uproto;
+
+ sum += sum1;
+ sum2 += sum3;
+
+ sum += sum2;
+ sum += sum4;
+
+#ifdef CONFIG_RISCV_ISA_ZBB
+ if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {
+ csum_t fold_temp;
+
+ /*
+ * Zbb is likely available when the kernel is compiled with Zbb
+ * support, so nop when Zbb is available and jump when Zbb is
+ * not available.
+ */
+ asm_volatile_goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,
+ RISCV_ISA_EXT_ZBB, 1)
+ :
+ :
+ :
+ : no_zbb);
+ asm(".option push \n\
+ .option arch,+zbb \n\
+ rori %[fold_temp], %[sum], 32 \n\
+ add %[sum], %[fold_temp], %[sum] \n\
+ srli %[sum], %[sum], 32 \n\
+ roriw %[fold_temp], %[sum], 16 \n\
+ addw %[sum], %[fold_temp], %[sum] \n\
+ .option pop"
+ : [sum] "+r" (sum), [fold_temp] "=&r" (fold_temp));
+ return (__force __sum16)~(sum >> 16);
+ }
+no_zbb:
+#endif // !CONFIG_RISCV_ISA_ZBB
+ sum += (sum >> 32) | (sum << 32);
+ sum >>= 32;
+ return csum_fold((__force __wsum)sum);
+}
+EXPORT_SYMBOL(csum_ipv6_magic);
+#endif // !CONFIG_32BIT
+
+#ifdef CONFIG_32BIT
+#define OFFSET_MASK 3
+#elif CONFIG_64BIT
+#define OFFSET_MASK 7
+#endif
+
+/*
+ * Perform a checksum on an arbitrary memory address.
+ * Algorithm accounts for buff being misaligned.
+ * If buff is not aligned, will over-read bytes but not use the bytes that it
+ * shouldn't. The same thing will occur on the tail-end of the read.
+ */
+unsigned int __no_sanitize_address do_csum(const unsigned char *buff, int len)
+{
+ unsigned int offset, shift;
+ csum_t csum, data;
+ const csum_t *ptr;
+
+ if (unlikely(len <= 0))
+ return 0;
+ /*
+ * To align the address, grab the whole first byte in buff.
+ * Since it is inside of a same byte, it will never cross pages or cache
+ * lines.
+ * Directly call KASAN with the alignment we will be using.
+ */
+ offset = (csum_t)buff & OFFSET_MASK;
+ kasan_check_read(buff, len);
+ ptr = (const csum_t *)(buff - offset);
+ len = len + offset - sizeof(csum_t);
+
+ /*
+ * Clear the most signifant bits that were over-read if buff was not
+ * aligned.
+ */
+ shift = offset * 8;
+ data = *ptr;
+#ifdef __LITTLE_ENDIAN
+ data = (data >> shift) << shift;
+#else
+ data = (data << shift) >> shift;
+#endif
+
+ /*
+ * Do 32-bit reads on RV32 and 64-bit reads otherwise. This should be
+ * faster than doing 32-bit reads on architectures that support larger
+ * reads.
+ */
+ while (len > 0) {
+ csum += data;
+ csum += csum < data;
+ len -= sizeof(csum_t);
+ ptr += 1;
+ data = *ptr;
+ }
+
+ /*
+ * Perform alignment (and over-read) bytes on the tail if any bytes
+ * leftover.
+ */
+ shift = len * -8;
+#ifdef __LITTLE_ENDIAN
+ data = (data << shift) >> shift;
+#else
+ data = (data >> shift) << shift;
+#endif
+ csum += data;
+ csum += csum < data;
+
+#ifdef CONFIG_RISCV_ISA_ZBB
+ if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {
+ /*
+ * Zbb is likely available when the kernel is compiled with Zbb
+ * support, so nop when Zbb is available and jump when Zbb is
+ * not available.
+ */
+ asm_volatile_goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,
+ RISCV_ISA_EXT_ZBB, 1)
+ :
+ :
+ :
+ : no_zbb);
+ } else {
+ if (!__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_ZBB))
+ goto no_zbb;
+ }
+
+ unsigned int fold_temp;
+
+#ifdef CONFIG_32BIT
+ asm_volatile_goto(".option push \n\
+ .option arch,+zbb \n\
+ rori %[fold_temp], %[csum], 16 \n\
+ andi %[offset], %[offset], 1 \n\
+ add %[csum], %[fold_temp], %[csum] \n\
+ beq %[offset], zero, %l[end] \n\
+ rev8 %[csum], %[csum] \n\
+ zext.h %[csum], %[csum] \n\
+ .option pop"
+ : [csum] "+r" (csum), [fold_temp] "=&r" (fold_temp)
+ : [offset] "r" (offset)
+ :
+ : end);
+
+ return csum;
+#else // !CONFIG_32BIT
+ asm_volatile_goto(".option push \n\
+ .option arch,+zbb \n\
+ rori %[fold_temp], %[csum], 32 \n\
+ add %[csum], %[fold_temp], %[csum] \n\
+ srli %[csum], %[csum], 32 \n\
+ roriw %[fold_temp], %[csum], 16 \n\
+ addw %[csum], %[fold_temp], %[csum] \n\
+ andi %[offset], %[offset], 1 \n\
+ beq %[offset], zero, %l[end] \n\
+ rev8 %[csum], %[csum] \n\
+ srli %[csum], %[csum], 32 \n\
+ zext.h %[csum], %[csum] \n\
+ .option pop"
+ : [csum] "+r" (csum), [fold_temp] "=&r" (fold_temp)
+ : [offset] "r" (offset)
+ :
+ : end);
+ return csum;
+#endif // !CONFIG_32BIT
+end:
+ return csum >> 16;
+no_zbb:
+#endif // CONFIG_RISCV_ISA_ZBB
+#ifdef CONFIG_32BIT
+#else // !CONFIG_32BIT
+ csum += (csum >> 32) | (csum << 32);
+ csum >>= 32;
+#endif // !CONFIG_32BIT
+ csum = (unsigned int)csum + (((unsigned int)csum >> 16) | ((unsigned int)csum << 16));
+ if (offset & 1)
+ return (unsigned short)swab32(csum);
+ return csum >> 16;
+}
Provide a 32 and 64 bit version of do_csum. When compiled for 32-bit will load from the buffer in groups of 32 bits, and when compiled for 64-bit will load in groups of 64 bits. Benchmarking by proxy compiling csum_ipv6_magic (64-bit version) for an x86 chip as well as running the riscv generated code in QEMU, discovered that summing in a tree-like structure is about 4% faster than doing 64-bit reads. Signed-off-by: Charlie Jenkins <charlie@rivosinc.com> --- arch/riscv/include/asm/checksum.h | 11 ++ arch/riscv/lib/Makefile | 1 + arch/riscv/lib/csum.c | 227 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 239 insertions(+)