diff mbox series

[1/3] arm64: lib: introduce ldp2/stp2 macro

Message ID 20210323073432.3422227-2-yangyingliang@huawei.com (mailing list archive)
State New, archived
Headers show
Series arm64: lib: improve copy performance | expand

Commit Message

Yang Yingliang March 23, 2021, 7:34 a.m. UTC 
Introduce ldp2/stp2 to load/store without add src/dst.

Signed-off-by: Yang Yingliang <yangyingliang@huawei.com>
---
 arch/arm64/include/asm/asm-uaccess.h | 16 ++++++++++++++++
 arch/arm64/lib/copy_from_user.S      |  8 ++++++++
 arch/arm64/lib/copy_in_user.S        |  8 ++++++++
 arch/arm64/lib/copy_to_user.S        |  8 ++++++++
 arch/arm64/lib/memcpy.S              |  8 ++++++++
 5 files changed, 48 insertions(+)
diff mbox series

Patch

diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h
index ccedf548dac9..129c08621df1 100644
--- a/arch/arm64/include/asm/asm-uaccess.h
+++ b/arch/arm64/include/asm/asm-uaccess.h
@@ -72,6 +72,14 @@  alternative_else_nop_endif
 		_asm_extable	8889b,\l;
 	.endm
 
+	.macro user_ldp2 l, reg1, reg2, addr, post_inc1, post_inc2
+8888:		ldtr	\reg1, [\addr, \post_inc1];
+8889:		ldtr	\reg2, [\addr, \post_inc2];
+
+		_asm_extable	8888b,\l;
+		_asm_extable	8889b,\l;
+	.endm
+
 	.macro user_stp l, reg1, reg2, addr, post_inc
 8888:		sttr	\reg1, [\addr];
 8889:		sttr	\reg2, [\addr, #8];
@@ -81,6 +89,14 @@  alternative_else_nop_endif
 		_asm_extable	8889b,\l;
 	.endm
 
+	.macro user_stp2 l, reg1, reg2, addr, post_inc1, post_inc2
+8888:		sttr	\reg1, [\addr, \post_inc1];
+8889:		sttr	\reg2, [\addr, \post_inc2];
+
+		_asm_extable	8888b,\l;
+		_asm_extable	8889b,\l;
+	.endm
+
 	.macro user_ldst l, inst, reg, addr, post_inc
 8888:		\inst		\reg, [\addr];
 		add		\addr, \addr, \post_inc;
diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
index 95cd62d67371..37308bcb338e 100644
--- a/arch/arm64/lib/copy_from_user.S
+++ b/arch/arm64/lib/copy_from_user.S
@@ -48,10 +48,18 @@ 
 	user_ldp 9998f, \reg1, \reg2, \ptr, \val
 	.endm
 
+	.macro ldp2 reg1, reg2, ptr, val1, val2
+	user_ldp2 9998f, \reg1, \reg2, \ptr, \val1, \val2
+	.endm
+
 	.macro stp1 reg1, reg2, ptr, val
 	stp \reg1, \reg2, [\ptr], \val
 	.endm
 
+	.macro stp2 reg1, reg2, ptr, val1, val2
+	stp \reg1, \reg2, [\ptr, \val1]
+	.endm
+
 end	.req	x5
 SYM_FUNC_START(__arch_copy_from_user)
 	add	end, x0, x2
diff --git a/arch/arm64/lib/copy_in_user.S b/arch/arm64/lib/copy_in_user.S
index 1f61cd0df062..5654f7098102 100644
--- a/arch/arm64/lib/copy_in_user.S
+++ b/arch/arm64/lib/copy_in_user.S
@@ -49,10 +49,18 @@ 
 	user_ldp 9998f, \reg1, \reg2, \ptr, \val
 	.endm
 
+	.macro ldp2 reg1, reg2, ptr, val1, val2
+	user_ldp2 9998f, \reg1, \reg2, \ptr, \val1, \val2
+	.endm
+
 	.macro stp1 reg1, reg2, ptr, val
 	user_stp 9998f, \reg1, \reg2, \ptr, \val
 	.endm
 
+	.macro stp2 reg1, reg2, ptr, val1, val2
+	user_stp2 9998f, \reg1, \reg2, \ptr, \val1, \val2
+	.endm
+
 end	.req	x5
 
 SYM_FUNC_START(__arch_copy_in_user)
diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S
index 043da90f5dd7..a1f95169ce04 100644
--- a/arch/arm64/lib/copy_to_user.S
+++ b/arch/arm64/lib/copy_to_user.S
@@ -47,10 +47,18 @@ 
 	ldp \reg1, \reg2, [\ptr], \val
 	.endm
 
+	.macro ldp2 reg1, reg2, ptr, val1, val2
+	ldp \reg1, \reg2, [\ptr, \val1]
+	.endm
+
 	.macro stp1 reg1, reg2, ptr, val
 	user_stp 9998f, \reg1, \reg2, \ptr, \val
 	.endm
 
+	.macro stp2 reg1, reg2, ptr, val1, val2
+	user_stp2 9998f, \reg1, \reg2, \ptr, \val1, \val2
+	.endm
+
 end	.req	x5
 SYM_FUNC_START(__arch_copy_to_user)
 	add	end, x0, x2
diff --git a/arch/arm64/lib/memcpy.S b/arch/arm64/lib/memcpy.S
index dc8d2a216a6e..9e0bfefd2673 100644
--- a/arch/arm64/lib/memcpy.S
+++ b/arch/arm64/lib/memcpy.S
@@ -52,10 +52,18 @@ 
 	ldp \reg1, \reg2, [\ptr], \val
 	.endm
 
+	.macro ldp2 reg1, reg2, ptr, val1, val2
+	ldp \reg1, \reg2, [\ptr, \val1]
+	.endm
+
 	.macro stp1 reg1, reg2, ptr, val
 	stp \reg1, \reg2, [\ptr], \val
 	.endm
 
+	.macro stp2 reg1, reg2, ptr, val1, val2
+	stp \reg1, \reg2, [\ptr, \val1]
+	.endm
+
 SYM_FUNC_START_ALIAS(__memcpy)
 SYM_FUNC_START_WEAK_PI(memcpy)
 #include "copy_template.S"