| Message ID | 20240320101851.2589698-3-glider@google.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | [v2,1/3] mm: kmsan: implement kmsan_memmove() | expand |
On Wed, 20 Mar 2024 at 11:19, Alexander Potapenko <glider@google.com> wrote: > > Memory accesses in copy_mc_to_kernel() and copy_mc_to_user() are performed > by assembly routines and are invisible to KASAN, KCSAN, and KMSAN. > Add hooks from instrumentation.h to tell the tools these functions have > memcpy/copy_from_user semantics. > > The call to copy_mc_fragile() in copy_mc_fragile_handle_tail() is left > intact, because the latter is only called from the assembly implementation > of copy_mc_fragile(), so the memory accesses in it are covered by the > instrumentation in copy_mc_to_kernel() and copy_mc_to_user(). > > Link: https://lore.kernel.org/all/3b7dbd88-0861-4638-b2d2-911c97a4cadf@I-love.SAKURA.ne.jp/ > Suggested-by: Linus Torvalds <torvalds@linux-foundation.org> > Signed-off-by: Alexander Potapenko <glider@google.com> > Cc: Dmitry Vyukov <dvyukov@google.com> > Cc: Marco Elver <elver@google.com> > Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp> Reviewed-by: Marco Elver <elver@google.com> > --- > v2: > - as requested by Linus Torvalds, move the instrumentation outside the > uaccess section > --- > arch/x86/lib/copy_mc.c | 21 +++++++++++++++++---- > 1 file changed, 17 insertions(+), 4 deletions(-) > > diff --git a/arch/x86/lib/copy_mc.c b/arch/x86/lib/copy_mc.c > index 6e8b7e600def5..97e88e58567bf 100644 > --- a/arch/x86/lib/copy_mc.c > +++ b/arch/x86/lib/copy_mc.c > @@ -4,6 +4,7 @@ > #include <linux/jump_label.h> > #include <linux/uaccess.h> > #include <linux/export.h> > +#include <linux/instrumented.h> > #include <linux/string.h> > #include <linux/types.h> > > @@ -61,10 +62,20 @@ unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned > */ > unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len) > { > - if (copy_mc_fragile_enabled) > - return copy_mc_fragile(dst, src, len); > - if (static_cpu_has(X86_FEATURE_ERMS)) > - return copy_mc_enhanced_fast_string(dst, src, len); > + unsigned long ret; > + > + if (copy_mc_fragile_enabled) { > + instrument_memcpy_before(dst, src, len); > + ret = copy_mc_fragile(dst, src, len); > + instrument_memcpy_after(dst, src, len, ret); > + return ret; > + } > + if (static_cpu_has(X86_FEATURE_ERMS)) { > + instrument_memcpy_before(dst, src, len); > + ret = copy_mc_enhanced_fast_string(dst, src, len); > + instrument_memcpy_after(dst, src, len, ret); > + return ret; > + } > memcpy(dst, src, len); > return 0; > } > @@ -75,6 +86,7 @@ unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, un > unsigned long ret; > > if (copy_mc_fragile_enabled) { > + instrument_copy_to_user(dst, src, len); > __uaccess_begin(); > ret = copy_mc_fragile((__force void *)dst, src, len); > __uaccess_end(); > @@ -82,6 +94,7 @@ unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, un > } > > if (static_cpu_has(X86_FEATURE_ERMS)) { > + instrument_copy_to_user(dst, src, len); > __uaccess_begin(); > ret = copy_mc_enhanced_fast_string((__force void *)dst, src, len); > __uaccess_end(); > -- > 2.44.0.291.gc1ea87d7ee-goog >
diff --git a/arch/x86/lib/copy_mc.c b/arch/x86/lib/copy_mc.c index 6e8b7e600def5..97e88e58567bf 100644 --- a/arch/x86/lib/copy_mc.c +++ b/arch/x86/lib/copy_mc.c @@ -4,6 +4,7 @@ #include <linux/jump_label.h> #include <linux/uaccess.h> #include <linux/export.h> +#include <linux/instrumented.h> #include <linux/string.h> #include <linux/types.h> @@ -61,10 +62,20 @@ unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned */ unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len) { - if (copy_mc_fragile_enabled) - return copy_mc_fragile(dst, src, len); - if (static_cpu_has(X86_FEATURE_ERMS)) - return copy_mc_enhanced_fast_string(dst, src, len); + unsigned long ret; + + if (copy_mc_fragile_enabled) { + instrument_memcpy_before(dst, src, len); + ret = copy_mc_fragile(dst, src, len); + instrument_memcpy_after(dst, src, len, ret); + return ret; + } + if (static_cpu_has(X86_FEATURE_ERMS)) { + instrument_memcpy_before(dst, src, len); + ret = copy_mc_enhanced_fast_string(dst, src, len); + instrument_memcpy_after(dst, src, len, ret); + return ret; + } memcpy(dst, src, len); return 0; } @@ -75,6 +86,7 @@ unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, un unsigned long ret; if (copy_mc_fragile_enabled) { + instrument_copy_to_user(dst, src, len); __uaccess_begin(); ret = copy_mc_fragile((__force void *)dst, src, len); __uaccess_end(); @@ -82,6 +94,7 @@ unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, un } if (static_cpu_has(X86_FEATURE_ERMS)) { + instrument_copy_to_user(dst, src, len); __uaccess_begin(); ret = copy_mc_enhanced_fast_string((__force void *)dst, src, len); __uaccess_end();
Memory accesses in copy_mc_to_kernel() and copy_mc_to_user() are performed by assembly routines and are invisible to KASAN, KCSAN, and KMSAN. Add hooks from instrumentation.h to tell the tools these functions have memcpy/copy_from_user semantics. The call to copy_mc_fragile() in copy_mc_fragile_handle_tail() is left intact, because the latter is only called from the assembly implementation of copy_mc_fragile(), so the memory accesses in it are covered by the instrumentation in copy_mc_to_kernel() and copy_mc_to_user(). Link: https://lore.kernel.org/all/3b7dbd88-0861-4638-b2d2-911c97a4cadf@I-love.SAKURA.ne.jp/ Suggested-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Alexander Potapenko <glider@google.com> Cc: Dmitry Vyukov <dvyukov@google.com> Cc: Marco Elver <elver@google.com> Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp> --- v2: - as requested by Linus Torvalds, move the instrumentation outside the uaccess section --- arch/x86/lib/copy_mc.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-)