@@ -30,8 +30,16 @@ struct alloc_tag {
struct alloc_tag_counters __percpu *counters;
} __aligned(8);
+struct alloc_tag_kernel_section {
+ struct alloc_tag *first_tag;
+ unsigned long count;
+};
+
struct alloc_tag_module_section {
- unsigned long start_addr;
+ union {
+ unsigned long start_addr;
+ struct alloc_tag *first_tag;
+ };
unsigned long end_addr;
/* used size */
unsigned long size;
@@ -13,6 +13,9 @@ struct codetag_module;
struct seq_buf;
struct module;
+#define CODETAG_SECTION_START_PREFIX "__start_"
+#define CODETAG_SECTION_STOP_PREFIX "__stop_"
+
/*
* An instance of this structure is created in a special ELF section at every
* code location being tagged. At runtime, the special section is treated as
@@ -9,7 +9,18 @@
#ifdef CONFIG_MEM_ALLOC_PROFILING
+#if !defined(CONFIG_PGALLOC_TAG_REF_BITS) || CONFIG_PGALLOC_TAG_REF_BITS > 32
+#define PGALLOC_TAG_DIRECT_REF
typedef union codetag_ref pgalloc_tag_ref;
+#else /* !defined(CONFIG_PGALLOC_TAG_REF_BITS) || CONFIG_PGALLOC_TAG_REF_BITS > 32 */
+#if CONFIG_PGALLOC_TAG_REF_BITS > 16
+typedef u32 pgalloc_tag_ref;
+#else
+typedef u16 pgalloc_tag_ref;
+#endif
+#endif /* !defined(CONFIG_PGALLOC_TAG_REF_BITS) || CONFIG_PGALLOC_TAG_REF_BITS > 32 */
+
+#ifdef PGALLOC_TAG_DIRECT_REF
static inline void read_pgref(pgalloc_tag_ref *pgref, union codetag_ref *ref)
{
@@ -20,6 +31,63 @@ static inline void write_pgref(pgalloc_tag_ref *pgref, union codetag_ref *ref)
{
pgref->ct = ref->ct;
}
+
+static inline void alloc_tag_sec_init(void) {}
+
+#else /* PGALLOC_TAG_DIRECT_REF */
+
+extern struct alloc_tag_kernel_section kernel_tags;
+extern struct alloc_tag_module_section module_tags;
+
+#define CODETAG_ID_NULL 0
+#define CODETAG_ID_EMPTY 1
+#define CODETAG_ID_FIRST 2
+
+static inline void read_pgref(pgalloc_tag_ref *pgref, union codetag_ref *ref)
+{
+ pgalloc_tag_ref idx = *pgref;
+
+ switch (idx) {
+ case (CODETAG_ID_NULL):
+ ref->ct = NULL;
+ break;
+ case (CODETAG_ID_EMPTY):
+ set_codetag_empty(ref);
+ break;
+ default:
+ idx -= CODETAG_ID_FIRST;
+ ref->ct = idx < kernel_tags.count ?
+ &kernel_tags.first_tag[idx].ct :
+ &module_tags.first_tag[idx - kernel_tags.count].ct;
+ }
+}
+
+static inline void write_pgref(pgalloc_tag_ref *pgref, union codetag_ref *ref)
+{
+ struct alloc_tag *tag;
+
+ if (!ref->ct) {
+ *pgref = CODETAG_ID_NULL;
+ return;
+ }
+
+ if (is_codetag_empty(ref)) {
+ *pgref = CODETAG_ID_EMPTY;
+ return;
+ }
+
+ tag = ct_to_alloc_tag(ref->ct);
+ if (tag >= kernel_tags.first_tag && tag < kernel_tags.first_tag + kernel_tags.count) {
+ *pgref = CODETAG_ID_FIRST + (tag - kernel_tags.first_tag);
+ return;
+ }
+
+ *pgref = CODETAG_ID_FIRST + kernel_tags.count + (tag - module_tags.first_tag);
+}
+
+void __init alloc_tag_sec_init(void);
+
+#endif /* PGALLOC_TAG_DIRECT_REF */
#include <linux/page_ext.h>
extern struct page_ext_operations page_alloc_tagging_ops;
@@ -197,6 +265,7 @@ static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}
static inline void pgalloc_tag_split(struct page *page, unsigned int nr) {}
static inline struct alloc_tag *pgalloc_tag_get(struct page *page) { return NULL; }
static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {}
+static inline void alloc_tag_sec_init(void) {}
#endif /* CONFIG_MEM_ALLOC_PROFILING */
@@ -1000,6 +1000,17 @@ config MEM_ALLOC_PROFILING_DEBUG
Adds warnings with helpful error messages for memory allocation
profiling.
+config PGALLOC_TAG_REF_BITS
+ int "Number of bits for page allocation tag reference (10-64)"
+ range 10 64
+ default "64"
+ depends on MEM_ALLOC_PROFILING
+ help
+ Number of bits used to encode a page allocation tag reference.
+
+ Smaller number results in less memory overhead but limits the number of
+ allocations which can be tagged (including allocations from modules).
+
source "lib/Kconfig.kasan"
source "lib/Kconfig.kfence"
source "lib/Kconfig.kmsan"
@@ -3,6 +3,7 @@
#include <linux/execmem.h>
#include <linux/fs.h>
#include <linux/gfp.h>
+#include <linux/kallsyms.h>
#include <linux/module.h>
#include <linux/page_ext.h>
#include <linux/pgalloc_tag.h>
@@ -11,13 +12,14 @@
#include <linux/seq_file.h>
static struct codetag_type *alloc_tag_cttype;
-static struct alloc_tag_module_section module_tags;
static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
/* A dummy object used to indicate an unloaded module */
static struct module unloaded_mod;
/* A dummy object used to indicate a module prepended area */
static struct module prepend_mod;
+struct alloc_tag_module_section module_tags;
+
DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
EXPORT_SYMBOL(_shared_alloc_tag);
@@ -157,6 +159,33 @@ static void __init procfs_init(void)
proc_create_seq("allocinfo", 0400, NULL, &allocinfo_seq_op);
}
+#ifndef PGALLOC_TAG_DIRECT_REF
+
+#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME)
+#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME)
+
+struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };
+
+void __init alloc_tag_sec_init(void)
+{
+ struct alloc_tag *last_codetag;
+
+ kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name(
+ SECTION_START(ALLOC_TAG_SECTION_NAME));
+ last_codetag = (struct alloc_tag *)kallsyms_lookup_name(
+ SECTION_STOP(ALLOC_TAG_SECTION_NAME));
+ kernel_tags.count = last_codetag - kernel_tags.first_tag;
+}
+
+static inline unsigned long alloc_tag_align(unsigned long val)
+{
+ if (val % sizeof(struct alloc_tag) == 0)
+ return val;
+ return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag);
+}
+
+#endif /* PGALLOC_TAG_DIRECT_REF */
+
static bool needs_section_mem(struct module *mod, unsigned long size)
{
return size >= sizeof(struct alloc_tag);
@@ -214,6 +243,21 @@ static void *reserve_module_tags(struct module *mod, unsigned long size,
if (!align)
align = 1;
+#ifndef PGALLOC_TAG_DIRECT_REF
+ /*
+ * If alloc_tag size is not a multiple of required alignment tag
+ * indexing does not work.
+ */
+ if (!IS_ALIGNED(sizeof(struct alloc_tag), align)) {
+ pr_err("%s: alignment %lu is incompatible with allocation tag indexing (CONFIG_PGALLOC_TAG_REF_BITS)",
+ mod->name, align);
+ return ERR_PTR(-EINVAL);
+ }
+
+ /* Ensure prepend consumes multiple of alloc_tag-sized blocks */
+ if (prepend)
+ prepend = alloc_tag_align(prepend);
+#endif /* PGALLOC_TAG_DIRECT_REF */
rcu_read_lock();
repeat:
/* Try finding exact size and hope the start is aligned */
@@ -462,6 +506,10 @@ static int __init alloc_tag_init(void)
return -ENOMEM;
module_tags.end_addr = module_tags.start_addr + module_tags_mem_sz;
+#ifndef PGALLOC_TAG_DIRECT_REF
+ /* Ensure the base is alloc_tag aligned */
+ module_tags.start_addr = alloc_tag_align(module_tags.start_addr);
+#endif
mt_set_in_rcu(&mod_area_mt);
alloc_tag_cttype = codetag_register_type(&desc);
if (IS_ERR(alloc_tag_cttype)) {
@@ -151,8 +151,8 @@ static struct codetag_range get_section_range(struct module *mod,
const char *section)
{
return (struct codetag_range) {
- get_symbol(mod, "__start_", section),
- get_symbol(mod, "__stop_", section),
+ get_symbol(mod, CODETAG_SECTION_START_PREFIX, section),
+ get_symbol(mod, CODETAG_SECTION_STOP_PREFIX, section),
};
}
@@ -2650,6 +2650,7 @@ void __init mm_core_init(void)
report_meminit();
kmsan_init_shadow();
stack_depot_early_init();
+ alloc_tag_sec_init();
mem_init();
kmem_cache_init();
/*
Introduce CONFIG_PGALLOC_TAG_REF_BITS to control the size of the page allocation tag references. When the size is configured to be less than a direct pointer, the tags are searched using an index stored as the tag reference. Signed-off-by: Suren Baghdasaryan <surenb@google.com> --- include/linux/alloc_tag.h | 10 +++++- include/linux/codetag.h | 3 ++ include/linux/pgalloc_tag.h | 69 +++++++++++++++++++++++++++++++++++++ lib/Kconfig.debug | 11 ++++++ lib/alloc_tag.c | 50 ++++++++++++++++++++++++++- lib/codetag.c | 4 +-- mm/mm_init.c | 1 + 7 files changed, 144 insertions(+), 4 deletions(-)