@@ -93,7 +93,9 @@ static inline int stack_depot_early_init(void) { return 0; }
*/
depot_stack_handle_t __stack_depot_save(unsigned long *entries,
unsigned int nr_entries,
- gfp_t gfp_flags, bool can_alloc);
+ gfp_t gfp_flags, bool can_alloc,
+ bool counter);
+void stack_depot_dec_count(depot_stack_handle_t handle);
/**
* stack_depot_save - Save a stack trace to stack depot
@@ -109,6 +111,10 @@ depot_stack_handle_t __stack_depot_save(unsigned long *entries,
*/
depot_stack_handle_t stack_depot_save(unsigned long *entries,
unsigned int nr_entries, gfp_t gfp_flags);
+depot_stack_handle_t stack_depot_save_action(unsigned long *entries,
+ unsigned int nr_entries,
+ gfp_t gfp_flags,
+ bool counter);
/**
* stack_depot_fetch - Fetch a stack trace from stack depot
@@ -59,6 +59,7 @@ struct stack_record {
u32 hash; /* Hash in the hash table */
u32 size; /* Number of stored frames */
union handle_parts handle;
+ refcount_t count; /* Number of the same repeated stacks */
unsigned long entries[]; /* Variable-sized array of frames */
};
@@ -304,6 +305,7 @@ depot_alloc_stack(unsigned long *entries, int size, u32 hash, void **prealloc)
stack->handle.offset = pool_offset >> DEPOT_STACK_ALIGN;
stack->handle.valid = 1;
stack->handle.extra = 0;
+ refcount_set(&stack->count, 1);
memcpy(stack->entries, entries, flex_array_size(stack, entries, size));
pool_offset += required_size;
@@ -349,9 +351,15 @@ static inline struct stack_record *find_stack(struct stack_record *bucket,
return NULL;
}
+static void stack_depot_inc_count(struct stack_record *stack)
+{
+ refcount_inc(&stack->count);
+}
+
depot_stack_handle_t __stack_depot_save(unsigned long *entries,
unsigned int nr_entries,
- gfp_t alloc_flags, bool can_alloc)
+ gfp_t alloc_flags, bool can_alloc,
+ bool counter)
{
struct stack_record *found = NULL, **bucket;
union handle_parts retval = { .handle = 0 };
@@ -436,8 +444,11 @@ depot_stack_handle_t __stack_depot_save(unsigned long *entries,
/* Stack depot didn't use this memory, free it. */
free_pages((unsigned long)prealloc, DEPOT_POOL_ORDER);
}
- if (found)
+ if (found) {
retval.handle = found->handle.handle;
+ if (counter)
+ stack_depot_inc_count(found);
+ }
fast_exit:
return retval.handle;
}
@@ -447,12 +458,20 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries,
unsigned int nr_entries,
gfp_t alloc_flags)
{
- return __stack_depot_save(entries, nr_entries, alloc_flags, true);
+ return __stack_depot_save(entries, nr_entries, alloc_flags, true, false);
}
EXPORT_SYMBOL_GPL(stack_depot_save);
-unsigned int stack_depot_fetch(depot_stack_handle_t handle,
- unsigned long **entries)
+depot_stack_handle_t stack_depot_save_action(unsigned long *entries,
+ unsigned int nr_entries,
+ gfp_t alloc_flags,
+ bool counter)
+{
+ return __stack_depot_save(entries, nr_entries, alloc_flags, true, counter);
+}
+EXPORT_SYMBOL_GPL(stack_depot_save_action);
+
+static struct stack_record *stack_depot_getstack(depot_stack_handle_t handle)
{
union handle_parts parts = { .handle = handle };
/*
@@ -464,25 +483,56 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle,
size_t offset = parts.offset << DEPOT_STACK_ALIGN;
struct stack_record *stack;
- *entries = NULL;
- if (!handle)
- return 0;
+ if(!handle)
+ return NULL;
if (parts.pool_index > pool_index_cached) {
WARN(1, "pool index %d out of bounds (%d) for stack id %08x\n",
- parts.pool_index, pool_index_cached, handle);
- return 0;
+ parts.pool_index, pool_index_cached, handle);
+ return NULL;
}
pool = stack_pools[parts.pool_index];
if (!pool)
- return 0;
+ return NULL;
+
stack = pool + offset;
+ return stack;
+}
+
+unsigned int stack_depot_fetch(depot_stack_handle_t handle,
+ unsigned long **entries)
+{
+ struct stack_record *stack;
+
+ *entries = NULL;
+ if (!handle)
+ return 0;
+
+ stack = stack_depot_getstack(handle);
+ if (!stack)
+ return 0;
*entries = stack->entries;
return stack->size;
}
EXPORT_SYMBOL_GPL(stack_depot_fetch);
+void stack_depot_dec_count(depot_stack_handle_t handle)
+{
+ struct stack_record *stack = NULL;
+
+ stack = stack_depot_getstack(handle);
+ if (stack) {
+ /*
+ * page_owner creates some stacks via create_dummy_stack().
+ * We are not interested in those, so make sure we only decrement
+ * "valid" stacks.
+ */
+ if (refcount_read(&stack->count) > 1)
+ refcount_dec(&stack->count);
+ }
+}
+
void stack_depot_print(depot_stack_handle_t stack)
{
unsigned long *entries;
@@ -43,7 +43,8 @@ depot_stack_handle_t kasan_save_stack(gfp_t flags, bool can_alloc)
unsigned int nr_entries;
nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0);
- return __stack_depot_save(entries, nr_entries, flags, can_alloc);
+ return __stack_depot_save(entries, nr_entries, flags, can_alloc,
+ false);
}
void kasan_set_track(struct kasan_track *track, gfp_t flags)
@@ -107,7 +107,7 @@ static inline struct page_owner *get_page_owner(struct page_ext *page_ext)
return (void *)page_ext + page_owner_ops.offset;
}
-static noinline depot_stack_handle_t save_stack(gfp_t flags)
+static noinline depot_stack_handle_t save_stack(gfp_t flags, bool counter)
{
unsigned long entries[PAGE_OWNER_STACK_DEPTH];
depot_stack_handle_t handle;
@@ -126,7 +126,7 @@ static noinline depot_stack_handle_t save_stack(gfp_t flags)
current->in_page_owner = 1;
nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2);
- handle = stack_depot_save(entries, nr_entries, flags);
+ handle = stack_depot_save_action(entries, nr_entries, flags, counter);
if (!handle)
handle = failure_handle;
@@ -139,6 +139,7 @@ void __reset_page_owner(struct page *page, unsigned short order)
int i;
struct page_ext *page_ext;
depot_stack_handle_t handle;
+ depot_stack_handle_t alloc_handle;
struct page_owner *page_owner;
u64 free_ts_nsec = local_clock();
@@ -146,7 +147,10 @@ void __reset_page_owner(struct page *page, unsigned short order)
if (unlikely(!page_ext))
return;
- handle = save_stack(GFP_NOWAIT | __GFP_NOWARN);
+ page_owner = get_page_owner(page_ext);
+ alloc_handle = page_owner->handle;
+
+ handle = save_stack(GFP_NOWAIT | __GFP_NOWARN, false);
for (i = 0; i < (1 << order); i++) {
__clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags);
page_owner = get_page_owner(page_ext);
@@ -155,6 +159,7 @@ void __reset_page_owner(struct page *page, unsigned short order)
page_ext = page_ext_next(page_ext);
}
page_ext_put(page_ext);
+ stack_depot_dec_count(alloc_handle);
}
static inline void __set_page_owner_handle(struct page_ext *page_ext,
@@ -189,7 +194,7 @@ noinline void __set_page_owner(struct page *page, unsigned short order,
struct page_ext *page_ext;
depot_stack_handle_t handle;
- handle = save_stack(gfp_mask);
+ handle = save_stack(gfp_mask, true);
page_ext = page_ext_get(page);
if (unlikely(!page_ext))
We want to filter out page_owner output and print only those stacks that have been repeated beyond a certain threshold. This gives us the chance to get rid of a lot of noise. In order to do that, we need to keep track of how many repeated stacks (for allocation) do we have, so we add a new refcount_t field in the stack_record struct. Note that this might increase the size of the struct for some architectures. E.g: x86_64 is not affected due to alignment, but x86 32bits might. The alternative would be to have some kind of struct like this: struct track_stacks { struct stack_record *stack; struct track_stacks *next; refcount_t stack_count; But ithat would imply to perform more allocations and glue everything together, which would make the code more complex, so I think that going with a new field in the struct stack_record is good enough. Note that on __set_page_owner_handle(), page_owner->handle is set, and on __reset_page_owner(), page_owner->free_handle is set. We are interested in page_owner->handle, so when __set_page_owner() gets called, we derive the stack_record struct from page_owner->handle, and we increment its refcount_t field; and when __reset_page_owner() gets called, we derive its stack_record from page_owner->handle() and we decrement its refcount_t field. Signed-off-by: Oscar Salvador <osalvador@suse.de> --- include/linux/stackdepot.h | 8 ++++- lib/stackdepot.c | 72 ++++++++++++++++++++++++++++++++------ mm/kasan/common.c | 3 +- mm/page_owner.c | 13 ++++--- 4 files changed, 79 insertions(+), 17 deletions(-)