diff mbox series

watchdog/mm: Allow dumping memory info in pretimeout

Message ID 20230608-pretimeout-oom-v1-1-542cc91062d7@axis.com (mailing list archive)
State New
Headers show
Series watchdog/mm: Allow dumping memory info in pretimeout | expand

Commit Message

Vincent Whitchurch June 9, 2023, 6:45 a.m. UTC 
On my (embedded) systems, the most common cause of hitting the watchdog
(pre)timeout is due to thrashing.  Diagnosing these problems is hard
without knowing the memory state at the point of the watchdog hit.  In
order to make this information available, add a module parameter to the
watchdog pretimeout panic governor to ask it to dump memory info and the
OOM task list (using a new helper in the OOM code) before triggering the
panic.

Signed-off-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
---
 drivers/watchdog/pretimeout_panic.c | 15 +++++++++++
 include/linux/oom.h                 |  5 ++++
 include/linux/sched/task.h          |  5 ++++
 mm/oom_kill.c                       | 54 ++++++++++++++++++++++++++++++++++++-
 4 files changed, 78 insertions(+), 1 deletion(-)


---
base-commit: 9561de3a55bed6bdd44a12820ba81ec416e705a7
change-id: 20230608-pretimeout-oom-99148438a1df

Best regards,

Comments

kernel test robot June 9, 2023, 9:01 a.m. UTC | #1 
Hi Vincent,

kernel test robot noticed the following build warnings:

[auto build test WARNING on 9561de3a55bed6bdd44a12820ba81ec416e705a7]

url:    https://github.com/intel-lab-lkp/linux/commits/Vincent-Whitchurch/watchdog-mm-Allow-dumping-memory-info-in-pretimeout/20230609-144807
base:   9561de3a55bed6bdd44a12820ba81ec416e705a7
patch link:    https://lore.kernel.org/r/20230608-pretimeout-oom-v1-1-542cc91062d7%40axis.com
patch subject: [PATCH] watchdog/mm: Allow dumping memory info in pretimeout
config: parisc-buildonly-randconfig-r006-20230608 (https://download.01.org/0day-ci/archive/20230609/202306091651.GsOxG35Q-lkp@intel.com/config)
compiler: hppa-linux-gcc (GCC) 12.3.0
reproduce (this is a W=1 build):
        mkdir -p ~/bin
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        git checkout 9561de3a55bed6bdd44a12820ba81ec416e705a7
        b4 shazam https://lore.kernel.org/r/20230608-pretimeout-oom-v1-1-542cc91062d7@axis.com
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.3.0 ~/bin/make.cross W=1 O=build_dir ARCH=parisc olddefconfig
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.3.0 ~/bin/make.cross W=1 O=build_dir ARCH=parisc SHELL=/bin/bash

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202306091651.GsOxG35Q-lkp@intel.com/

All warnings (new ones prefixed by >>):

>> mm/oom_kill.c:156:21: warning: no previous prototype for 'find_trylock_task_mm' [-Wmissing-prototypes]
     156 | struct task_struct *find_trylock_task_mm(struct task_struct *p)
         |                     ^~~~~~~~~~~~~~~~~~~~


vim +/find_trylock_task_mm +156 mm/oom_kill.c

   151	
   152	/*
   153	 * Identical to the above, except that we avoid tasks which we can't lock, to
   154	 * avoid deadlocks when called from an interrupt handler.
   155	 */
 > 156	struct task_struct *find_trylock_task_mm(struct task_struct *p)
   157	{
   158		struct task_struct *t;
   159	
   160		rcu_read_lock();
   161	
   162		for_each_thread(p, t) {
   163			if (!task_trylock(t))
   164				continue;
   165			if (likely(t->mm))
   166				goto found;
   167			task_unlock(t);
   168		}
   169		t = NULL;
   170	found:
   171		rcu_read_unlock();
   172	
   173		return t;
   174	}
   175
kernel test robot June 9, 2023, 9:12 a.m. UTC | #2 
Hi Vincent,

kernel test robot noticed the following build warnings:

[auto build test WARNING on 9561de3a55bed6bdd44a12820ba81ec416e705a7]

url:    https://github.com/intel-lab-lkp/linux/commits/Vincent-Whitchurch/watchdog-mm-Allow-dumping-memory-info-in-pretimeout/20230609-144807
base:   9561de3a55bed6bdd44a12820ba81ec416e705a7
patch link:    https://lore.kernel.org/r/20230608-pretimeout-oom-v1-1-542cc91062d7%40axis.com
patch subject: [PATCH] watchdog/mm: Allow dumping memory info in pretimeout
config: hexagon-buildonly-randconfig-r001-20230608 (https://download.01.org/0day-ci/archive/20230609/202306091716.pcrXYphk-lkp@intel.com/config)
compiler: clang version 17.0.0 (https://github.com/llvm/llvm-project.git 4a5ac14ee968ff0ad5d2cc1ffa0299048db4c88a)
reproduce (this is a W=1 build):
        mkdir -p ~/bin
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        git checkout 9561de3a55bed6bdd44a12820ba81ec416e705a7
        b4 shazam https://lore.kernel.org/r/20230608-pretimeout-oom-v1-1-542cc91062d7@axis.com
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang ~/bin/make.cross W=1 O=build_dir ARCH=hexagon olddefconfig
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang ~/bin/make.cross W=1 O=build_dir ARCH=hexagon SHELL=/bin/bash

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202306091716.pcrXYphk-lkp@intel.com/

All warnings (new ones prefixed by >>):

   In file included from mm/oom_kill.c:30:
   In file included from include/linux/swap.h:9:
   In file included from include/linux/memcontrol.h:13:
   In file included from include/linux/cgroup.h:26:
   In file included from include/linux/kernel_stat.h:9:
   In file included from include/linux/interrupt.h:11:
   In file included from include/linux/hardirq.h:11:
   In file included from ./arch/hexagon/include/generated/asm/hardirq.h:1:
   In file included from include/asm-generic/hardirq.h:17:
   In file included from include/linux/irq.h:20:
   In file included from include/linux/io.h:13:
   In file included from arch/hexagon/include/asm/io.h:334:
   include/asm-generic/io.h:547:31: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
     547 |         val = __raw_readb(PCI_IOBASE + addr);
         |                           ~~~~~~~~~~ ^
   include/asm-generic/io.h:560:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
     560 |         val = __le16_to_cpu((__le16 __force)__raw_readw(PCI_IOBASE + addr));
         |                                                         ~~~~~~~~~~ ^
   include/uapi/linux/byteorder/little_endian.h:37:51: note: expanded from macro '__le16_to_cpu'
      37 | #define __le16_to_cpu(x) ((__force __u16)(__le16)(x))
         |                                                   ^
   In file included from mm/oom_kill.c:30:
   In file included from include/linux/swap.h:9:
   In file included from include/linux/memcontrol.h:13:
   In file included from include/linux/cgroup.h:26:
   In file included from include/linux/kernel_stat.h:9:
   In file included from include/linux/interrupt.h:11:
   In file included from include/linux/hardirq.h:11:
   In file included from ./arch/hexagon/include/generated/asm/hardirq.h:1:
   In file included from include/asm-generic/hardirq.h:17:
   In file included from include/linux/irq.h:20:
   In file included from include/linux/io.h:13:
   In file included from arch/hexagon/include/asm/io.h:334:
   include/asm-generic/io.h:573:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
     573 |         val = __le32_to_cpu((__le32 __force)__raw_readl(PCI_IOBASE + addr));
         |                                                         ~~~~~~~~~~ ^
   include/uapi/linux/byteorder/little_endian.h:35:51: note: expanded from macro '__le32_to_cpu'
      35 | #define __le32_to_cpu(x) ((__force __u32)(__le32)(x))
         |                                                   ^
   In file included from mm/oom_kill.c:30:
   In file included from include/linux/swap.h:9:
   In file included from include/linux/memcontrol.h:13:
   In file included from include/linux/cgroup.h:26:
   In file included from include/linux/kernel_stat.h:9:
   In file included from include/linux/interrupt.h:11:
   In file included from include/linux/hardirq.h:11:
   In file included from ./arch/hexagon/include/generated/asm/hardirq.h:1:
   In file included from include/asm-generic/hardirq.h:17:
   In file included from include/linux/irq.h:20:
   In file included from include/linux/io.h:13:
   In file included from arch/hexagon/include/asm/io.h:334:
   include/asm-generic/io.h:584:33: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
     584 |         __raw_writeb(value, PCI_IOBASE + addr);
         |                             ~~~~~~~~~~ ^
   include/asm-generic/io.h:594:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
     594 |         __raw_writew((u16 __force)cpu_to_le16(value), PCI_IOBASE + addr);
         |                                                       ~~~~~~~~~~ ^
   include/asm-generic/io.h:604:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
     604 |         __raw_writel((u32 __force)cpu_to_le32(value), PCI_IOBASE + addr);
         |                                                       ~~~~~~~~~~ ^
>> mm/oom_kill.c:156:21: warning: no previous prototype for function 'find_trylock_task_mm' [-Wmissing-prototypes]
     156 | struct task_struct *find_trylock_task_mm(struct task_struct *p)
         |                     ^
   mm/oom_kill.c:156:1: note: declare 'static' if the function is not intended to be used outside of this translation unit
     156 | struct task_struct *find_trylock_task_mm(struct task_struct *p)
         | ^
         | static 
   7 warnings generated.


vim +/find_trylock_task_mm +156 mm/oom_kill.c

   151	
   152	/*
   153	 * Identical to the above, except that we avoid tasks which we can't lock, to
   154	 * avoid deadlocks when called from an interrupt handler.
   155	 */
 > 156	struct task_struct *find_trylock_task_mm(struct task_struct *p)
   157	{
   158		struct task_struct *t;
   159	
   160		rcu_read_lock();
   161	
   162		for_each_thread(p, t) {
   163			if (!task_trylock(t))
   164				continue;
   165			if (likely(t->mm))
   166				goto found;
   167			task_unlock(t);
   168		}
   169		t = NULL;
   170	found:
   171		rcu_read_unlock();
   172	
   173		return t;
   174	}
   175
kernel test robot June 9, 2023, 9:24 a.m. UTC | #3 
Hi Vincent,

kernel test robot noticed the following build errors:

[auto build test ERROR on 9561de3a55bed6bdd44a12820ba81ec416e705a7]

url:    https://github.com/intel-lab-lkp/linux/commits/Vincent-Whitchurch/watchdog-mm-Allow-dumping-memory-info-in-pretimeout/20230609-144807
base:   9561de3a55bed6bdd44a12820ba81ec416e705a7
patch link:    https://lore.kernel.org/r/20230608-pretimeout-oom-v1-1-542cc91062d7%40axis.com
patch subject: [PATCH] watchdog/mm: Allow dumping memory info in pretimeout
config: i386-randconfig-i051-20230608 (https://download.01.org/0day-ci/archive/20230609/202306091730.papweTh9-lkp@intel.com/config)
compiler: gcc-12 (Debian 12.2.0-14) 12.2.0
reproduce (this is a W=1 build):
        git checkout 9561de3a55bed6bdd44a12820ba81ec416e705a7
        b4 shazam https://lore.kernel.org/r/20230608-pretimeout-oom-v1-1-542cc91062d7@axis.com
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        make W=1 O=build_dir ARCH=i386 olddefconfig
        make W=1 O=build_dir ARCH=i386 SHELL=/bin/bash

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202306091730.papweTh9-lkp@intel.com/

All errors (new ones prefixed by >>, old ones prefixed by <<):

>> ERROR: modpost: "__show_mem" [drivers/watchdog/pretimeout_panic.ko] undefined!
diff mbox series

Patch

diff --git a/drivers/watchdog/pretimeout_panic.c b/drivers/watchdog/pretimeout_panic.c
index 2cc3c41d2be5b..52d686fa541c7 100644
--- a/drivers/watchdog/pretimeout_panic.c
+++ b/drivers/watchdog/pretimeout_panic.c
@@ -5,10 +5,15 @@ 
 
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/mm.h>
+#include <linux/oom.h>
 #include <linux/watchdog.h>
 
 #include "watchdog_pretimeout.h"
 
+static unsigned long dump_min_rss_bytes;
+module_param(dump_min_rss_bytes, ulong, 0644);
+
 /**
  * pretimeout_panic - Panic on watchdog pretimeout event
  * @wdd - watchdog_device
@@ -17,6 +22,16 @@ 
  */
 static void pretimeout_panic(struct watchdog_device *wdd)
 {
+	/*
+	 * Since the root cause is not certain to be low memory, only print
+	 * tasks with RSS above a configurable limit, to avoid losing
+	 * potentially more important messages from the log.
+	 */
+	if (dump_min_rss_bytes) {
+		show_mem(SHOW_MEM_FILTER_NODES, NULL);
+		oom_dump_tasks(DIV_ROUND_UP(dump_min_rss_bytes, PAGE_SIZE));
+	}
+
 	panic("watchdog pretimeout event\n");
 }
 
diff --git a/include/linux/oom.h b/include/linux/oom.h
index 7d0c9c48a0c54..1451fe2c38d78 100644
--- a/include/linux/oom.h
+++ b/include/linux/oom.h
@@ -52,6 +52,9 @@  struct oom_control {
 
 	/* Used to print the constraint info. */
 	enum oom_constraint constraint;
+
+	bool dump_trylock;
+	unsigned long dump_min_rss_pages;
 };
 
 extern struct mutex oom_lock;
@@ -102,6 +105,8 @@  long oom_badness(struct task_struct *p,
 
 extern bool out_of_memory(struct oom_control *oc);
 
+extern void oom_dump_tasks(unsigned long min_rss_pages);
+
 extern void exit_oom_victim(void);
 
 extern int register_oom_notifier(struct notifier_block *nb);
diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h
index e0f5ac90a228b..e8a68b2a3e829 100644
--- a/include/linux/sched/task.h
+++ b/include/linux/sched/task.h
@@ -183,6 +183,11 @@  static inline void task_lock(struct task_struct *p)
 	spin_lock(&p->alloc_lock);
 }
 
+static inline int task_trylock(struct task_struct *p)
+{
+	return spin_trylock(&p->alloc_lock);
+}
+
 static inline void task_unlock(struct task_struct *p)
 {
 	spin_unlock(&p->alloc_lock);
diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index 044e1eed720ee..87e554a1c152f 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -149,6 +149,30 @@  struct task_struct *find_lock_task_mm(struct task_struct *p)
 	return t;
 }
 
+/*
+ * Identical to the above, except that we avoid tasks which we can't lock, to
+ * avoid deadlocks when called from an interrupt handler.
+ */
+struct task_struct *find_trylock_task_mm(struct task_struct *p)
+{
+	struct task_struct *t;
+
+	rcu_read_lock();
+
+	for_each_thread(p, t) {
+		if (!task_trylock(t))
+			continue;
+		if (likely(t->mm))
+			goto found;
+		task_unlock(t);
+	}
+	t = NULL;
+found:
+	rcu_read_unlock();
+
+	return t;
+}
+
 /*
  * order == -1 means the oom kill is required by sysrq, otherwise only
  * for display purposes.
@@ -390,15 +414,26 @@  static int dump_task(struct task_struct *p, void *arg)
 	if (!is_memcg_oom(oc) && !oom_cpuset_eligible(p, oc))
 		return 0;
 
-	task = find_lock_task_mm(p);
+	task = oc->dump_trylock ? find_trylock_task_mm(p) :
+				  find_lock_task_mm(p);
 	if (!task) {
 		/*
 		 * All of p's threads have already detached their mm's. There's
 		 * no need to report them; they can't be oom killed anyway.
+		 *
+		 * Or we got here from an interrupt and the task lock is
+		 * locked, in which case we're forced to ignore this task to
+		 * avoid deadlocks.
 		 */
 		return 0;
 	}
 
+	if (oc->dump_min_rss_pages &&
+	    get_mm_rss(task->mm) < oc->dump_min_rss_pages) {
+		task_unlock(task);
+		return 0;
+	}
+
 	pr_info("[%7d] %5d %5d %8lu %8lu %8ld %8lu         %5hd %s\n",
 		task->pid, from_kuid(&init_user_ns, task_uid(task)),
 		task->tgid, task->mm->total_vm, get_mm_rss(task->mm),
@@ -437,6 +472,23 @@  static void dump_tasks(struct oom_control *oc)
 	}
 }
 
+void oom_dump_tasks(unsigned long min_rss_pages)
+{
+	const gfp_t gfp_mask = GFP_KERNEL;
+	struct oom_control oc = {
+		.zonelist = node_zonelist(first_memory_node, gfp_mask),
+		.nodemask = NULL,
+		.memcg = NULL,
+		.gfp_mask = gfp_mask,
+		.order = -1,
+		.dump_min_rss_pages = min_rss_pages,
+		.dump_trylock = in_interrupt(),
+	};
+
+	dump_tasks(&oc);
+}
+EXPORT_SYMBOL_GPL(oom_dump_tasks);
+
 static void dump_oom_summary(struct oom_control *oc, struct task_struct *victim)
 {
 	/* one line summary of the oom killer context. */