diff mbox series

[isar-cip-core,v6] initramfs-overlay-hook: Check file system of INITRAMFS_OVERLAY_STORAGE_DEVICE

Message ID 20240805091905.1818691-1-clara.kowalsky@siemens.com (mailing list archive)
State Accepted
Headers show
Series [isar-cip-core,v6] initramfs-overlay-hook: Check file system of INITRAMFS_OVERLAY_STORAGE_DEVICE | expand

Commit Message

Clara Kowalsky Aug. 5, 2024, 9:19 a.m. UTC 
In case of ext*, this detects and fixes file system errors in the
partition device before doing the partition mount.
If the partition turns out to be broken, the factory state is recovered.

Signed-off-by: Clara Kowalsky <clara.kowalsky@siemens.com>
---
 .../initramfs-overlay-hook/files/overlay.hook      | 11 +++++++++--
 .../files/overlay.script.tmpl                      | 11 ++++++++++-
 .../files/overlay_recovery_action.script           | 14 ++++++++++++++
 .../initramfs-overlay-hook_0.1.bb                  |  8 +++++++-
 4 files changed, 40 insertions(+), 4 deletions(-)
 create mode 100644 recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script

Comments

Jan Kiszka Aug. 5, 2024, 2:24 p.m. UTC | #1 
On 05.08.24 11:19, Clara Kowalsky wrote:
> In case of ext*, this detects and fixes file system errors in the
> partition device before doing the partition mount.
> If the partition turns out to be broken, the factory state is recovered.
> 
> Signed-off-by: Clara Kowalsky <clara.kowalsky@siemens.com>
> ---
>  .../initramfs-overlay-hook/files/overlay.hook      | 11 +++++++++--
>  .../files/overlay.script.tmpl                      | 11 ++++++++++-
>  .../files/overlay_recovery_action.script           | 14 ++++++++++++++
>  .../initramfs-overlay-hook_0.1.bb                  |  8 +++++++-
>  4 files changed, 40 insertions(+), 4 deletions(-)
>  create mode 100644 recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script
> 
> diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook b/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
> index 8b00ecf..6f634c5 100644
> --- a/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
> +++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
> @@ -22,6 +22,13 @@ esac
>  
>  . /usr/share/initramfs-tools/hook-functions
>  
> +hook_error() {
> +    echo "(ERROR): $1" >&2
> +    exit 1
> +}
> +
>  manual_add_modules overlay
> -copy_exec /usr/bin/mountpoint
> -copy_exec /usr/bin/awk
> +copy_exec /usr/bin/mountpoint || hook_error "/usr/bin/mountpoint not found"
> +copy_exec /usr/bin/awk || hook_error "/usr/bin/awk not found"
> +copy_exec /usr/sbin/e2fsck || hook_error "/usr/sbin/e2fsck not found"
> +copy_exec /usr/sbin/mke2fs || hook_error "/usr/sbin/mke2fs not found"
> diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl b/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
> index 42eb59c..c655a4f 100644
> --- a/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
> +++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
> @@ -31,12 +31,21 @@ ovl_partition_device="${INITRAMFS_OVERLAY_STORAGE_DEVICE}"
>  ovl_storage_path="${INITRAMFS_OVERLAY_STORAGE_PATH}"
>  ovl_lower_dirs="${INITRAMFS_OVERLAY_PATHS}"
>  ovl_mount_option="${INITRAMFS_OVERLAY_MOUNT_OPTION}"
> +ovl_recovery_script="/scripts/${INITRAMFS_OVERLAY_RECOVERY_SCRIPT}"
>  
>  root_mount_storage=${rootmnt}${ovl_storage_path}
>  storage_mount_point="$(echo "${ovl_storage_path}" | awk -F/ '{print FS$2}' )"
> +partition_fstype=$(get_fstype "${ovl_partition_device}")
>  
>  if ! mountpoint -q "${rootmnt}${storage_mount_point}"; then
> -	if ! mount -t $(get_fstype ${ovl_partition_device}) \
> +	case $partition_fstype in
> +	ext*)
> +		if ! e2fsck -p -f "$ovl_partition_device" && [ -x "$ovl_recovery_script" ]; then
> +			"$ovl_recovery_script" "$partition_fstype" "$ovl_partition_device"
> +		fi
> +		;;
> +	esac
> +	if ! mount -t ${partition_fstype} \
>  		 -o ${ovl_mount_option} \
>  		 ${ovl_partition_device} \
>  		 ${rootmnt}${storage_mount_point}; then
> diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script b/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script
> new file mode 100644
> index 0000000..d13111d
> --- /dev/null
> +++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script
> @@ -0,0 +1,14 @@
> +#!/bin/sh
> +#
> +# CIP Core, generic profile
> +#
> +# Copyright (c) Siemens AG, 2024
> +#
> +# Authors:
> +#  Clara Kowalsky <clara.kowalsky@siemens.com>
> +#
> +
> +partition_fstype="$1"
> +ovl_partition_device="$2"
> +
> +mke2fs -t "$partition_fstype" "$ovl_partition_device"
> diff --git a/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb b/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb
> index 955748f..7097130 100644
> --- a/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb
> +++ b/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb
> @@ -12,9 +12,12 @@
>  
>  inherit dpkg-raw
>  
> +INITRAMFS_OVERLAY_RECOVERY_SCRIPT ??= "overlay_recovery_action.script"
> +
>  SRC_URI += " \
>      file://overlay.hook \
>      file://overlay.script.tmpl \
> +    file://${INITRAMFS_OVERLAY_RECOVERY_SCRIPT} \
>      "
>  
>  # The variable INITRAMFS_OVERLAY_PATHS contains the directories which are
> @@ -35,7 +38,8 @@ TEMPLATE_FILES = "overlay.script.tmpl"
>  TEMPLATE_VARS += " INITRAMFS_OVERLAY_STORAGE_PATH \
>      INITRAMFS_OVERLAY_PATHS \
>      INITRAMFS_OVERLAY_STORAGE_DEVICE \
> -    INITRAMFS_OVERLAY_MOUNT_OPTION"
> +    INITRAMFS_OVERLAY_MOUNT_OPTION \
> +    INITRAMFS_OVERLAY_RECOVERY_SCRIPT"
>  
>  DEBIAN_DEPENDS = "initramfs-tools, awk, coreutils, util-linux"
>  
> @@ -48,4 +52,6 @@ do_install() {
>          "${D}/usr/share/initramfs-tools/hooks/overlay"
>      install -m 0755 "${WORKDIR}/overlay.script" \
>          "${D}/usr/share/initramfs-tools/scripts/local-bottom/overlay"
> +    install -m 0755 "${WORKDIR}/${INITRAMFS_OVERLAY_RECOVERY_SCRIPT}" \
> +        "${D}/usr/share/initramfs-tools/scripts"
>  }

Thanks, applied.

Jan
Sai.Sathujoda@toshiba-tsip.com Aug. 19, 2024, 9:34 a.m. UTC | #2 
Hello all,

File system errors are detected in `/dev/disk/by-label/var` which is triggering `mke2fs` to re-create the file system during reboot.

In LAVA jobs, mke2fs is asking for proceed prompt (y/n) because of which all software update jobs get stuck leading to a timeout error. -> https://lava.ciplatform.org/scheduler/job/1180009#L1213

Currently I am not sure why file system errors are detected in `/dev/disk/by-label/var` during reboot, but this needs to be handled in software update test automation may be using mke2fs -FF -t "$partition_fstype" "$ovl_partition_device".

I request everyone to provide their opinions.

Thanks and regards,
Sai Ashrith Sathujoda (T S I P)

-----Original Message-----
From: cip-dev@lists.cip-project.org <cip-dev@lists.cip-project.org> On Behalf Of Clara Kowalsky via lists.cip-project.org
Sent: Monday, August 5, 2024 2:49 PM
To: jan.kiszka@siemens.com; cip-dev@lists.cip-project.org
Cc: quirin.gylstorff@siemens.com; Clara Kowalsky <clara.kowalsky@siemens.com>
Subject: [cip-dev] [isar-cip-core][PATCH v6] initramfs-overlay-hook: Check file system of INITRAMFS_OVERLAY_STORAGE_DEVICE

In case of ext*, this detects and fixes file system errors in the partition device before doing the partition mount.
If the partition turns out to be broken, the factory state is recovered.

Signed-off-by: Clara Kowalsky <clara.kowalsky@siemens.com>
---
 .../initramfs-overlay-hook/files/overlay.hook      | 11 +++++++++--
 .../files/overlay.script.tmpl                      | 11 ++++++++++-
 .../files/overlay_recovery_action.script           | 14 ++++++++++++++
 .../initramfs-overlay-hook_0.1.bb                  |  8 +++++++-
 4 files changed, 40 insertions(+), 4 deletions(-)  create mode 100644 recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script

diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook b/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
index 8b00ecf..6f634c5 100644
--- a/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
+++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
@@ -22,6 +22,13 @@ esac
 
 . /usr/share/initramfs-tools/hook-functions
 
+hook_error() {
+    echo "(ERROR): $1" >&2
+    exit 1
+}
+
 manual_add_modules overlay
-copy_exec /usr/bin/mountpoint
-copy_exec /usr/bin/awk
+copy_exec /usr/bin/mountpoint || hook_error "/usr/bin/mountpoint not found"
+copy_exec /usr/bin/awk || hook_error "/usr/bin/awk not found"
+copy_exec /usr/sbin/e2fsck || hook_error "/usr/sbin/e2fsck not found"
+copy_exec /usr/sbin/mke2fs || hook_error "/usr/sbin/mke2fs not found"
diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl b/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
index 42eb59c..c655a4f 100644
--- a/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
+++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
@@ -31,12 +31,21 @@ ovl_partition_device="${INITRAMFS_OVERLAY_STORAGE_DEVICE}"
 ovl_storage_path="${INITRAMFS_OVERLAY_STORAGE_PATH}"
 ovl_lower_dirs="${INITRAMFS_OVERLAY_PATHS}"
 ovl_mount_option="${INITRAMFS_OVERLAY_MOUNT_OPTION}"
+ovl_recovery_script="/scripts/${INITRAMFS_OVERLAY_RECOVERY_SCRIPT}"
 
 root_mount_storage=${rootmnt}${ovl_storage_path}
 storage_mount_point="$(echo "${ovl_storage_path}" | awk -F/ '{print FS$2}' )"
+partition_fstype=$(get_fstype "${ovl_partition_device}")
 
 if ! mountpoint -q "${rootmnt}${storage_mount_point}"; then
-	if ! mount -t $(get_fstype ${ovl_partition_device}) \
+	case $partition_fstype in
+	ext*)
+		if ! e2fsck -p -f "$ovl_partition_device" && [ -x "$ovl_recovery_script" ]; then
+			"$ovl_recovery_script" "$partition_fstype" "$ovl_partition_device"
+		fi
+		;;
+	esac
+	if ! mount -t ${partition_fstype} \
 		 -o ${ovl_mount_option} \
 		 ${ovl_partition_device} \
 		 ${rootmnt}${storage_mount_point}; then diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script b/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script
new file mode 100644
index 0000000..d13111d
--- /dev/null
+++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_ac
+++ tion.script
@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+# CIP Core, generic profile
+#
+# Copyright (c) Siemens AG, 2024
+#
+# Authors:
+#  Clara Kowalsky <clara.kowalsky@siemens.com> #
+
+partition_fstype="$1"
+ovl_partition_device="$2"
+
+mke2fs -t "$partition_fstype" "$ovl_partition_device"
diff --git a/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb b/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb
index 955748f..7097130 100644
--- a/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb
+++ b/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.
+++ 1.bb
@@ -12,9 +12,12 @@
 
 inherit dpkg-raw
 
+INITRAMFS_OVERLAY_RECOVERY_SCRIPT ??= "overlay_recovery_action.script"
+
 SRC_URI += " \
     file://overlay.hook \
     file://overlay.script.tmpl \
+    file://${INITRAMFS_OVERLAY_RECOVERY_SCRIPT} \
     "
 
 # The variable INITRAMFS_OVERLAY_PATHS contains the directories which are @@ -35,7 +38,8 @@ TEMPLATE_FILES = "overlay.script.tmpl"
 TEMPLATE_VARS += " INITRAMFS_OVERLAY_STORAGE_PATH \
     INITRAMFS_OVERLAY_PATHS \
     INITRAMFS_OVERLAY_STORAGE_DEVICE \
-    INITRAMFS_OVERLAY_MOUNT_OPTION"
+    INITRAMFS_OVERLAY_MOUNT_OPTION \
+    INITRAMFS_OVERLAY_RECOVERY_SCRIPT"
 
 DEBIAN_DEPENDS = "initramfs-tools, awk, coreutils, util-linux"
 
@@ -48,4 +52,6 @@ do_install() {
         "${D}/usr/share/initramfs-tools/hooks/overlay"
     install -m 0755 "${WORKDIR}/overlay.script" \
         "${D}/usr/share/initramfs-tools/scripts/local-bottom/overlay"
+    install -m 0755 "${WORKDIR}/${INITRAMFS_OVERLAY_RECOVERY_SCRIPT}" \
+        "${D}/usr/share/initramfs-tools/scripts"
 }
--
2.46.0
Jan Kiszka Aug. 19, 2024, 9:38 a.m. UTC | #3 
On 19.08.24 11:34, Sai.Sathujoda@toshiba-tsip.com wrote:
> Hello all,
> 
> File system errors are detected in `/dev/disk/by-label/var` which is triggering `mke2fs` to re-create the file system during reboot.
> 
> In LAVA jobs, mke2fs is asking for proceed prompt (y/n) because of which all software update jobs get stuck leading to a timeout error. -> https://lava.ciplatform.org/scheduler/job/1180009#L1213
> 
> Currently I am not sure why file system errors are detected in `/dev/disk/by-label/var` during reboot, but this needs to be handled in software update test automation may be using mke2fs -FF -t "$partition_fstype" "$ovl_partition_device".
> 
> I request everyone to provide their opinions.

So, we have two topics:

1. How to prevent getting stuck if /var is corrupt in way that it
   falls back to interactive mode (-FF?)?
2. What is causing this issue in our test case?

Before fixing 1. we should understand 2., I would say.

Jan

> 
> Thanks and regards,
> Sai Ashrith Sathujoda (T S I P)
> 
> -----Original Message-----
> From: cip-dev@lists.cip-project.org <cip-dev@lists.cip-project.org> On Behalf Of Clara Kowalsky via lists.cip-project.org
> Sent: Monday, August 5, 2024 2:49 PM
> To: jan.kiszka@siemens.com; cip-dev@lists.cip-project.org
> Cc: quirin.gylstorff@siemens.com; Clara Kowalsky <clara.kowalsky@siemens.com>
> Subject: [cip-dev] [isar-cip-core][PATCH v6] initramfs-overlay-hook: Check file system of INITRAMFS_OVERLAY_STORAGE_DEVICE
> 
> In case of ext*, this detects and fixes file system errors in the partition device before doing the partition mount.
> If the partition turns out to be broken, the factory state is recovered.
> 
> Signed-off-by: Clara Kowalsky <clara.kowalsky@siemens.com>
> ---
>  .../initramfs-overlay-hook/files/overlay.hook      | 11 +++++++++--
>  .../files/overlay.script.tmpl                      | 11 ++++++++++-
>  .../files/overlay_recovery_action.script           | 14 ++++++++++++++
>  .../initramfs-overlay-hook_0.1.bb                  |  8 +++++++-
>  4 files changed, 40 insertions(+), 4 deletions(-)  create mode 100644 recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script
> 
> diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook b/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
> index 8b00ecf..6f634c5 100644
> --- a/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
> +++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
> @@ -22,6 +22,13 @@ esac
>  
>  . /usr/share/initramfs-tools/hook-functions
>  
> +hook_error() {
> +    echo "(ERROR): $1" >&2
> +    exit 1
> +}
> +
>  manual_add_modules overlay
> -copy_exec /usr/bin/mountpoint
> -copy_exec /usr/bin/awk
> +copy_exec /usr/bin/mountpoint || hook_error "/usr/bin/mountpoint not found"
> +copy_exec /usr/bin/awk || hook_error "/usr/bin/awk not found"
> +copy_exec /usr/sbin/e2fsck || hook_error "/usr/sbin/e2fsck not found"
> +copy_exec /usr/sbin/mke2fs || hook_error "/usr/sbin/mke2fs not found"
> diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl b/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
> index 42eb59c..c655a4f 100644
> --- a/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
> +++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
> @@ -31,12 +31,21 @@ ovl_partition_device="${INITRAMFS_OVERLAY_STORAGE_DEVICE}"
>  ovl_storage_path="${INITRAMFS_OVERLAY_STORAGE_PATH}"
>  ovl_lower_dirs="${INITRAMFS_OVERLAY_PATHS}"
>  ovl_mount_option="${INITRAMFS_OVERLAY_MOUNT_OPTION}"
> +ovl_recovery_script="/scripts/${INITRAMFS_OVERLAY_RECOVERY_SCRIPT}"
>  
>  root_mount_storage=${rootmnt}${ovl_storage_path}
>  storage_mount_point="$(echo "${ovl_storage_path}" | awk -F/ '{print FS$2}' )"
> +partition_fstype=$(get_fstype "${ovl_partition_device}")
>  
>  if ! mountpoint -q "${rootmnt}${storage_mount_point}"; then
> -	if ! mount -t $(get_fstype ${ovl_partition_device}) \
> +	case $partition_fstype in
> +	ext*)
> +		if ! e2fsck -p -f "$ovl_partition_device" && [ -x "$ovl_recovery_script" ]; then
> +			"$ovl_recovery_script" "$partition_fstype" "$ovl_partition_device"
> +		fi
> +		;;
> +	esac
> +	if ! mount -t ${partition_fstype} \
>  		 -o ${ovl_mount_option} \
>  		 ${ovl_partition_device} \
>  		 ${rootmnt}${storage_mount_point}; then diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script b/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script
> new file mode 100644
> index 0000000..d13111d
> --- /dev/null
> +++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_ac
> +++ tion.script
> @@ -0,0 +1,14 @@
> +#!/bin/sh
> +#
> +# CIP Core, generic profile
> +#
> +# Copyright (c) Siemens AG, 2024
> +#
> +# Authors:
> +#  Clara Kowalsky <clara.kowalsky@siemens.com> #
> +
> +partition_fstype="$1"
> +ovl_partition_device="$2"
> +
> +mke2fs -t "$partition_fstype" "$ovl_partition_device"
> diff --git a/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb b/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb
> index 955748f..7097130 100644
> --- a/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb
> +++ b/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.
> +++ 1.bb
> @@ -12,9 +12,12 @@
>  
>  inherit dpkg-raw
>  
> +INITRAMFS_OVERLAY_RECOVERY_SCRIPT ??= "overlay_recovery_action.script"
> +
>  SRC_URI += " \
>      file://overlay.hook \
>      file://overlay.script.tmpl \
> +    file://${INITRAMFS_OVERLAY_RECOVERY_SCRIPT} \
>      "
>  
>  # The variable INITRAMFS_OVERLAY_PATHS contains the directories which are @@ -35,7 +38,8 @@ TEMPLATE_FILES = "overlay.script.tmpl"
>  TEMPLATE_VARS += " INITRAMFS_OVERLAY_STORAGE_PATH \
>      INITRAMFS_OVERLAY_PATHS \
>      INITRAMFS_OVERLAY_STORAGE_DEVICE \
> -    INITRAMFS_OVERLAY_MOUNT_OPTION"
> +    INITRAMFS_OVERLAY_MOUNT_OPTION \
> +    INITRAMFS_OVERLAY_RECOVERY_SCRIPT"
>  
>  DEBIAN_DEPENDS = "initramfs-tools, awk, coreutils, util-linux"
>  
> @@ -48,4 +52,6 @@ do_install() {
>          "${D}/usr/share/initramfs-tools/hooks/overlay"
>      install -m 0755 "${WORKDIR}/overlay.script" \
>          "${D}/usr/share/initramfs-tools/scripts/local-bottom/overlay"
> +    install -m 0755 "${WORKDIR}/${INITRAMFS_OVERLAY_RECOVERY_SCRIPT}" \
> +        "${D}/usr/share/initramfs-tools/scripts"
>  }
> --
> 2.46.0
> 
>
Florian Bezdeka Aug. 19, 2024, 10:04 a.m. UTC | #4 
On Mon, 2024-08-19 at 11:38 +0200, Jan Kiszka via lists.cip-project.org
wrote:
> On 19.08.24 11:34, Sai.Sathujoda@toshiba-tsip.com wrote:
> > Hello all,
> > 
> > File system errors are detected in `/dev/disk/by-label/var` which is triggering `mke2fs` to re-create the file system during reboot.
> > 
> > In LAVA jobs, mke2fs is asking for proceed prompt (y/n) because of which all software update jobs get stuck leading to a timeout error. -> https://lava.ciplatform.org/scheduler/job/1180009#L1213
> > 
> > Currently I am not sure why file system errors are detected in `/dev/disk/by-label/var` during reboot, but this needs to be handled in software update test automation may be using mke2fs -FF -t "$partition_fstype" "$ovl_partition_device".
> > 
> > I request everyone to provide their opinions.
> 
> So, we have two topics:
> 
> 1. How to prevent getting stuck if /var is corrupt in way that it
>    falls back to interactive mode (-FF?)?
> 2. What is causing this issue in our test case?
> 
> Before fixing 1. we should understand 2., I would say.

Is there a device reboot involved? If yes, make sure that you "sync"
your file system to disk before returning from the LAVA test suite.

It might happen that LAVA powers the device off too early (due to
incorrect timeout settings, ...) before all changes were written.

HTH,
Florian

> 
> Jan
> 
> > 
> > Thanks and regards,
> > Sai Ashrith Sathujoda (T S I P)
> > 
> > -----Original Message-----
> > From: cip-dev@lists.cip-project.org <cip-dev@lists.cip-project.org> On Behalf Of Clara Kowalsky via lists.cip-project.org
> > Sent: Monday, August 5, 2024 2:49 PM
> > To: jan.kiszka@siemens.com; cip-dev@lists.cip-project.org
> > Cc: quirin.gylstorff@siemens.com; Clara Kowalsky <clara.kowalsky@siemens.com>
> > Subject: [cip-dev] [isar-cip-core][PATCH v6] initramfs-overlay-hook: Check file system of INITRAMFS_OVERLAY_STORAGE_DEVICE
> > 
> > In case of ext*, this detects and fixes file system errors in the partition device before doing the partition mount.
> > If the partition turns out to be broken, the factory state is recovered.
> > 
> > Signed-off-by: Clara Kowalsky <clara.kowalsky@siemens.com>
> > ---
> >  .../initramfs-overlay-hook/files/overlay.hook      | 11 +++++++++--
> >  .../files/overlay.script.tmpl                      | 11 ++++++++++-
> >  .../files/overlay_recovery_action.script           | 14 ++++++++++++++
> >  .../initramfs-overlay-hook_0.1.bb                  |  8 +++++++-
> >  4 files changed, 40 insertions(+), 4 deletions(-)  create mode 100644 recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script
> > 
> > diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook b/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
> > index 8b00ecf..6f634c5 100644
> > --- a/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
> > +++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
> > @@ -22,6 +22,13 @@ esac
> >  
> >  . /usr/share/initramfs-tools/hook-functions
> >  
> > +hook_error() {
> > +    echo "(ERROR): $1" >&2
> > +    exit 1
> > +}
> > +
> >  manual_add_modules overlay
> > -copy_exec /usr/bin/mountpoint
> > -copy_exec /usr/bin/awk
> > +copy_exec /usr/bin/mountpoint || hook_error "/usr/bin/mountpoint not found"
> > +copy_exec /usr/bin/awk || hook_error "/usr/bin/awk not found"
> > +copy_exec /usr/sbin/e2fsck || hook_error "/usr/sbin/e2fsck not found"
> > +copy_exec /usr/sbin/mke2fs || hook_error "/usr/sbin/mke2fs not found"
> > diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl b/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
> > index 42eb59c..c655a4f 100644
> > --- a/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
> > +++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
> > @@ -31,12 +31,21 @@ ovl_partition_device="${INITRAMFS_OVERLAY_STORAGE_DEVICE}"
> >  ovl_storage_path="${INITRAMFS_OVERLAY_STORAGE_PATH}"
> >  ovl_lower_dirs="${INITRAMFS_OVERLAY_PATHS}"
> >  ovl_mount_option="${INITRAMFS_OVERLAY_MOUNT_OPTION}"
> > +ovl_recovery_script="/scripts/${INITRAMFS_OVERLAY_RECOVERY_SCRIPT}"
> >  
> >  root_mount_storage=${rootmnt}${ovl_storage_path}
> >  storage_mount_point="$(echo "${ovl_storage_path}" | awk -F/ '{print FS$2}' )"
> > +partition_fstype=$(get_fstype "${ovl_partition_device}")
> >  
> >  if ! mountpoint -q "${rootmnt}${storage_mount_point}"; then
> > -	if ! mount -t $(get_fstype ${ovl_partition_device}) \
> > +	case $partition_fstype in
> > +	ext*)
> > +		if ! e2fsck -p -f "$ovl_partition_device" && [ -x "$ovl_recovery_script" ]; then
> > +			"$ovl_recovery_script" "$partition_fstype" "$ovl_partition_device"
> > +		fi
> > +		;;
> > +	esac
> > +	if ! mount -t ${partition_fstype} \
> >  		 -o ${ovl_mount_option} \
> >  		 ${ovl_partition_device} \
> >  		 ${rootmnt}${storage_mount_point}; then diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script b/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script
> > new file mode 100644
> > index 0000000..d13111d
> > --- /dev/null
> > +++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_ac
> > +++ tion.script
> > @@ -0,0 +1,14 @@
> > +#!/bin/sh
> > +#
> > +# CIP Core, generic profile
> > +#
> > +# Copyright (c) Siemens AG, 2024
> > +#
> > +# Authors:
> > +#  Clara Kowalsky <clara.kowalsky@siemens.com> #
> > +
> > +partition_fstype="$1"
> > +ovl_partition_device="$2"
> > +
> > +mke2fs -t "$partition_fstype" "$ovl_partition_device"
> > diff --git a/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb b/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb
> > index 955748f..7097130 100644
> > --- a/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb
> > +++ b/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.
> > +++ 1.bb
> > @@ -12,9 +12,12 @@
> >  
> >  inherit dpkg-raw
> >  
> > +INITRAMFS_OVERLAY_RECOVERY_SCRIPT ??= "overlay_recovery_action.script"
> > +
> >  SRC_URI += " \
> >      file://overlay.hook \
> >      file://overlay.script.tmpl \
> > +    file://${INITRAMFS_OVERLAY_RECOVERY_SCRIPT} \
> >      "
> >  
> >  # The variable INITRAMFS_OVERLAY_PATHS contains the directories which are @@ -35,7 +38,8 @@ TEMPLATE_FILES = "overlay.script.tmpl"
> >  TEMPLATE_VARS += " INITRAMFS_OVERLAY_STORAGE_PATH \
> >      INITRAMFS_OVERLAY_PATHS \
> >      INITRAMFS_OVERLAY_STORAGE_DEVICE \
> > -    INITRAMFS_OVERLAY_MOUNT_OPTION"
> > +    INITRAMFS_OVERLAY_MOUNT_OPTION \
> > +    INITRAMFS_OVERLAY_RECOVERY_SCRIPT"
> >  
> >  DEBIAN_DEPENDS = "initramfs-tools, awk, coreutils, util-linux"
> >  
> > @@ -48,4 +52,6 @@ do_install() {
> >          "${D}/usr/share/initramfs-tools/hooks/overlay"
> >      install -m 0755 "${WORKDIR}/overlay.script" \
> >          "${D}/usr/share/initramfs-tools/scripts/local-bottom/overlay"
> > +    install -m 0755 "${WORKDIR}/${INITRAMFS_OVERLAY_RECOVERY_SCRIPT}" \
> > +        "${D}/usr/share/initramfs-tools/scripts"
> >  }
> > --
> > 2.46.0
> > 
> > 
> 
> -- 
> Siemens AG, Technology
> Linux Expert Center
Gylstorff Quirin Aug. 19, 2024, 11:43 a.m. UTC | #5 
On 8/19/24 12:04 PM, Florian Bezdeka wrote:
> On Mon, 2024-08-19 at 11:38 +0200, Jan Kiszka via lists.cip-project.org
> wrote:
>> On 19.08.24 11:34, Sai.Sathujoda@toshiba-tsip.com wrote:
>>> Hello all,
>>>
>>> File system errors are detected in `/dev/disk/by-label/var` which is triggering `mke2fs` to re-create the file system during reboot.
>>>
>>> In LAVA jobs, mke2fs is asking for proceed prompt (y/n) because of which all software update jobs get stuck leading to a timeout error. -> https://lava.ciplatform.org/scheduler/job/1180009#L1213
>>>
>>> Currently I am not sure why file system errors are detected in `/dev/disk/by-label/var` during reboot, but this needs to be handled in software update test automation may be using mke2fs -FF -t "$partition_fstype" "$ovl_partition_device".
>>>
>>> I request everyone to provide their opinions.
>>
>> So, we have two topics:
>>
>> 1. How to prevent getting stuck if /var is corrupt in way that it
>>     falls back to interactive mode (-FF?)?

This  problem occurs as e2fsck returns  a 1 in case it need to repair 
the filesystem[1].

I will post a fix later.

In case of a forced power cycle a minimal error can happen.

[1] https://man7.org/linux/man-pages/man8/e2fsck.8.html#EXIT_CODE

Quirin
>> 2. What is causing this issue in our test case?
>>
>> Before fixing 1. we should understand 2., I would say.
> 
> Is there a device reboot involved? If yes, make sure that you "sync"
> your file system to disk before returning from the LAVA test suite.
> 
> It might happen that LAVA powers the device off too early (due to
> incorrect timeout settings, ...) before all changes were written.
> 
> HTH,
> Florian
> 
>>
>> Jan
>>
>>>
>>> Thanks and regards,
>>> Sai Ashrith Sathujoda (T S I P)
>>>
>>> -----Original Message-----
>>> From: cip-dev@lists.cip-project.org <cip-dev@lists.cip-project.org> On Behalf Of Clara Kowalsky via lists.cip-project.org
>>> Sent: Monday, August 5, 2024 2:49 PM
>>> To: jan.kiszka@siemens.com; cip-dev@lists.cip-project.org
>>> Cc: quirin.gylstorff@siemens.com; Clara Kowalsky <clara.kowalsky@siemens.com>
>>> Subject: [cip-dev] [isar-cip-core][PATCH v6] initramfs-overlay-hook: Check file system of INITRAMFS_OVERLAY_STORAGE_DEVICE
>>>
>>> In case of ext*, this detects and fixes file system errors in the partition device before doing the partition mount.
>>> If the partition turns out to be broken, the factory state is recovered.
>>>
>>> Signed-off-by: Clara Kowalsky <clara.kowalsky@siemens.com>
>>> ---
>>>   .../initramfs-overlay-hook/files/overlay.hook      | 11 +++++++++--
>>>   .../files/overlay.script.tmpl                      | 11 ++++++++++-
>>>   .../files/overlay_recovery_action.script           | 14 ++++++++++++++
>>>   .../initramfs-overlay-hook_0.1.bb                  |  8 +++++++-
>>>   4 files changed, 40 insertions(+), 4 deletions(-)  create mode 100644 recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script
>>>
>>> diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook b/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
>>> index 8b00ecf..6f634c5 100644
>>> --- a/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
>>> +++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
>>> @@ -22,6 +22,13 @@ esac
>>>   
>>>   . /usr/share/initramfs-tools/hook-functions
>>>   
>>> +hook_error() {
>>> +    echo "(ERROR): $1" >&2
>>> +    exit 1
>>> +}
>>> +
>>>   manual_add_modules overlay
>>> -copy_exec /usr/bin/mountpoint
>>> -copy_exec /usr/bin/awk
>>> +copy_exec /usr/bin/mountpoint || hook_error "/usr/bin/mountpoint not found"
>>> +copy_exec /usr/bin/awk || hook_error "/usr/bin/awk not found"
>>> +copy_exec /usr/sbin/e2fsck || hook_error "/usr/sbin/e2fsck not found"
>>> +copy_exec /usr/sbin/mke2fs || hook_error "/usr/sbin/mke2fs not found"
>>> diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl b/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
>>> index 42eb59c..c655a4f 100644
>>> --- a/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
>>> +++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
>>> @@ -31,12 +31,21 @@ ovl_partition_device="${INITRAMFS_OVERLAY_STORAGE_DEVICE}"
>>>   ovl_storage_path="${INITRAMFS_OVERLAY_STORAGE_PATH}"
>>>   ovl_lower_dirs="${INITRAMFS_OVERLAY_PATHS}"
>>>   ovl_mount_option="${INITRAMFS_OVERLAY_MOUNT_OPTION}"
>>> +ovl_recovery_script="/scripts/${INITRAMFS_OVERLAY_RECOVERY_SCRIPT}"
>>>   
>>>   root_mount_storage=${rootmnt}${ovl_storage_path}
>>>   storage_mount_point="$(echo "${ovl_storage_path}" | awk -F/ '{print FS$2}' )"
>>> +partition_fstype=$(get_fstype "${ovl_partition_device}")
>>>   
>>>   if ! mountpoint -q "${rootmnt}${storage_mount_point}"; then
>>> -	if ! mount -t $(get_fstype ${ovl_partition_device}) \
>>> +	case $partition_fstype in
>>> +	ext*)
>>> +		if ! e2fsck -p -f "$ovl_partition_device" && [ -x "$ovl_recovery_script" ]; then
>>> +			"$ovl_recovery_script" "$partition_fstype" "$ovl_partition_device"
>>> +		fi
>>> +		;;
>>> +	esac
>>> +	if ! mount -t ${partition_fstype} \
>>>   		 -o ${ovl_mount_option} \
>>>   		 ${ovl_partition_device} \
>>>   		 ${rootmnt}${storage_mount_point}; then diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script b/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script
>>> new file mode 100644
>>> index 0000000..d13111d
>>> --- /dev/null
>>> +++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_ac
>>> +++ tion.script
>>> @@ -0,0 +1,14 @@
>>> +#!/bin/sh
>>> +#
>>> +# CIP Core, generic profile
>>> +#
>>> +# Copyright (c) Siemens AG, 2024
>>> +#
>>> +# Authors:
>>> +#  Clara Kowalsky <clara.kowalsky@siemens.com> #
>>> +
>>> +partition_fstype="$1"
>>> +ovl_partition_device="$2"
>>> +
>>> +mke2fs -t "$partition_fstype" "$ovl_partition_device"
>>> diff --git a/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb b/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb
>>> index 955748f..7097130 100644
>>> --- a/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb
>>> +++ b/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.
>>> +++ 1.bb
>>> @@ -12,9 +12,12 @@
>>>   
>>>   inherit dpkg-raw
>>>   
>>> +INITRAMFS_OVERLAY_RECOVERY_SCRIPT ??= "overlay_recovery_action.script"
>>> +
>>>   SRC_URI += " \
>>>       file://overlay.hook \
>>>       file://overlay.script.tmpl \
>>> +    file://${INITRAMFS_OVERLAY_RECOVERY_SCRIPT} \
>>>       "
>>>   
>>>   # The variable INITRAMFS_OVERLAY_PATHS contains the directories which are @@ -35,7 +38,8 @@ TEMPLATE_FILES = "overlay.script.tmpl"
>>>   TEMPLATE_VARS += " INITRAMFS_OVERLAY_STORAGE_PATH \
>>>       INITRAMFS_OVERLAY_PATHS \
>>>       INITRAMFS_OVERLAY_STORAGE_DEVICE \
>>> -    INITRAMFS_OVERLAY_MOUNT_OPTION"
>>> +    INITRAMFS_OVERLAY_MOUNT_OPTION \
>>> +    INITRAMFS_OVERLAY_RECOVERY_SCRIPT"
>>>   
>>>   DEBIAN_DEPENDS = "initramfs-tools, awk, coreutils, util-linux"
>>>   
>>> @@ -48,4 +52,6 @@ do_install() {
>>>           "${D}/usr/share/initramfs-tools/hooks/overlay"
>>>       install -m 0755 "${WORKDIR}/overlay.script" \
>>>           "${D}/usr/share/initramfs-tools/scripts/local-bottom/overlay"
>>> +    install -m 0755 "${WORKDIR}/${INITRAMFS_OVERLAY_RECOVERY_SCRIPT}" \
>>> +        "${D}/usr/share/initramfs-tools/scripts"
>>>   }
>>> --
>>> 2.46.0
>>>
>>>
>>
>> -- 
>> Siemens AG, Technology
>> Linux Expert Center
>
diff mbox series

Patch

diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook b/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
index 8b00ecf..6f634c5 100644
--- a/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
+++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay.hook
@@ -22,6 +22,13 @@  esac
 
 . /usr/share/initramfs-tools/hook-functions
 
+hook_error() {
+    echo "(ERROR): $1" >&2
+    exit 1
+}
+
 manual_add_modules overlay
-copy_exec /usr/bin/mountpoint
-copy_exec /usr/bin/awk
+copy_exec /usr/bin/mountpoint || hook_error "/usr/bin/mountpoint not found"
+copy_exec /usr/bin/awk || hook_error "/usr/bin/awk not found"
+copy_exec /usr/sbin/e2fsck || hook_error "/usr/sbin/e2fsck not found"
+copy_exec /usr/sbin/mke2fs || hook_error "/usr/sbin/mke2fs not found"
diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl b/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
index 42eb59c..c655a4f 100644
--- a/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
+++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay.script.tmpl
@@ -31,12 +31,21 @@  ovl_partition_device="${INITRAMFS_OVERLAY_STORAGE_DEVICE}"
 ovl_storage_path="${INITRAMFS_OVERLAY_STORAGE_PATH}"
 ovl_lower_dirs="${INITRAMFS_OVERLAY_PATHS}"
 ovl_mount_option="${INITRAMFS_OVERLAY_MOUNT_OPTION}"
+ovl_recovery_script="/scripts/${INITRAMFS_OVERLAY_RECOVERY_SCRIPT}"
 
 root_mount_storage=${rootmnt}${ovl_storage_path}
 storage_mount_point="$(echo "${ovl_storage_path}" | awk -F/ '{print FS$2}' )"
+partition_fstype=$(get_fstype "${ovl_partition_device}")
 
 if ! mountpoint -q "${rootmnt}${storage_mount_point}"; then
-	if ! mount -t $(get_fstype ${ovl_partition_device}) \
+	case $partition_fstype in
+	ext*)
+		if ! e2fsck -p -f "$ovl_partition_device" && [ -x "$ovl_recovery_script" ]; then
+			"$ovl_recovery_script" "$partition_fstype" "$ovl_partition_device"
+		fi
+		;;
+	esac
+	if ! mount -t ${partition_fstype} \
 		 -o ${ovl_mount_option} \
 		 ${ovl_partition_device} \
 		 ${rootmnt}${storage_mount_point}; then
diff --git a/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script b/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script
new file mode 100644
index 0000000..d13111d
--- /dev/null
+++ b/recipes-initramfs/initramfs-overlay-hook/files/overlay_recovery_action.script
@@ -0,0 +1,14 @@ 
+#!/bin/sh
+#
+# CIP Core, generic profile
+#
+# Copyright (c) Siemens AG, 2024
+#
+# Authors:
+#  Clara Kowalsky <clara.kowalsky@siemens.com>
+#
+
+partition_fstype="$1"
+ovl_partition_device="$2"
+
+mke2fs -t "$partition_fstype" "$ovl_partition_device"
diff --git a/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb b/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb
index 955748f..7097130 100644
--- a/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb
+++ b/recipes-initramfs/initramfs-overlay-hook/initramfs-overlay-hook_0.1.bb
@@ -12,9 +12,12 @@ 
 
 inherit dpkg-raw
 
+INITRAMFS_OVERLAY_RECOVERY_SCRIPT ??= "overlay_recovery_action.script"
+
 SRC_URI += " \
     file://overlay.hook \
     file://overlay.script.tmpl \
+    file://${INITRAMFS_OVERLAY_RECOVERY_SCRIPT} \
     "
 
 # The variable INITRAMFS_OVERLAY_PATHS contains the directories which are
@@ -35,7 +38,8 @@  TEMPLATE_FILES = "overlay.script.tmpl"
 TEMPLATE_VARS += " INITRAMFS_OVERLAY_STORAGE_PATH \
     INITRAMFS_OVERLAY_PATHS \
     INITRAMFS_OVERLAY_STORAGE_DEVICE \
-    INITRAMFS_OVERLAY_MOUNT_OPTION"
+    INITRAMFS_OVERLAY_MOUNT_OPTION \
+    INITRAMFS_OVERLAY_RECOVERY_SCRIPT"
 
 DEBIAN_DEPENDS = "initramfs-tools, awk, coreutils, util-linux"
 
@@ -48,4 +52,6 @@  do_install() {
         "${D}/usr/share/initramfs-tools/hooks/overlay"
     install -m 0755 "${WORKDIR}/overlay.script" \
         "${D}/usr/share/initramfs-tools/scripts/local-bottom/overlay"
+    install -m 0755 "${WORKDIR}/${INITRAMFS_OVERLAY_RECOVERY_SCRIPT}" \
+        "${D}/usr/share/initramfs-tools/scripts"
 }