diff mbox series

[v2,1/2] automation: fix false success in qemu tests

Message ID 9b60c6473aa084e4e47c64aa2347db6e3ca9ac91.1724967614.git.victorm.lira@amd.com (mailing list archive)
State New
Headers show
Series automation: test script fixes (pipefail) | expand

Commit Message

Lira, Victor M Aug. 29, 2024, 10:34 p.m. UTC 
From: Victor Lira <victorm.lira@amd.com>

Fix flaw in qemu-*.sh tests that producess a false success. The following
lines produces success despite the "expect" script producing nonzero exit
status:

    set +e
...
    ./automation/scripts/qemu-key.exp | sed 's/\r\+$//'
    (end of file)

The default exit status for a pipeline using "|" operator is that of the
rightmost command. Fix this by setting the "pipefail" option in the shell,
and removing "set +e" allowing the expect script to determine the result.

Signed-off-by: Victor Lira <victorm.lira@amd.com>
---
Cc: Andrew Cooper <andrew.cooper3@citrix.com>
Cc: Jan Beulich <jbeulich@suse.com>
Cc: Julien Grall <julien@xen.org>
Cc: Stefano Stabellini <sstabellini@kernel.org>
Cc: Michal Orzel <michal.orzel@amd.com>
Cc: xen-devel@lists.xenproject.org
---
 automation/scripts/qemu-alpine-x86_64.sh        | 3 +--
 automation/scripts/qemu-key.exp                 | 2 +-
 automation/scripts/qemu-smoke-dom0-arm32.sh     | 3 +--
 automation/scripts/qemu-smoke-dom0-arm64.sh     | 3 +--
 automation/scripts/qemu-smoke-dom0less-arm32.sh | 3 +--
 automation/scripts/qemu-smoke-dom0less-arm64.sh | 3 +--
 automation/scripts/qemu-smoke-ppc64le.sh        | 3 +--
 automation/scripts/qemu-smoke-riscv64.sh        | 3 +--
 automation/scripts/qemu-smoke-x86-64.sh         | 3 +--
 automation/scripts/qemu-xtf-dom0less-arm64.sh   | 3 +--
 10 files changed, 10 insertions(+), 19 deletions(-)

--
2.25.1

Comments

Stefano Stabellini Aug. 30, 2024, 12:48 a.m. UTC | #1 
On Thu, 29 Aug 2024, victorm.lira@amd.com wrote:
> From: Victor Lira <victorm.lira@amd.com>
> 
> Fix flaw in qemu-*.sh tests that producess a false success. The following
> lines produces success despite the "expect" script producing nonzero exit
> status:
> 
>     set +e
> ...
>     ./automation/scripts/qemu-key.exp | sed 's/\r\+$//'
>     (end of file)
> 
> The default exit status for a pipeline using "|" operator is that of the
> rightmost command. Fix this by setting the "pipefail" option in the shell,
> and removing "set +e" allowing the expect script to determine the result.
> 
> Signed-off-by: Victor Lira <victorm.lira@amd.com>


Reviewed-by: Stefano Stabellini <sstabellini@kernel.org>


> ---
> Cc: Andrew Cooper <andrew.cooper3@citrix.com>
> Cc: Jan Beulich <jbeulich@suse.com>
> Cc: Julien Grall <julien@xen.org>
> Cc: Stefano Stabellini <sstabellini@kernel.org>
> Cc: Michal Orzel <michal.orzel@amd.com>
> Cc: xen-devel@lists.xenproject.org
> ---
>  automation/scripts/qemu-alpine-x86_64.sh        | 3 +--
>  automation/scripts/qemu-key.exp                 | 2 +-
>  automation/scripts/qemu-smoke-dom0-arm32.sh     | 3 +--
>  automation/scripts/qemu-smoke-dom0-arm64.sh     | 3 +--
>  automation/scripts/qemu-smoke-dom0less-arm32.sh | 3 +--
>  automation/scripts/qemu-smoke-dom0less-arm64.sh | 3 +--
>  automation/scripts/qemu-smoke-ppc64le.sh        | 3 +--
>  automation/scripts/qemu-smoke-riscv64.sh        | 3 +--
>  automation/scripts/qemu-smoke-x86-64.sh         | 3 +--
>  automation/scripts/qemu-xtf-dom0less-arm64.sh   | 3 +--
>  10 files changed, 10 insertions(+), 19 deletions(-)
> 
> diff --git a/automation/scripts/qemu-alpine-x86_64.sh b/automation/scripts/qemu-alpine-x86_64.sh
> index 42a89e86b021..93914c41bc24 100755
> --- a/automation/scripts/qemu-alpine-x86_64.sh
> +++ b/automation/scripts/qemu-alpine-x86_64.sh
> @@ -1,6 +1,6 @@
>  #!/bin/bash
> 
> -set -ex
> +set -ex -o pipefail
> 
>  # DomU Busybox
>  cd binaries
> @@ -76,7 +76,6 @@ EOF
> 
>  # Run the test
>  rm -f smoke.serial
> -set +e
>  export QEMU_CMD="qemu-system-x86_64 \
>      -cpu qemu64,+svm \
>      -m 2G -smp 2 \
> diff --git a/automation/scripts/qemu-key.exp b/automation/scripts/qemu-key.exp
> index 787f1f08cb96..66c416483146 100755
> --- a/automation/scripts/qemu-key.exp
> +++ b/automation/scripts/qemu-key.exp
> @@ -14,7 +14,7 @@ eval spawn $env(QEMU_CMD)
> 
>  expect_after {
>      -re "(.*)\r" {
> -        exp_continue
> +        exp_continue -continue_timer
>      }
>      timeout {send_error "ERROR-Timeout!\n"; exit 1}
>      eof {send_error "ERROR-EOF!\n"; exit 1}
> diff --git a/automation/scripts/qemu-smoke-dom0-arm32.sh b/automation/scripts/qemu-smoke-dom0-arm32.sh
> index fd64b19358ae..7c282eff3a58 100755
> --- a/automation/scripts/qemu-smoke-dom0-arm32.sh
> +++ b/automation/scripts/qemu-smoke-dom0-arm32.sh
> @@ -1,6 +1,6 @@
>  #!/bin/bash
> 
> -set -ex
> +set -ex -o pipefail
> 
>  serial_log="$(pwd)/smoke.serial"
> 
> @@ -77,7 +77,6 @@ git clone --depth 1 https://gitlab.com/xen-project/imagebuilder.git
>  bash imagebuilder/scripts/uboot-script-gen -t tftp -d . -c config
> 
>  rm -f ${serial_log}
> -set +e
>  export QEMU_CMD="./qemu-system-arm \
>     -machine virt \
>     -machine virtualization=true \
> diff --git a/automation/scripts/qemu-smoke-dom0-arm64.sh b/automation/scripts/qemu-smoke-dom0-arm64.sh
> index e0cea742b0a0..81f210f7f50e 100755
> --- a/automation/scripts/qemu-smoke-dom0-arm64.sh
> +++ b/automation/scripts/qemu-smoke-dom0-arm64.sh
> @@ -1,6 +1,6 @@
>  #!/bin/bash
> 
> -set -ex
> +set -ex -o pipefail
> 
>  # DomU Busybox
>  cd binaries
> @@ -93,7 +93,6 @@ bash imagebuilder/scripts/uboot-script-gen -t tftp -d binaries/ -c binaries/conf
> 
>  # Run the test
>  rm -f smoke.serial
> -set +e
>  export QEMU_CMD="./binaries/qemu-system-aarch64 \
>      -machine virtualization=true \
>      -cpu cortex-a57 -machine type=virt \
> diff --git a/automation/scripts/qemu-smoke-dom0less-arm32.sh b/automation/scripts/qemu-smoke-dom0less-arm32.sh
> index e824cb7c2a3d..38e8a0b0bd7d 100755
> --- a/automation/scripts/qemu-smoke-dom0less-arm32.sh
> +++ b/automation/scripts/qemu-smoke-dom0less-arm32.sh
> @@ -1,6 +1,6 @@
>  #!/bin/bash
> 
> -set -ex
> +set -ex -o pipefail
> 
>  test_variant=$1
> 
> @@ -130,7 +130,6 @@ bash imagebuilder/scripts/uboot-script-gen -t tftp -d . -c config
> 
>  # Run the test
>  rm -f ${serial_log}
> -set +e
>  export QEMU_CMD="./qemu-system-arm \
>      -machine virt \
>      -machine virtualization=true \
> diff --git a/automation/scripts/qemu-smoke-dom0less-arm64.sh b/automation/scripts/qemu-smoke-dom0less-arm64.sh
> index f42ba5d196bc..ea67650e17da 100755
> --- a/automation/scripts/qemu-smoke-dom0less-arm64.sh
> +++ b/automation/scripts/qemu-smoke-dom0less-arm64.sh
> @@ -1,6 +1,6 @@
>  #!/bin/bash
> 
> -set -ex
> +set -ex -o pipefail
> 
>  test_variant=$1
> 
> @@ -204,7 +204,6 @@ bash imagebuilder/scripts/uboot-script-gen -t tftp -d binaries/ -c binaries/conf
> 
>  # Run the test
>  rm -f smoke.serial
> -set +e
>  export QEMU_CMD="./binaries/qemu-system-aarch64 \
>      -machine virtualization=true \
>      -cpu cortex-a57 -machine type=virt,gic-version=$gic_version \
> diff --git a/automation/scripts/qemu-smoke-ppc64le.sh b/automation/scripts/qemu-smoke-ppc64le.sh
> index 594f92c19cc2..49e189c37058 100755
> --- a/automation/scripts/qemu-smoke-ppc64le.sh
> +++ b/automation/scripts/qemu-smoke-ppc64le.sh
> @@ -1,6 +1,6 @@
>  #!/bin/bash
> 
> -set -ex
> +set -ex -o pipefail
> 
>  serial_log="$(pwd)/smoke.serial"
> 
> @@ -9,7 +9,6 @@ machine=$1
> 
>  # Run the test
>  rm -f ${serial_log}
> -set +e
> 
>  export QEMU_CMD="qemu-system-ppc64 \
>      -bios skiboot.lid \
> diff --git a/automation/scripts/qemu-smoke-riscv64.sh b/automation/scripts/qemu-smoke-riscv64.sh
> index c2595f657ff3..422ee03e0d26 100755
> --- a/automation/scripts/qemu-smoke-riscv64.sh
> +++ b/automation/scripts/qemu-smoke-riscv64.sh
> @@ -1,10 +1,9 @@
>  #!/bin/bash
> 
> -set -ex
> +set -ex -o pipefail
> 
>  # Run the test
>  rm -f smoke.serial
> -set +e
> 
>  export QEMU_CMD="qemu-system-riscv64 \
>      -M virt \
> diff --git a/automation/scripts/qemu-smoke-x86-64.sh b/automation/scripts/qemu-smoke-x86-64.sh
> index 3440b1761db4..7495185d9fc6 100755
> --- a/automation/scripts/qemu-smoke-x86-64.sh
> +++ b/automation/scripts/qemu-smoke-x86-64.sh
> @@ -1,6 +1,6 @@
>  #!/bin/bash
> 
> -set -ex
> +set -ex -o pipefail
> 
>  # variant should be either pv or pvh
>  variant=$1
> @@ -15,7 +15,6 @@ case $variant in
>  esac
> 
>  rm -f smoke.serial
> -set +e
>  export QEMU_CMD="qemu-system-x86_64 -nographic -kernel binaries/xen \
>          -initrd xtf/tests/example/$k \
>          -append \"loglvl=all console=com1 noreboot console_timestamps=boot $extra\" \
> diff --git a/automation/scripts/qemu-xtf-dom0less-arm64.sh b/automation/scripts/qemu-xtf-dom0less-arm64.sh
> index 4042fe50602b..acef1637e25b 100755
> --- a/automation/scripts/qemu-xtf-dom0less-arm64.sh
> +++ b/automation/scripts/qemu-xtf-dom0less-arm64.sh
> @@ -1,6 +1,6 @@
>  #!/bin/bash
> 
> -set -ex
> +set -ex -o pipefail
> 
>  # Name of the XTF test
>  xtf_test=$1
> @@ -50,7 +50,6 @@ bash imagebuilder/scripts/uboot-script-gen -t tftp -d binaries/ -c binaries/conf
> 
>  # Run the test
>  rm -f smoke.serial
> -set +e
>  export QEMU_CMD="./binaries/qemu-system-aarch64 \
>      -machine virtualization=true \
>      -cpu cortex-a57 -machine type=virt \
> --
> 2.25.1
>
diff mbox series

Patch

diff --git a/automation/scripts/qemu-alpine-x86_64.sh b/automation/scripts/qemu-alpine-x86_64.sh
index 42a89e86b021..93914c41bc24 100755
--- a/automation/scripts/qemu-alpine-x86_64.sh
+++ b/automation/scripts/qemu-alpine-x86_64.sh
@@ -1,6 +1,6 @@ 
 #!/bin/bash

-set -ex
+set -ex -o pipefail

 # DomU Busybox
 cd binaries
@@ -76,7 +76,6 @@  EOF

 # Run the test
 rm -f smoke.serial
-set +e
 export QEMU_CMD="qemu-system-x86_64 \
     -cpu qemu64,+svm \
     -m 2G -smp 2 \
diff --git a/automation/scripts/qemu-key.exp b/automation/scripts/qemu-key.exp
index 787f1f08cb96..66c416483146 100755
--- a/automation/scripts/qemu-key.exp
+++ b/automation/scripts/qemu-key.exp
@@ -14,7 +14,7 @@  eval spawn $env(QEMU_CMD)

 expect_after {
     -re "(.*)\r" {
-        exp_continue
+        exp_continue -continue_timer
     }
     timeout {send_error "ERROR-Timeout!\n"; exit 1}
     eof {send_error "ERROR-EOF!\n"; exit 1}
diff --git a/automation/scripts/qemu-smoke-dom0-arm32.sh b/automation/scripts/qemu-smoke-dom0-arm32.sh
index fd64b19358ae..7c282eff3a58 100755
--- a/automation/scripts/qemu-smoke-dom0-arm32.sh
+++ b/automation/scripts/qemu-smoke-dom0-arm32.sh
@@ -1,6 +1,6 @@ 
 #!/bin/bash

-set -ex
+set -ex -o pipefail

 serial_log="$(pwd)/smoke.serial"

@@ -77,7 +77,6 @@  git clone --depth 1 https://gitlab.com/xen-project/imagebuilder.git
 bash imagebuilder/scripts/uboot-script-gen -t tftp -d . -c config

 rm -f ${serial_log}
-set +e
 export QEMU_CMD="./qemu-system-arm \
    -machine virt \
    -machine virtualization=true \
diff --git a/automation/scripts/qemu-smoke-dom0-arm64.sh b/automation/scripts/qemu-smoke-dom0-arm64.sh
index e0cea742b0a0..81f210f7f50e 100755
--- a/automation/scripts/qemu-smoke-dom0-arm64.sh
+++ b/automation/scripts/qemu-smoke-dom0-arm64.sh
@@ -1,6 +1,6 @@ 
 #!/bin/bash

-set -ex
+set -ex -o pipefail

 # DomU Busybox
 cd binaries
@@ -93,7 +93,6 @@  bash imagebuilder/scripts/uboot-script-gen -t tftp -d binaries/ -c binaries/conf

 # Run the test
 rm -f smoke.serial
-set +e
 export QEMU_CMD="./binaries/qemu-system-aarch64 \
     -machine virtualization=true \
     -cpu cortex-a57 -machine type=virt \
diff --git a/automation/scripts/qemu-smoke-dom0less-arm32.sh b/automation/scripts/qemu-smoke-dom0less-arm32.sh
index e824cb7c2a3d..38e8a0b0bd7d 100755
--- a/automation/scripts/qemu-smoke-dom0less-arm32.sh
+++ b/automation/scripts/qemu-smoke-dom0less-arm32.sh
@@ -1,6 +1,6 @@ 
 #!/bin/bash

-set -ex
+set -ex -o pipefail

 test_variant=$1

@@ -130,7 +130,6 @@  bash imagebuilder/scripts/uboot-script-gen -t tftp -d . -c config

 # Run the test
 rm -f ${serial_log}
-set +e
 export QEMU_CMD="./qemu-system-arm \
     -machine virt \
     -machine virtualization=true \
diff --git a/automation/scripts/qemu-smoke-dom0less-arm64.sh b/automation/scripts/qemu-smoke-dom0less-arm64.sh
index f42ba5d196bc..ea67650e17da 100755
--- a/automation/scripts/qemu-smoke-dom0less-arm64.sh
+++ b/automation/scripts/qemu-smoke-dom0less-arm64.sh
@@ -1,6 +1,6 @@ 
 #!/bin/bash

-set -ex
+set -ex -o pipefail

 test_variant=$1

@@ -204,7 +204,6 @@  bash imagebuilder/scripts/uboot-script-gen -t tftp -d binaries/ -c binaries/conf

 # Run the test
 rm -f smoke.serial
-set +e
 export QEMU_CMD="./binaries/qemu-system-aarch64 \
     -machine virtualization=true \
     -cpu cortex-a57 -machine type=virt,gic-version=$gic_version \
diff --git a/automation/scripts/qemu-smoke-ppc64le.sh b/automation/scripts/qemu-smoke-ppc64le.sh
index 594f92c19cc2..49e189c37058 100755
--- a/automation/scripts/qemu-smoke-ppc64le.sh
+++ b/automation/scripts/qemu-smoke-ppc64le.sh
@@ -1,6 +1,6 @@ 
 #!/bin/bash

-set -ex
+set -ex -o pipefail

 serial_log="$(pwd)/smoke.serial"

@@ -9,7 +9,6 @@  machine=$1

 # Run the test
 rm -f ${serial_log}
-set +e

 export QEMU_CMD="qemu-system-ppc64 \
     -bios skiboot.lid \
diff --git a/automation/scripts/qemu-smoke-riscv64.sh b/automation/scripts/qemu-smoke-riscv64.sh
index c2595f657ff3..422ee03e0d26 100755
--- a/automation/scripts/qemu-smoke-riscv64.sh
+++ b/automation/scripts/qemu-smoke-riscv64.sh
@@ -1,10 +1,9 @@ 
 #!/bin/bash

-set -ex
+set -ex -o pipefail

 # Run the test
 rm -f smoke.serial
-set +e

 export QEMU_CMD="qemu-system-riscv64 \
     -M virt \
diff --git a/automation/scripts/qemu-smoke-x86-64.sh b/automation/scripts/qemu-smoke-x86-64.sh
index 3440b1761db4..7495185d9fc6 100755
--- a/automation/scripts/qemu-smoke-x86-64.sh
+++ b/automation/scripts/qemu-smoke-x86-64.sh
@@ -1,6 +1,6 @@ 
 #!/bin/bash

-set -ex
+set -ex -o pipefail

 # variant should be either pv or pvh
 variant=$1
@@ -15,7 +15,6 @@  case $variant in
 esac

 rm -f smoke.serial
-set +e
 export QEMU_CMD="qemu-system-x86_64 -nographic -kernel binaries/xen \
         -initrd xtf/tests/example/$k \
         -append \"loglvl=all console=com1 noreboot console_timestamps=boot $extra\" \
diff --git a/automation/scripts/qemu-xtf-dom0less-arm64.sh b/automation/scripts/qemu-xtf-dom0less-arm64.sh
index 4042fe50602b..acef1637e25b 100755
--- a/automation/scripts/qemu-xtf-dom0less-arm64.sh
+++ b/automation/scripts/qemu-xtf-dom0less-arm64.sh
@@ -1,6 +1,6 @@ 
 #!/bin/bash

-set -ex
+set -ex -o pipefail

 # Name of the XTF test
 xtf_test=$1
@@ -50,7 +50,6 @@  bash imagebuilder/scripts/uboot-script-gen -t tftp -d binaries/ -c binaries/conf

 # Run the test
 rm -f smoke.serial
-set +e
 export QEMU_CMD="./binaries/qemu-system-aarch64 \
     -machine virtualization=true \
     -cpu cortex-a57 -machine type=virt \