diff mbox series

[3/5] automation: Add the expect script with test case for FVP

Message ID 20231207135318.1912846-4-Henry.Wang@arm.com (mailing list archive)
State Superseded
Headers show
Series automation: Support running FVP Dom0 smoke test for Arm | expand

Commit Message

Henry Wang Dec. 7, 2023, 1:53 p.m. UTC 
To interact with the FVP (for example entering the U-Boot shell
and transferring the files by TFTP), we need to connect the
corresponding port by the telnet first. Use an expect script to
simplify the automation of the whole "interacting with FVP" stuff.

The expect script will firstly detect the IP of the host, then
connect to the telnet port of the FVP, set the `serverip` and `ipaddr`
for the TFTP service in the U-Boot shell, and finally boot Xen from
U-Boot and wait for the expected results by Xen, Dom0 and DomU.

Signed-off-by: Henry Wang <Henry.Wang@arm.com>
---
 .../expect/fvp-base-smoke-dom0-arm64.exp      | 73 +++++++++++++++++++
 1 file changed, 73 insertions(+)
 create mode 100755 automation/scripts/expect/fvp-base-smoke-dom0-arm64.exp

Comments

Stefano Stabellini Dec. 8, 2023, 1:38 a.m. UTC | #1 
On Thu, 7 Dec 2023, Henry Wang wrote:
> To interact with the FVP (for example entering the U-Boot shell
> and transferring the files by TFTP), we need to connect the
> corresponding port by the telnet first. Use an expect script to
> simplify the automation of the whole "interacting with FVP" stuff.
> 
> The expect script will firstly detect the IP of the host, then
> connect to the telnet port of the FVP, set the `serverip` and `ipaddr`
> for the TFTP service in the U-Boot shell, and finally boot Xen from
> U-Boot and wait for the expected results by Xen, Dom0 and DomU.

I am not an expert in "expect" but this script looks great


> Signed-off-by: Henry Wang <Henry.Wang@arm.com>
> ---
>  .../expect/fvp-base-smoke-dom0-arm64.exp      | 73 +++++++++++++++++++
>  1 file changed, 73 insertions(+)
>  create mode 100755 automation/scripts/expect/fvp-base-smoke-dom0-arm64.exp
> 
> diff --git a/automation/scripts/expect/fvp-base-smoke-dom0-arm64.exp b/automation/scripts/expect/fvp-base-smoke-dom0-arm64.exp
> new file mode 100755
> index 0000000000..25d9a5f81c
> --- /dev/null
> +++ b/automation/scripts/expect/fvp-base-smoke-dom0-arm64.exp
> @@ -0,0 +1,73 @@
> +#!/usr/bin/expect
> +
> +set timeout 2000
> +
> +# Command to use to run must be given as first argument
> +# if options are required, quotes must be used:
> +# xxx.exp "cmd opt1 opt2"
> +set runcmd [lindex $argv 0]
> +
> +# Maximum number of line to be printed, this can be used to prevent runs to
> +# go forever on errors when Xen is rebooting
> +set maxline 1000
> +
> +# Configure slow parameters used with send -s
> +# This allows us to slow down console writes to prevent issues with slow
> +# emulators or targets.
> +# Format here is: {NUM TIME} which reads as wait TIME seconds each NUM of
> +# characters, here we send 1 char each 100ms
> +set send_slow {1 .1}
> +
> +proc test_boot {{maxline} {host_ip}} {
> +    expect_after {
> +        -re "(.*)\r" {
> +            if {$maxline != 0} {
> +                set maxline [expr {$maxline - 1}]
> +                if {$maxline <= 0} {
> +                    send_user "ERROR-Toomuch!\n"
> +                    exit 2
> +                }
> +            }
> +            exp_continue
> +        }
> +        timeout {send_user "ERROR-Timeout!\n"; exit 3}
> +        eof {send_user "ERROR-EOF!\n"; exit 4}
> +    }

Why do we need this "expect_after" ?


> +    # Extract the telnet port numbers from FVP output, because the telnet ports
> +    # are not guaranteed to be fixed numbers.
> +    expect -re {terminal_0: Listening for serial connection on port [0-9]+}
> +    set terminal_0 $expect_out(0,string)
> +    if {[regexp {port (\d+)} $terminal_0 match port_0]} {
> +        puts "terminal_0 port is $port_0"
> +    } else {
> +        puts "terminal_0 port not found"
> +        exit 5
> +    }
> +
> +    spawn bash -c "telnet localhost $port_0"
> +    expect -re "Hit any key to stop autoboot.*"
> +    send -s "  \r"
> +    send -s "setenv serverip $host_ip; setenv ipaddr $host_ip; tftpb 0x80200000 boot.scr; source 0x80200000\r"
> +
> +    # Initial Xen boot
> +    expect -re "\(XEN\).*Freed .* init memory."
> +
> +    # Dom0 and DomU
> +    expect -re "Domain-0.*"
> +    expect -re "BusyBox.*"
> +    expect -re "/ #.*"

This is clear, excellent


> +}
> +
> +# Get host IP
> +spawn bash -c "hostname -I | awk '{print \$1}'"
> +expect -re {(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})}

Why d{1,3}?


> +set host_ip $expect_out(0,string)
> +
> +# Start the FVP and run the test
> +spawn bash -c "$runcmd"
> +
> +test_boot 2000 "$host_ip"
> +
> +send_user "\nExecution with SUCCESS\n"

Won't this always return SUCCESS even in case of failure?


> +exit 0
> -- 
> 2.25.1
>
Henry Wang Dec. 8, 2023, 2:03 a.m. UTC | #2 
Hi Stefano,

> On Dec 8, 2023, at 09:38, Stefano Stabellini <sstabellini@kernel.org> wrote:
> 
> On Thu, 7 Dec 2023, Henry Wang wrote:
>> To interact with the FVP (for example entering the U-Boot shell
>> and transferring the files by TFTP), we need to connect the
>> corresponding port by the telnet first. Use an expect script to
>> simplify the automation of the whole "interacting with FVP" stuff.
>> 
>> The expect script will firstly detect the IP of the host, then
>> connect to the telnet port of the FVP, set the `serverip` and `ipaddr`
>> for the TFTP service in the U-Boot shell, and finally boot Xen from
>> U-Boot and wait for the expected results by Xen, Dom0 and DomU.
> 
> I am not an expert in "expect" but this script looks great

:)

> 
> 
>> Signed-off-by: Henry Wang <Henry.Wang@arm.com>
>> ---
>> .../expect/fvp-base-smoke-dom0-arm64.exp      | 73 +++++++++++++++++++
>> 1 file changed, 73 insertions(+)
>> create mode 100755 automation/scripts/expect/fvp-base-smoke-dom0-arm64.exp
>> 
>> +
>> +proc test_boot {{maxline} {host_ip}} {
>> +    expect_after {
>> +        -re "(.*)\r" {
>> +            if {$maxline != 0} {
>> +                set maxline [expr {$maxline - 1}]
>> +                if {$maxline <= 0} {
>> +                    send_user "ERROR-Toomuch!\n"
>> +                    exit 2
>> +                }
>> +            }
>> +            exp_continue
>> +        }
>> +        timeout {send_user "ERROR-Timeout!\n"; exit 3}
>> +        eof {send_user "ERROR-EOF!\n"; exit 4}
>> +    }
> 
> Why do we need this "expect_after" ?

It is basically for the error handling. Either there is too many characters triggered
by some misconfiguration of Xen/Kernel leading to Xen/Kernel constantly reboots,
or the code stuck somehow, we have a way to stop the script instead of waiting in
the infinity loop.

>> +    # Extract the telnet port numbers from FVP output, because the telnet ports
>> +    # are not guaranteed to be fixed numbers.
>> +    expect -re {terminal_0: Listening for serial connection on port [0-9]+}
>> +    set terminal_0 $expect_out(0,string)
>> +    if {[regexp {port (\d+)} $terminal_0 match port_0]} {
>> +        puts "terminal_0 port is $port_0"
>> +    } else {
>> +        puts "terminal_0 port not found"
>> +        exit 5
>> +    }
>> +
>> +    spawn bash -c "telnet localhost $port_0"
>> +    expect -re "Hit any key to stop autoboot.*"
>> +    send -s "  \r"
>> +    send -s "setenv serverip $host_ip; setenv ipaddr $host_ip; tftpb 0x80200000 boot.scr; source 0x80200000\r"
>> +
>> +    # Initial Xen boot
>> +    expect -re "\(XEN\).*Freed .* init memory."
>> +
>> +    # Dom0 and DomU
>> +    expect -re "Domain-0.*"
>> +    expect -re "BusyBox.*"
>> +    expect -re "/ #.*"
> 
> This is clear, excellent
> 
>> +}
>> +
>> +# Get host IP
>> +spawn bash -c "hostname -I | awk '{print \$1}'"
>> +expect -re {(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})}
> 
> Why d{1,3}?

I think that is because the IP pattern, we at least have one digit and at most 3.

>> +set host_ip $expect_out(0,string)
>> +
>> +# Start the FVP and run the test
>> +spawn bash -c "$runcmd"
>> +
>> +test_boot 2000 "$host_ip"
>> +
>> +send_user "\nExecution with SUCCESS\n"
> 
> Won't this always return SUCCESS even in case of failure?

IMHO, if things fails, we have various exit code (1-5) for each failure case. For example,
if the FVP port somehow cannot be found, we exit with error code 5. This will basically lead
us to the only case where the failure is caused by the script fails to wait for the expected
string/regexp, and this case we have the timeout failure triggered by my above-mentioned
expect_after block.

Kind regards,
Henry

>> +exit 0
>> -- 
>> 2.25.1
>>
Henry Wang Dec. 8, 2023, 3:01 a.m. UTC | #3 
Hi Stefano,

> On Dec 8, 2023, at 10:03, Henry Wang <Henry.Wang@arm.com> wrote:
> 
> Hi Stefano,
> 
>> On Dec 8, 2023, at 09:38, Stefano Stabellini <sstabellini@kernel.org> wrote:
>>> +set host_ip $expect_out(0,string)
>>> +
>>> +# Start the FVP and run the test
>>> +spawn bash -c "$runcmd"
>>> +
>>> +test_boot 2000 "$host_ip"
>>> +
>>> +send_user "\nExecution with SUCCESS\n"
>> 
>> Won't this always return SUCCESS even in case of failure?
> 
> IMHO, if things fails, we have various exit code (1-5) for each failure case. For example,
> if the FVP port somehow cannot be found, we exit with error code 5. This will basically lead
> us to the only case where the failure is caused by the script fails to wait for the expected
> string/regexp, and this case we have the timeout failure triggered by my above-mentioned
> expect_after block.

I did a test to see if I break the expect script by adding below hunk:
```
--- a/automation/scripts/expect/fvp-base-smoke-dom0-arm64.exp
+++ b/automation/scripts/expect/fvp-base-smoke-dom0-arm64.exp
@@ -51,6 +51,7 @@ proc test_boot {{maxline} {host_ip}} {
     send -s "setenv serverip $host_ip; setenv ipaddr $host_ip; tftpb 0x80200000 boot.scr; source 0x80200000\r"

     # Initial Xen boot
+    expect -re "this is a hack to break the build"
     expect -re "\(XEN\).*Freed .* init memory."

     # Dom0 and DomU
```
The timeout did happen in the expect script after the set timeout, see [1]

However the job still passes, and I believe this is caused by the shell script:
```
./automation/scripts/expect/fvp-base-smoke-dom0-arm64.exp \
    "/FVP/FVP_Base_RevC-2xAEMvA/Base_RevC_AEMvA_pkg/models/Linux64_armv8l_GCC-9.3/FVP_Base_RevC-2xAEMvA \
    -C bp.vis.disable_visualisation=1 \
    -C bp.ve_sysregs.exit_on_shutdown=1 \
    -C bp.secure_memory=0 \
    -C cache_state_modelled=0 \
    -C cluster0.has_arm_v8-4=1 \
    -C cluster1.has_arm_v8-4=1 \
    ${TERM0_CFG} ${TERM1_CFG} ${TERM2_CFG} ${TERM3_CFG} \
    ${VIRTIO_USER_NETWORK_CFG} \
    -C bp.secureflashloader.fname=$(pwd)/binaries/bl1.bin \
    -C bp.flashloader0.fname=$(pwd)/binaries/fip.bin" |& \
        tee smoke.serial

exit 0
```

The “|& tee smoke.serial” hides the error propagated by the expect script. I will send a v2 to fix it.

[1] https://gitlab.com/xen-project/people/henryw/xen/-/jobs/5708263782/artifacts/file/smoke.serial

Kind regards,
Henry

> 
> Kind regards,
> Henry
> 
>>> +exit 0
>>> -- 
>>> 2.25.1
>>> 
>
diff mbox series

Patch

diff --git a/automation/scripts/expect/fvp-base-smoke-dom0-arm64.exp b/automation/scripts/expect/fvp-base-smoke-dom0-arm64.exp
new file mode 100755
index 0000000000..25d9a5f81c
--- /dev/null
+++ b/automation/scripts/expect/fvp-base-smoke-dom0-arm64.exp
@@ -0,0 +1,73 @@ 
+#!/usr/bin/expect
+
+set timeout 2000
+
+# Command to use to run must be given as first argument
+# if options are required, quotes must be used:
+# xxx.exp "cmd opt1 opt2"
+set runcmd [lindex $argv 0]
+
+# Maximum number of line to be printed, this can be used to prevent runs to
+# go forever on errors when Xen is rebooting
+set maxline 1000
+
+# Configure slow parameters used with send -s
+# This allows us to slow down console writes to prevent issues with slow
+# emulators or targets.
+# Format here is: {NUM TIME} which reads as wait TIME seconds each NUM of
+# characters, here we send 1 char each 100ms
+set send_slow {1 .1}
+
+proc test_boot {{maxline} {host_ip}} {
+    expect_after {
+        -re "(.*)\r" {
+            if {$maxline != 0} {
+                set maxline [expr {$maxline - 1}]
+                if {$maxline <= 0} {
+                    send_user "ERROR-Toomuch!\n"
+                    exit 2
+                }
+            }
+            exp_continue
+        }
+        timeout {send_user "ERROR-Timeout!\n"; exit 3}
+        eof {send_user "ERROR-EOF!\n"; exit 4}
+    }
+
+    # Extract the telnet port numbers from FVP output, because the telnet ports
+    # are not guaranteed to be fixed numbers.
+    expect -re {terminal_0: Listening for serial connection on port [0-9]+}
+    set terminal_0 $expect_out(0,string)
+    if {[regexp {port (\d+)} $terminal_0 match port_0]} {
+        puts "terminal_0 port is $port_0"
+    } else {
+        puts "terminal_0 port not found"
+        exit 5
+    }
+
+    spawn bash -c "telnet localhost $port_0"
+    expect -re "Hit any key to stop autoboot.*"
+    send -s "  \r"
+    send -s "setenv serverip $host_ip; setenv ipaddr $host_ip; tftpb 0x80200000 boot.scr; source 0x80200000\r"
+
+    # Initial Xen boot
+    expect -re "\(XEN\).*Freed .* init memory."
+
+    # Dom0 and DomU
+    expect -re "Domain-0.*"
+    expect -re "BusyBox.*"
+    expect -re "/ #.*"
+}
+
+# Get host IP
+spawn bash -c "hostname -I | awk '{print \$1}'"
+expect -re {(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})}
+set host_ip $expect_out(0,string)
+
+# Start the FVP and run the test
+spawn bash -c "$runcmd"
+
+test_boot 2000 "$host_ip"
+
+send_user "\nExecution with SUCCESS\n"
+exit 0