From patchwork Fri Jan 27 17:59:15 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Alexandru Elisei X-Patchwork-Id: 13119138 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 87253C61D97 for ; Fri, 27 Jan 2023 17:59:35 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S234916AbjA0R7e (ORCPT ); Fri, 27 Jan 2023 12:59:34 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:42130 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S234407AbjA0R7c (ORCPT ); Fri, 27 Jan 2023 12:59:32 -0500 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by lindbergh.monkeyblade.net (Postfix) with ESMTP id 349857D2B0 for ; Fri, 27 Jan 2023 09:59:30 -0800 (PST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id CBE782F; Fri, 27 Jan 2023 10:00:11 -0800 (PST) Received: from monolith.localdoman (unknown [172.31.20.19]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 357DD3F5A1; Fri, 27 Jan 2023 09:59:29 -0800 (PST) From: Alexandru Elisei To: andrew.jones@linux.dev, kvm@vger.kernel.org, kvmarm@lists.linux.dev Subject: [kvm-unit-tests PATCH v5 1/2] arm/psci: Test that CPU 1 has been successfully brought online Date: Fri, 27 Jan 2023 17:59:15 +0000 Message-Id: <20230127175916.65389-2-alexandru.elisei@arm.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230127175916.65389-1-alexandru.elisei@arm.com> References: <20230127175916.65389-1-alexandru.elisei@arm.com> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org For the PSCI CPU_ON function test, all other CPUs perform a CPU_ON call that target CPU 1. The test is considered a success if CPU_ON returns PSCI SUCCESS exactly once, and for the rest of the calls PSCI ALREADY_ON. Enhance the test by checking that CPU 1 is actually online and able to execute code. Also make the test more robust by checking that the CPU_ON call returns, instead of assuming that it will always succeed and hanging indefinitely if it doesn't. Since the CPU 1 thread is now being set up properly by kvm-unit-tests when being brought online, it becomes possible to add other tests in the future that require all CPUs. The include header order in arm/psci.c has been changed to be in alphabetic order. This means moving the errata.h include before libcflat.h, which causes compilation to fail because of missing includes in errata.h. Fix that also by including the needed header in errata.h. Signed-off-by: Alexandru Elisei Reviewed-by: Andrew Jones --- arm/psci.c | 67 ++++++++++++++++++++++++++++++++++++++--------- lib/arm/asm/smp.h | 9 ++++++- lib/arm/smp.c | 4 --- lib/errata.h | 2 ++ 4 files changed, 64 insertions(+), 18 deletions(-) diff --git a/arm/psci.c b/arm/psci.c index efa0722c0566..f7238f8e0bbd 100644 --- a/arm/psci.c +++ b/arm/psci.c @@ -7,11 +7,14 @@ * * This work is licensed under the terms of the GNU LGPL, version 2. */ -#include #include +#include + +#include +#include #include -#include #include +#include static bool invalid_function_exception; @@ -72,46 +75,84 @@ static int cpu_on_ret[NR_CPUS]; static cpumask_t cpu_on_ready, cpu_on_done; static volatile int cpu_on_start; -static void cpu_on_secondary_entry(void) +extern void secondary_entry(void); +static void cpu_on_do_wake_target(void) { int cpu = smp_processor_id(); cpumask_set_cpu(cpu, &cpu_on_ready); while (!cpu_on_start) cpu_relax(); - cpu_on_ret[cpu] = psci_cpu_on(cpus[1], __pa(halt)); + cpu_on_ret[cpu] = psci_cpu_on(cpus[1], __pa(secondary_entry)); cpumask_set_cpu(cpu, &cpu_on_done); } +static void cpu_on_target(void) +{ + int cpu = smp_processor_id(); + + cpumask_set_cpu(cpu, &cpu_on_done); +} + +extern struct secondary_data secondary_data; + +/* Open code the setup part from smp_boot_secondary(). */ +static void psci_cpu_on_prepare_secondary(int cpu, secondary_entry_fn entry) +{ + secondary_data.stack = thread_stack_alloc(); + secondary_data.entry = entry; + mmu_mark_disabled(cpu); +} + static bool psci_cpu_on_test(void) { bool failed = false; int ret_success = 0; - int cpu; - - cpumask_set_cpu(1, &cpu_on_ready); - cpumask_set_cpu(1, &cpu_on_done); + int i, cpu; for_each_present_cpu(cpu) { if (cpu < 2) continue; - smp_boot_secondary(cpu, cpu_on_secondary_entry); + smp_boot_secondary(cpu, cpu_on_do_wake_target); } cpumask_set_cpu(0, &cpu_on_ready); + cpumask_set_cpu(1, &cpu_on_ready); while (!cpumask_full(&cpu_on_ready)) cpu_relax(); + /* + * Configure CPU 1 after all secondaries are online to avoid + * secondary_data being overwritten. + */ + psci_cpu_on_prepare_secondary(1, cpu_on_target); + cpu_on_start = 1; smp_mb(); - cpu_on_ret[0] = psci_cpu_on(cpus[1], __pa(halt)); + cpu_on_ret[0] = psci_cpu_on(cpus[1], __pa(secondary_entry)); cpumask_set_cpu(0, &cpu_on_done); - while (!cpumask_full(&cpu_on_done)) - cpu_relax(); + report_info("waiting for CPU1 to come online..."); + for (i = 0; i < 100; i++) { + mdelay(10); + if (cpumask_full(&cpu_on_done)) + break; + } - for_each_present_cpu(cpu) { + if (!cpumask_full(&cpu_on_done)) { + for_each_present_cpu(cpu) { + if (!cpumask_test_cpu(cpu, &cpu_on_done)) { + if (cpu == 1) + report_info("CPU1 failed to come online"); + else + report_info("CPU%d failed to online CPU1", cpu); + } + } + failed = true; + } + + for_each_cpu(cpu, &cpu_on_done) { if (cpu == 1) continue; if (cpu_on_ret[cpu] == PSCI_RET_SUCCESS) { diff --git a/lib/arm/asm/smp.h b/lib/arm/asm/smp.h index 077afde85520..dee4c1a883e7 100644 --- a/lib/arm/asm/smp.h +++ b/lib/arm/asm/smp.h @@ -10,6 +10,14 @@ #define smp_processor_id() (current_thread_info()->cpu) +typedef void (*secondary_entry_fn)(void); + +struct secondary_data { + void *stack; /* must be first member of struct */ + secondary_entry_fn entry; +}; +extern struct secondary_data secondary_data; + extern bool cpu0_calls_idle; extern void halt(void); @@ -48,7 +56,6 @@ static inline void set_cpu_idle(int cpu, bool idle) cpumask_clear_cpu(cpu, &cpu_idle_mask); } -typedef void (*secondary_entry_fn)(void); extern void smp_boot_secondary(int cpu, secondary_entry_fn entry); extern void on_cpu_async(int cpu, void (*func)(void *data), void *data); extern void on_cpu(int cpu, void (*func)(void *data), void *data); diff --git a/lib/arm/smp.c b/lib/arm/smp.c index 98a5054e039b..1d470d1aab45 100644 --- a/lib/arm/smp.c +++ b/lib/arm/smp.c @@ -21,10 +21,6 @@ cpumask_t cpu_present_mask; cpumask_t cpu_online_mask; cpumask_t cpu_idle_mask; -struct secondary_data { - void *stack; /* must be first member of struct */ - secondary_entry_fn entry; -}; struct secondary_data secondary_data; static struct spinlock lock; diff --git a/lib/errata.h b/lib/errata.h index 5af0eb3bf8e2..de8205d8b370 100644 --- a/lib/errata.h +++ b/lib/errata.h @@ -6,6 +6,8 @@ */ #ifndef _ERRATA_H_ #define _ERRATA_H_ +#include + #include "config.h" #ifndef CONFIG_ERRATA_FORCE From patchwork Fri Jan 27 17:59:16 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Alexandru Elisei X-Patchwork-Id: 13119139 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 44A88C54EAA for ; Fri, 27 Jan 2023 17:59:36 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S234834AbjA0R7f (ORCPT ); Fri, 27 Jan 2023 12:59:35 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:42140 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S234014AbjA0R7c (ORCPT ); Fri, 27 Jan 2023 12:59:32 -0500 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by lindbergh.monkeyblade.net (Postfix) with ESMTP id 2B3657D986 for ; Fri, 27 Jan 2023 09:59:31 -0800 (PST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id C916415BF; Fri, 27 Jan 2023 10:00:12 -0800 (PST) Received: from monolith.localdoman (unknown [172.31.20.19]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 301B63F5A1; Fri, 27 Jan 2023 09:59:30 -0800 (PST) From: Alexandru Elisei To: andrew.jones@linux.dev, kvm@vger.kernel.org, kvmarm@lists.linux.dev Subject: [kvm-unit-tests PATCH v5 2/2] arm/psci: Add PSCI CPU_OFF test case Date: Fri, 27 Jan 2023 17:59:16 +0000 Message-Id: <20230127175916.65389-3-alexandru.elisei@arm.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230127175916.65389-1-alexandru.elisei@arm.com> References: <20230127175916.65389-1-alexandru.elisei@arm.com> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org From: Nikita Venkatesh The test uses the following method. The primary CPU brings up all the secondary CPUs, which are held in a wait loop. Once the primary releases the CPUs, each of the secondary CPUs proceed to issue CPU_OFF. The primary CPU then checks for the status of the individual CPU_OFF request. There is a chance that some CPUs might return from the CPU_OFF function call after the primary CPU has finished the scan. There is no foolproof method to handle this, but the test tries its best to eliminate these false positives by introducing an extra delay if all the CPUs are reported offline after the initial scan. Signed-off-by: Nikita Venkatesh [ Alex E: Skip CPU_OFF test if CPU_ON failed, drop cpu_off_success in favour of checking AFFINITY_INFO, commit message tweaking ] Signed-off-by: Alexandru Elisei --- Decided to drop Drew's Reviewed-by tag because the changes are not trivial from the previous version. arm/psci.c | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 5 deletions(-) diff --git a/arm/psci.c b/arm/psci.c index f7238f8e0bbd..7034d8ebe6e1 100644 --- a/arm/psci.c +++ b/arm/psci.c @@ -72,8 +72,9 @@ static bool psci_affinity_info_off(void) } static int cpu_on_ret[NR_CPUS]; -static cpumask_t cpu_on_ready, cpu_on_done; +static cpumask_t cpu_on_ready, cpu_on_done, cpu_off_done; static volatile int cpu_on_start; +static volatile int cpu_off_start; extern void secondary_entry(void); static void cpu_on_do_wake_target(void) @@ -171,9 +172,71 @@ static bool psci_cpu_on_test(void) return !failed; } -int main(void) +static void cpu_off_secondary_entry(void *data) +{ + int cpu = smp_processor_id(); + + while (!cpu_off_start) + cpu_relax(); + cpumask_set_cpu(cpu, &cpu_off_done); + cpu_psci_cpu_die(); +} + +static bool psci_cpu_off_test(void) +{ + bool failed = false; + int i, count, cpu; + + for_each_present_cpu(cpu) { + if (cpu == 0) + continue; + on_cpu_async(cpu, cpu_off_secondary_entry, NULL); + } + + cpumask_set_cpu(0, &cpu_off_done); + + cpu_off_start = 1; + report_info("waiting for the CPUs to be offlined..."); + while (!cpumask_full(&cpu_off_done)) + cpu_relax(); + + /* Allow all the other CPUs to complete the operation */ + for (i = 0; i < 100; i++) { + mdelay(10); + + count = 0; + for_each_present_cpu(cpu) { + if (cpu == 0) + continue; + if (psci_affinity_info(cpus[cpu], 0) != PSCI_0_2_AFFINITY_LEVEL_OFF) + count++; + } + if (count > 0) + continue; + } + + /* Try to catch CPUs that return from CPU_OFF. */ + if (count == 0) + mdelay(100); + + for_each_present_cpu(cpu) { + if (cpu == 0) + continue; + if (cpu_idle(cpu)) { + report_info("CPU%d failed to be offlined", cpu); + if (psci_affinity_info(cpus[cpu], 0) == PSCI_0_2_AFFINITY_LEVEL_OFF) + report_info("AFFINITY_INFO incorrectly reports CPU%d as offline", cpu); + failed = true; + } + } + + return !failed; +} + +int main(int argc, char **argv) { int ver = psci_invoke(PSCI_0_2_FN_PSCI_VERSION, 0, 0, 0); + bool cpu_on_success = true; report_prefix_push("psci"); @@ -188,10 +251,17 @@ int main(void) report(psci_affinity_info_on(), "affinity-info-on"); report(psci_affinity_info_off(), "affinity-info-off"); - if (ERRATA(6c7a5dce22b3)) - report(psci_cpu_on_test(), "cpu-on"); - else + if (ERRATA(6c7a5dce22b3)) { + cpu_on_success = psci_cpu_on_test(); + report(cpu_on_success, "cpu-on"); + } else { report_skip("Skipping unsafe cpu-on test. Set ERRATA_6c7a5dce22b3=y to enable."); + } + + if (!cpu_on_success) + report_skip("Skipping cpu-off test because the cpu-on test failed"); + else + report(psci_cpu_off_test(), "cpu-off"); done: #if 0