From patchwork Wed Jan 18 14:49:11 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Alexandru Elisei X-Patchwork-Id: 13106477 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id F3A1AC32793 for ; Wed, 18 Jan 2023 14:55:49 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229512AbjAROzs (ORCPT ); Wed, 18 Jan 2023 09:55:48 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:53312 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231768AbjAROzQ (ORCPT ); Wed, 18 Jan 2023 09:55:16 -0500 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by lindbergh.monkeyblade.net (Postfix) with ESMTP id C35E94ED03 for ; Wed, 18 Jan 2023 06:49:36 -0800 (PST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id C61B71477; Wed, 18 Jan 2023 06:50:18 -0800 (PST) Received: from e121798.cambridge.arm.com (e121798.cambridge.arm.com [10.1.196.158]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 05D223F67D; Wed, 18 Jan 2023 06:49:35 -0800 (PST) From: Alexandru Elisei To: andrew.jones@linux.dev, kvm@vger.kernel.org, kvmarm@lists.linux.dev Subject: [kvm-unit-tests PATCH v4 1/2] arm/psci: Test that CPU 1 has been successfully brought online Date: Wed, 18 Jan 2023 14:49:11 +0000 Message-Id: <20230118144912.32049-2-alexandru.elisei@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20230118144912.32049-1-alexandru.elisei@arm.com> References: <20230118144912.32049-1-alexandru.elisei@arm.com> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org For the PSCI CPU_ON function test, all other CPUs perform a CPU_ON call that target CPU 1. The test is considered a success if CPU_ON returns PSCI SUCCESS exactly once, and for the rest of the calls PSCI ALREADY_ON. Enhance the test by checking that CPU 1 is actually online and able to execute code. Also make the test more robust by checking that the CPU_ON call returns, instead of assuming that it will always succeed and hanging indefinitely if it doesn't. Since the CPU 1 thread is now being set up properly by kvm-unit-tests when being brought online, it becomes possible to add other tests in the future that require all CPUs. The include header order in arm/psci.c has been changed to be in alphabetic order. This means moving the errata.h include before libcflat.h, which causes compilation to fail because of missing includes in errata.h. Fix that also by including the needed header in errata.h. Signed-off-by: Alexandru Elisei --- arm/psci.c | 60 ++++++++++++++++++++++++++++++++++++----------- lib/arm/asm/smp.h | 1 + lib/arm/smp.c | 12 +++++++--- lib/errata.h | 2 ++ 4 files changed, 58 insertions(+), 17 deletions(-) diff --git a/arm/psci.c b/arm/psci.c index efa0722c0566..e96be941953b 100644 --- a/arm/psci.c +++ b/arm/psci.c @@ -7,11 +7,13 @@ * * This work is licensed under the terms of the GNU LGPL, version 2. */ -#include #include +#include + +#include #include -#include #include +#include static bool invalid_function_exception; @@ -72,14 +74,23 @@ static int cpu_on_ret[NR_CPUS]; static cpumask_t cpu_on_ready, cpu_on_done; static volatile int cpu_on_start; -static void cpu_on_secondary_entry(void) +extern void secondary_entry(void); +static void cpu_on_do_wake_target(void) { int cpu = smp_processor_id(); cpumask_set_cpu(cpu, &cpu_on_ready); while (!cpu_on_start) cpu_relax(); - cpu_on_ret[cpu] = psci_cpu_on(cpus[1], __pa(halt)); + cpu_on_ret[cpu] = psci_cpu_on(cpus[1], __pa(secondary_entry)); + cpumask_set_cpu(cpu, &cpu_on_done); +} + +static void cpu_on_target(void) +{ + int cpu = smp_processor_id(); + + cpu_on_ret[cpu] = PSCI_RET_ALREADY_ON; cpumask_set_cpu(cpu, &cpu_on_done); } @@ -87,33 +98,53 @@ static bool psci_cpu_on_test(void) { bool failed = false; int ret_success = 0; - int cpu; - - cpumask_set_cpu(1, &cpu_on_ready); - cpumask_set_cpu(1, &cpu_on_done); + int i, cpu; for_each_present_cpu(cpu) { if (cpu < 2) continue; - smp_boot_secondary(cpu, cpu_on_secondary_entry); + smp_boot_secondary(cpu, cpu_on_do_wake_target); } cpumask_set_cpu(0, &cpu_on_ready); + cpumask_set_cpu(1, &cpu_on_ready); while (!cpumask_full(&cpu_on_ready)) cpu_relax(); + /* + * Wait for all other CPUs to be online before configuring the thread + * for the target CPU, as all secondaries are set up using the same + * global variable. + */ + smp_prepare_secondary(1, cpu_on_target); + cpu_on_start = 1; smp_mb(); - cpu_on_ret[0] = psci_cpu_on(cpus[1], __pa(halt)); + cpu_on_ret[0] = psci_cpu_on(cpus[1], __pa(secondary_entry)); cpumask_set_cpu(0, &cpu_on_done); - while (!cpumask_full(&cpu_on_done)) - cpu_relax(); + report_info("waiting for CPU1 to come online..."); + for (i = 0; i < 10; i++) { + mdelay(100); + if (cpumask_full(&cpu_on_done)) + break; + } + + if (!cpumask_full(&cpu_on_done)) { + for_each_present_cpu(cpu) { + if (!cpumask_test_cpu(cpu, &cpu_on_done)) { + if (cpu == 1) + report_info("CPU1 failed to come online"); + else + report_info("CPU%d failed to online CPU1", cpu); + } + } + failed = true; + goto out; + } for_each_present_cpu(cpu) { - if (cpu == 1) - continue; if (cpu_on_ret[cpu] == PSCI_RET_SUCCESS) { ret_success++; } else if (cpu_on_ret[cpu] != PSCI_RET_ALREADY_ON) { @@ -127,6 +158,7 @@ static bool psci_cpu_on_test(void) failed = true; } +out: return !failed; } diff --git a/lib/arm/asm/smp.h b/lib/arm/asm/smp.h index 077afde85520..ff2ef8f88247 100644 --- a/lib/arm/asm/smp.h +++ b/lib/arm/asm/smp.h @@ -49,6 +49,7 @@ static inline void set_cpu_idle(int cpu, bool idle) } typedef void (*secondary_entry_fn)(void); +extern void smp_prepare_secondary(int cpu, secondary_entry_fn entry); extern void smp_boot_secondary(int cpu, secondary_entry_fn entry); extern void on_cpu_async(int cpu, void (*func)(void *data), void *data); extern void on_cpu(int cpu, void (*func)(void *data), void *data); diff --git a/lib/arm/smp.c b/lib/arm/smp.c index 98a5054e039b..947f417f4aea 100644 --- a/lib/arm/smp.c +++ b/lib/arm/smp.c @@ -58,13 +58,19 @@ secondary_entry_fn secondary_cinit(void) return entry; } -static void __smp_boot_secondary(int cpu, secondary_entry_fn entry) +void smp_prepare_secondary(int cpu, secondary_entry_fn entry) { - int ret; - secondary_data.stack = thread_stack_alloc(); secondary_data.entry = entry; mmu_mark_disabled(cpu); +} + +static void __smp_boot_secondary(int cpu, secondary_entry_fn entry) +{ + int ret; + + smp_prepare_secondary(cpu, entry); + ret = cpu_psci_cpu_boot(cpu); assert(ret == 0); diff --git a/lib/errata.h b/lib/errata.h index 5af0eb3bf8e2..de8205d8b370 100644 --- a/lib/errata.h +++ b/lib/errata.h @@ -6,6 +6,8 @@ */ #ifndef _ERRATA_H_ #define _ERRATA_H_ +#include + #include "config.h" #ifndef CONFIG_ERRATA_FORCE From patchwork Wed Jan 18 14:49:12 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Alexandru Elisei X-Patchwork-Id: 13106478 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 04A29C32793 for ; Wed, 18 Jan 2023 14:55:53 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229880AbjAROzu (ORCPT ); Wed, 18 Jan 2023 09:55:50 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:55368 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231775AbjAROzS (ORCPT ); Wed, 18 Jan 2023 09:55:18 -0500 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by lindbergh.monkeyblade.net (Postfix) with ESMTP id 423B963082 for ; Wed, 18 Jan 2023 06:49:38 -0800 (PST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 3C4B1169C; Wed, 18 Jan 2023 06:50:20 -0800 (PST) Received: from e121798.cambridge.arm.com (e121798.cambridge.arm.com [10.1.196.158]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 701C53F67D; Wed, 18 Jan 2023 06:49:37 -0800 (PST) From: Alexandru Elisei To: andrew.jones@linux.dev, kvm@vger.kernel.org, kvmarm@lists.linux.dev Subject: [kvm-unit-tests PATCH v4 2/2] arm/psci: Add PSCI CPU_OFF test case Date: Wed, 18 Jan 2023 14:49:12 +0000 Message-Id: <20230118144912.32049-3-alexandru.elisei@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20230118144912.32049-1-alexandru.elisei@arm.com> References: <20230118144912.32049-1-alexandru.elisei@arm.com> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org From: Nikita Venkatesh The test uses the following method. The primary CPU brings up all the secondary CPUs, which are held in a wait loop. Once the primary releases the CPUs, each of the secondary CPUs proceed to issue PSCI_CPU_OFF. This is indicated by a cpumask and also the status of the call is updated by the secondary CPU in cpu_off_done[]. The primary CPU waits for all the secondary CPUs to update the cpumask and then proceeds to check for the status of the individual CPU CPU_OFF request. There is a chance that some CPUs might fail at the CPU_OFF request and come back and update the status once the primary CPU has finished the scan. There is no fool proof method to handle this. As of now, we add a 1sec delay between the cpumask check and the scan for the status. Signed-off-by: Nikita Venkatesh [ Alex E: Skip CPU_OFF test if CPU_ON failed ] Signed-off-by: Alexandru Elisei Reviewed-by: Andrew Jones --- arm/psci.c | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 5 deletions(-) diff --git a/arm/psci.c b/arm/psci.c index e96be941953b..d045616bfcd4 100644 --- a/arm/psci.c +++ b/arm/psci.c @@ -15,6 +15,8 @@ #include #include +#define CPU_OFF_TEST_WAIT_TIME 1000 + static bool invalid_function_exception; #ifdef __arm__ @@ -71,8 +73,10 @@ static bool psci_affinity_info_off(void) } static int cpu_on_ret[NR_CPUS]; -static cpumask_t cpu_on_ready, cpu_on_done; +static bool cpu_off_success[NR_CPUS]; +static cpumask_t cpu_on_ready, cpu_on_done, cpu_off_done; static volatile int cpu_on_start; +static volatile int cpu_off_start; extern void secondary_entry(void); static void cpu_on_do_wake_target(void) @@ -94,6 +98,20 @@ static void cpu_on_target(void) cpumask_set_cpu(cpu, &cpu_on_done); } +static void cpu_off_secondary_entry(void *data) +{ + int cpu = smp_processor_id(); + + while (!cpu_off_start) + cpu_relax(); + /* On to the CPU off test */ + cpu_off_success[cpu] = true; + cpumask_set_cpu(cpu, &cpu_off_done); + cpu_psci_cpu_die(); + /* The CPU shouldn't execute the next steps. */ + cpu_off_success[cpu] = false; +} + static bool psci_cpu_on_test(void) { bool failed = false; @@ -162,9 +180,45 @@ out: return !failed; } -int main(void) +static bool psci_cpu_off_test(void) +{ + bool failed = false; + int cpu; + + for_each_present_cpu(cpu) { + if (cpu == 0) + continue; + on_cpu_async(cpu, cpu_off_secondary_entry, NULL); + } + + cpumask_set_cpu(0, &cpu_off_done); + + report_info("starting CPU_OFF test..."); + + cpu_off_start = 1; + while (!cpumask_full(&cpu_off_done)) + cpu_relax(); + + /* Allow all the other CPUs to complete the operation */ + mdelay(CPU_OFF_TEST_WAIT_TIME); + + for_each_present_cpu(cpu) { + if (cpu == 0) + continue; + + if (!cpu_off_success[cpu]) { + report_info("CPU%d could not be turned off", cpu); + failed = true; + } + } + + return !failed; +} + +int main(int argc, char **argv) { int ver = psci_invoke(PSCI_0_2_FN_PSCI_VERSION, 0, 0, 0); + bool cpu_on_success = true; report_prefix_push("psci"); @@ -179,10 +233,17 @@ int main(void) report(psci_affinity_info_on(), "affinity-info-on"); report(psci_affinity_info_off(), "affinity-info-off"); - if (ERRATA(6c7a5dce22b3)) - report(psci_cpu_on_test(), "cpu-on"); - else + if (ERRATA(6c7a5dce22b3)) { + cpu_on_success = psci_cpu_on_test(); + report(cpu_on_success, "cpu-on"); + } else { report_skip("Skipping unsafe cpu-on test. Set ERRATA_6c7a5dce22b3=y to enable."); + } + + if (!cpu_on_success) + report_skip("Skipping cpu-off test because the cpu-on test failed"); + else + report(psci_cpu_off_test(), "cpu-off"); done: #if 0