| Message ID | 20230215010717.3612794-12-rananta@google.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | Add support for vPMU selftests | expand |
Hi Raghu, On Tue, Feb 14, 2023 at 5:07 PM Raghavendra Rao Ananta <rananta@google.com> wrote: > > Implement a stress test for KVM by frequently force-migrating the > vCPU to random pCPUs in the system. This would validate the > save/restore functionality of KVM and starting/stopping of > PMU counters as necessary. > > Signed-off-by: Raghavendra Rao Ananta <rananta@google.com> > --- > .../testing/selftests/kvm/aarch64/vpmu_test.c | 195 +++++++++++++++++- > 1 file changed, 193 insertions(+), 2 deletions(-) > > diff --git a/tools/testing/selftests/kvm/aarch64/vpmu_test.c b/tools/testing/selftests/kvm/aarch64/vpmu_test.c > index 5c166df245589..0c9d801f4e602 100644 > --- a/tools/testing/selftests/kvm/aarch64/vpmu_test.c > +++ b/tools/testing/selftests/kvm/aarch64/vpmu_test.c > @@ -19,9 +19,15 @@ > * higher exception levels (EL2, EL3). Verify this functionality by > * configuring and trying to count the events for EL2 in the guest. > * > + * 4. Since the PMU registers are per-cpu, stress KVM by frequently > + * migrating the guest vCPU to random pCPUs in the system, and check > + * if the vPMU is still behaving as expected. > + * > * Copyright (c) 2022 Google LLC. > * > */ > +#define _GNU_SOURCE > + > #include <kvm_util.h> > #include <processor.h> > #include <test_util.h> > @@ -30,6 +36,11 @@ > #include <linux/arm-smccc.h> > #include <linux/bitfield.h> > #include <linux/bitmap.h> > +#include <stdlib.h> > +#include <pthread.h> > +#include <sys/sysinfo.h> > + > +#include "delay.h" > > /* The max number of the PMU event counters (excluding the cycle counter) */ > #define ARMV8_PMU_MAX_GENERAL_COUNTERS (ARMV8_PMU_MAX_COUNTERS - 1) > @@ -37,6 +48,8 @@ > /* The max number of event numbers that's supported */ > #define ARMV8_PMU_MAX_EVENTS 64 > > +#define msecs_to_usecs(msec) ((msec) * 1000LL) > + > /* > * The macros and functions below for reading/writing PMEV{CNTR,TYPER}<n>_EL0 > * were basically copied from arch/arm64/kernel/perf_event.c. > @@ -265,6 +278,7 @@ enum test_stage { > TEST_STAGE_COUNTER_ACCESS = 1, > TEST_STAGE_KVM_EVENT_FILTER, > TEST_STAGE_KVM_EVTYPE_FILTER, > + TEST_STAGE_VCPU_MIGRATION, > }; > > struct guest_data { > @@ -275,6 +289,19 @@ struct guest_data { > > static struct guest_data guest_data; > > +#define VCPU_MIGRATIONS_TEST_ITERS_DEF 1000 > +#define VCPU_MIGRATIONS_TEST_MIGRATION_FREQ_MS 2 > + > +struct test_args { > + int vcpu_migration_test_iter; > + int vcpu_migration_test_migrate_freq_ms; > +}; > + > +static struct test_args test_args = { > + .vcpu_migration_test_iter = VCPU_MIGRATIONS_TEST_ITERS_DEF, > + .vcpu_migration_test_migrate_freq_ms = VCPU_MIGRATIONS_TEST_MIGRATION_FREQ_MS, > +}; > + > static void guest_sync_handler(struct ex_regs *regs) > { > uint64_t esr, ec; > @@ -352,7 +379,6 @@ static bool pmu_event_is_supported(uint64_t event) > GUEST_ASSERT_3(!(_tval & mask), _tval, mask, set_expected);\ > } > > - > /* > * Extra instructions inserted by the compiler would be difficult to compensate > * for, so hand assemble everything between, and including, the PMCR accesses > @@ -459,6 +485,13 @@ static void test_event_count(uint64_t event, int pmc_idx, bool expect_count) > } > } > > +static void test_basic_pmu_functionality(void) > +{ > + /* Test events on generic and cycle counters */ > + test_instructions_count(0, true); > + test_cycles_count(true); > +} > + > /* > * Check if @mask bits in {PMCNTEN,PMINTEN,PMOVS}{SET,CLR} registers > * are set or cleared as specified in @set_expected. > @@ -748,6 +781,16 @@ static void guest_evtype_filter_test(void) > GUEST_ASSERT_2(cnt == 0, cnt, typer); > } > > +static void guest_vcpu_migration_test(void) > +{ > + /* > + * While the userspace continuously migrates this vCPU to random pCPUs, > + * run basic PMU functionalities and verify the results. > + */ > + while (test_args.vcpu_migration_test_iter--) > + test_basic_pmu_functionality(); > +} > + > static void guest_code(void) > { > switch (guest_data.test_stage) { > @@ -760,6 +803,9 @@ static void guest_code(void) > case TEST_STAGE_KVM_EVTYPE_FILTER: > guest_evtype_filter_test(); > break; > + case TEST_STAGE_VCPU_MIGRATION: > + guest_vcpu_migration_test(); > + break; > default: > GUEST_ASSERT_1(0, guest_data.test_stage); > } > @@ -837,6 +883,7 @@ create_vpmu_vm(void *guest_code, struct kvm_pmu_event_filter *pmu_event_filters) > > vpmu_vm->vm = vm = vm_create(1); > vm_init_descriptor_tables(vm); > + > /* Catch exceptions for easier debugging */ > for (ec = 0; ec < ESR_EC_NUM; ec++) { > vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, ec, > @@ -881,6 +928,8 @@ static void run_vcpu(struct kvm_vcpu *vcpu) > struct ucall uc; > > sync_global_to_guest(vcpu->vm, guest_data); > + sync_global_to_guest(vcpu->vm, test_args); > + > vcpu_run(vcpu); > switch (get_ucall(vcpu, &uc)) { > case UCALL_ABORT: > @@ -1098,11 +1147,112 @@ static void run_kvm_evtype_filter_test(void) > destroy_vpmu_vm(vpmu_vm); > } > > +struct vcpu_migrate_data { > + struct vpmu_vm *vpmu_vm; > + pthread_t *pt_vcpu; Nit: Originally, I wasn't sure what 'pt' stands for. Also, the 'pt_vcpu' made me think this would be a pointer to a vCPU. Perhaps renaming this to 'vcpu_pthread' might be more clear ? > + bool vcpu_done; > +}; > + > +static void *run_vcpus_migrate_test_func(void *arg) > +{ > + struct vcpu_migrate_data *migrate_data = arg; > + struct vpmu_vm *vpmu_vm = migrate_data->vpmu_vm; > + > + run_vcpu(vpmu_vm->vcpu); > + migrate_data->vcpu_done = true; > + > + return NULL; > +} > + > +static uint32_t get_pcpu(void) > +{ > + uint32_t pcpu; > + unsigned int nproc_conf; > + cpu_set_t online_cpuset; > + > + nproc_conf = get_nprocs_conf(); > + sched_getaffinity(0, sizeof(cpu_set_t), &online_cpuset); > + > + /* Randomly find an available pCPU to place the vCPU on */ > + do { > + pcpu = rand() % nproc_conf; > + } while (!CPU_ISSET(pcpu, &online_cpuset)); > + > + return pcpu; > +} > + > +static int migrate_vcpu(struct vcpu_migrate_data *migrate_data) Nit: You might want to pass a pthread_t rather than migrate_data unless the function uses some more fields of the data in the following patches. > +{ > + int ret; > + cpu_set_t cpuset; > + uint32_t new_pcpu = get_pcpu(); > + > + CPU_ZERO(&cpuset); > + CPU_SET(new_pcpu, &cpuset); > + > + pr_debug("Migrating vCPU to pCPU: %u\n", new_pcpu); > + > + ret = pthread_setaffinity_np(*migrate_data->pt_vcpu, sizeof(cpuset), &cpuset); > + > + /* Allow the error where the vCPU thread is already finished */ > + TEST_ASSERT(ret == 0 || ret == ESRCH, > + "Failed to migrate the vCPU to pCPU: %u; ret: %d\n", new_pcpu, ret); > + > + return ret; > +} > + > +static void *vcpus_migrate_func(void *arg) > +{ > + struct vcpu_migrate_data *migrate_data = arg; > + > + while (!migrate_data->vcpu_done) { > + usleep(msecs_to_usecs(test_args.vcpu_migration_test_migrate_freq_ms)); > + migrate_vcpu(migrate_data); > + } > + > + return NULL; > +} > + > +static void run_vcpu_migration_test(uint64_t pmcr_n) > +{ > + int ret; > + struct vpmu_vm *vpmu_vm; > + pthread_t pt_vcpu, pt_sched; > + struct vcpu_migrate_data migrate_data = { > + .pt_vcpu = &pt_vcpu, > + .vcpu_done = false, > + }; > + > + __TEST_REQUIRE(get_nprocs() >= 2, "At least two pCPUs needed for vCPU migration test"); Considering that get_pcpu() chooses the target CPU from CPUs returned from sched_getaffinity(), I would think the test should use the number of the bits set in the returned cpu_set_t from sched_getaffinity() here instead of get_nprocs(), as those numbers could be different (e.g. if the test runs with taskset with a subset of the CPUs on the system). > + > + guest_data.test_stage = TEST_STAGE_VCPU_MIGRATION; > + guest_data.expected_pmcr_n = pmcr_n; > + > + migrate_data.vpmu_vm = vpmu_vm = create_vpmu_vm(guest_code, NULL); > + > + /* Initialize random number generation for migrating vCPUs to random pCPUs */ > + srand(time(NULL)); > + > + /* Spawn a vCPU thread */ > + ret = pthread_create(&pt_vcpu, NULL, run_vcpus_migrate_test_func, &migrate_data); > + TEST_ASSERT(!ret, "Failed to create the vCPU thread"); > + > + /* Spawn a scheduler thread to force-migrate vCPUs to various pCPUs */ > + ret = pthread_create(&pt_sched, NULL, vcpus_migrate_func, &migrate_data); Why do you want to spawn another thread to run vcpus_migrate_func(), rather than calling that from the current thread ? > + TEST_ASSERT(!ret, "Failed to create the scheduler thread for migrating the vCPUs"); > + > + pthread_join(pt_sched, NULL); > + pthread_join(pt_vcpu, NULL); > + > + destroy_vpmu_vm(vpmu_vm); > +} > + > static void run_tests(uint64_t pmcr_n) > { > run_counter_access_tests(pmcr_n); > run_kvm_event_filter_test(); > run_kvm_evtype_filter_test(); > + run_vcpu_migration_test(pmcr_n); > } > > /* > @@ -1121,12 +1271,53 @@ static uint64_t get_pmcr_n_limit(void) > return FIELD_GET(ARMV8_PMU_PMCR_N, pmcr); > } > > -int main(void) > +static void print_help(char *name) > +{ > + pr_info("Usage: %s [-h] [-i vcpu_migration_test_iterations] [-m vcpu_migration_freq_ms]\n", > + name); > + pr_info("\t-i: Number of iterations of vCPU migrations test (default: %u)\n", > + VCPU_MIGRATIONS_TEST_ITERS_DEF); > + pr_info("\t-m: Frequency (in ms) of vCPUs to migrate to different pCPU. (default: %u)\n", > + VCPU_MIGRATIONS_TEST_MIGRATION_FREQ_MS); > + pr_info("\t-h: print this help screen\n"); > +} > + > +static bool parse_args(int argc, char *argv[]) > +{ > + int opt; > + > + while ((opt = getopt(argc, argv, "hi:m:")) != -1) { > + switch (opt) { > + case 'i': > + test_args.vcpu_migration_test_iter = > + atoi_positive("Nr vCPU migration iterations", optarg); > + break; > + case 'm': > + test_args.vcpu_migration_test_migrate_freq_ms = > + atoi_positive("vCPU migration frequency", optarg); > + break; > + case 'h': > + default: > + goto err; > + } > + } > + > + return true; > + > +err: > + print_help(argv[0]); > + return false; > +} > + > +int main(int argc, char *argv[]) > { > uint64_t pmcr_n; > > TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_PMU_V3)); > > + if (!parse_args(argc, argv)) > + exit(KSFT_SKIP); > + > pmcr_n = get_pmcr_n_limit(); > run_tests(pmcr_n); > > -- > 2.39.1.581.gbfd45094c4-goog > Thanks, Reiji
On Mon, Mar 6, 2023 at 7:44 PM Reiji Watanabe <reijiw@google.com> wrote: > > Hi Raghu, > > On Tue, Feb 14, 2023 at 5:07 PM Raghavendra Rao Ananta > <rananta@google.com> wrote: > > > > Implement a stress test for KVM by frequently force-migrating the > > vCPU to random pCPUs in the system. This would validate the > > save/restore functionality of KVM and starting/stopping of > > PMU counters as necessary. > > > > Signed-off-by: Raghavendra Rao Ananta <rananta@google.com> > > --- > > .../testing/selftests/kvm/aarch64/vpmu_test.c | 195 +++++++++++++++++- > > 1 file changed, 193 insertions(+), 2 deletions(-) > > > > diff --git a/tools/testing/selftests/kvm/aarch64/vpmu_test.c b/tools/testing/selftests/kvm/aarch64/vpmu_test.c > > index 5c166df245589..0c9d801f4e602 100644 > > --- a/tools/testing/selftests/kvm/aarch64/vpmu_test.c > > +++ b/tools/testing/selftests/kvm/aarch64/vpmu_test.c > > @@ -19,9 +19,15 @@ > > * higher exception levels (EL2, EL3). Verify this functionality by > > * configuring and trying to count the events for EL2 in the guest. > > * > > + * 4. Since the PMU registers are per-cpu, stress KVM by frequently > > + * migrating the guest vCPU to random pCPUs in the system, and check > > + * if the vPMU is still behaving as expected. > > + * > > * Copyright (c) 2022 Google LLC. > > * > > */ > > +#define _GNU_SOURCE > > + > > #include <kvm_util.h> > > #include <processor.h> > > #include <test_util.h> > > @@ -30,6 +36,11 @@ > > #include <linux/arm-smccc.h> > > #include <linux/bitfield.h> > > #include <linux/bitmap.h> > > +#include <stdlib.h> > > +#include <pthread.h> > > +#include <sys/sysinfo.h> > > + > > +#include "delay.h" > > > > /* The max number of the PMU event counters (excluding the cycle counter) */ > > #define ARMV8_PMU_MAX_GENERAL_COUNTERS (ARMV8_PMU_MAX_COUNTERS - 1) > > @@ -37,6 +48,8 @@ > > /* The max number of event numbers that's supported */ > > #define ARMV8_PMU_MAX_EVENTS 64 > > > > +#define msecs_to_usecs(msec) ((msec) * 1000LL) > > + > > /* > > * The macros and functions below for reading/writing PMEV{CNTR,TYPER}<n>_EL0 > > * were basically copied from arch/arm64/kernel/perf_event.c. > > @@ -265,6 +278,7 @@ enum test_stage { > > TEST_STAGE_COUNTER_ACCESS = 1, > > TEST_STAGE_KVM_EVENT_FILTER, > > TEST_STAGE_KVM_EVTYPE_FILTER, > > + TEST_STAGE_VCPU_MIGRATION, > > }; > > > > struct guest_data { > > @@ -275,6 +289,19 @@ struct guest_data { > > > > static struct guest_data guest_data; > > > > +#define VCPU_MIGRATIONS_TEST_ITERS_DEF 1000 > > +#define VCPU_MIGRATIONS_TEST_MIGRATION_FREQ_MS 2 > > + > > +struct test_args { > > + int vcpu_migration_test_iter; > > + int vcpu_migration_test_migrate_freq_ms; > > +}; > > + > > +static struct test_args test_args = { > > + .vcpu_migration_test_iter = VCPU_MIGRATIONS_TEST_ITERS_DEF, > > + .vcpu_migration_test_migrate_freq_ms = VCPU_MIGRATIONS_TEST_MIGRATION_FREQ_MS, > > +}; > > + > > static void guest_sync_handler(struct ex_regs *regs) > > { > > uint64_t esr, ec; > > @@ -352,7 +379,6 @@ static bool pmu_event_is_supported(uint64_t event) > > GUEST_ASSERT_3(!(_tval & mask), _tval, mask, set_expected);\ > > } > > > > - > > /* > > * Extra instructions inserted by the compiler would be difficult to compensate > > * for, so hand assemble everything between, and including, the PMCR accesses > > @@ -459,6 +485,13 @@ static void test_event_count(uint64_t event, int pmc_idx, bool expect_count) > > } > > } > > > > +static void test_basic_pmu_functionality(void) > > +{ > > + /* Test events on generic and cycle counters */ > > + test_instructions_count(0, true); > > + test_cycles_count(true); > > +} > > + > > /* > > * Check if @mask bits in {PMCNTEN,PMINTEN,PMOVS}{SET,CLR} registers > > * are set or cleared as specified in @set_expected. > > @@ -748,6 +781,16 @@ static void guest_evtype_filter_test(void) > > GUEST_ASSERT_2(cnt == 0, cnt, typer); > > } > > > > +static void guest_vcpu_migration_test(void) > > +{ > > + /* > > + * While the userspace continuously migrates this vCPU to random pCPUs, > > + * run basic PMU functionalities and verify the results. > > + */ > > + while (test_args.vcpu_migration_test_iter--) > > + test_basic_pmu_functionality(); > > +} > > + > > static void guest_code(void) > > { > > switch (guest_data.test_stage) { > > @@ -760,6 +803,9 @@ static void guest_code(void) > > case TEST_STAGE_KVM_EVTYPE_FILTER: > > guest_evtype_filter_test(); > > break; > > + case TEST_STAGE_VCPU_MIGRATION: > > + guest_vcpu_migration_test(); > > + break; > > default: > > GUEST_ASSERT_1(0, guest_data.test_stage); > > } > > @@ -837,6 +883,7 @@ create_vpmu_vm(void *guest_code, struct kvm_pmu_event_filter *pmu_event_filters) > > > > vpmu_vm->vm = vm = vm_create(1); > > vm_init_descriptor_tables(vm); > > + > > /* Catch exceptions for easier debugging */ > > for (ec = 0; ec < ESR_EC_NUM; ec++) { > > vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, ec, > > @@ -881,6 +928,8 @@ static void run_vcpu(struct kvm_vcpu *vcpu) > > struct ucall uc; > > > > sync_global_to_guest(vcpu->vm, guest_data); > > + sync_global_to_guest(vcpu->vm, test_args); > > + > > vcpu_run(vcpu); > > switch (get_ucall(vcpu, &uc)) { > > case UCALL_ABORT: > > @@ -1098,11 +1147,112 @@ static void run_kvm_evtype_filter_test(void) > > destroy_vpmu_vm(vpmu_vm); > > } > > > > +struct vcpu_migrate_data { > > + struct vpmu_vm *vpmu_vm; > > + pthread_t *pt_vcpu; > > Nit: Originally, I wasn't sure what 'pt' stands for. > Also, the 'pt_vcpu' made me think this would be a pointer to a vCPU. > Perhaps renaming this to 'vcpu_pthread' might be more clear ? > Haha, no problem. I'll change it to vcpu_pthread. > > > + bool vcpu_done; > > +}; > > + > > +static void *run_vcpus_migrate_test_func(void *arg) > > +{ > > + struct vcpu_migrate_data *migrate_data = arg; > > + struct vpmu_vm *vpmu_vm = migrate_data->vpmu_vm; > > + > > + run_vcpu(vpmu_vm->vcpu); > > + migrate_data->vcpu_done = true; > > + > > + return NULL; > > +} > > + > > +static uint32_t get_pcpu(void) > > +{ > > + uint32_t pcpu; > > + unsigned int nproc_conf; > > + cpu_set_t online_cpuset; > > + > > + nproc_conf = get_nprocs_conf(); > > + sched_getaffinity(0, sizeof(cpu_set_t), &online_cpuset); > > + > > + /* Randomly find an available pCPU to place the vCPU on */ > > + do { > > + pcpu = rand() % nproc_conf; > > + } while (!CPU_ISSET(pcpu, &online_cpuset)); > > + > > + return pcpu; > > +} > > + > > +static int migrate_vcpu(struct vcpu_migrate_data *migrate_data) > > Nit: You might want to pass a pthread_t rather than migrate_data > unless the function uses some more fields of the data in the > following patches. > The upcoming patch, which introduces multiple-vcpus, moves migrate_date into a global array (one element per-vCPU). That patch passes only the vcpu index as an arg to migrate_vcpu(). I originally thought we would embed more stuff into migrate_data, and passed this. But I guess I can just pass pthread_t. > > +{ > > + int ret; > > + cpu_set_t cpuset; > > + uint32_t new_pcpu = get_pcpu(); > > + > > + CPU_ZERO(&cpuset); > > + CPU_SET(new_pcpu, &cpuset); > > + > > + pr_debug("Migrating vCPU to pCPU: %u\n", new_pcpu); > > + > > + ret = pthread_setaffinity_np(*migrate_data->pt_vcpu, sizeof(cpuset), &cpuset); > > + > > + /* Allow the error where the vCPU thread is already finished */ > > + TEST_ASSERT(ret == 0 || ret == ESRCH, > > + "Failed to migrate the vCPU to pCPU: %u; ret: %d\n", new_pcpu, ret); > > + > > + return ret; > > +} > > + > > +static void *vcpus_migrate_func(void *arg) > > +{ > > + struct vcpu_migrate_data *migrate_data = arg; > > + > > + while (!migrate_data->vcpu_done) { > > + usleep(msecs_to_usecs(test_args.vcpu_migration_test_migrate_freq_ms)); > > + migrate_vcpu(migrate_data); > > + } > > + > > + return NULL; > > +} > > + > > +static void run_vcpu_migration_test(uint64_t pmcr_n) > > +{ > > + int ret; > > + struct vpmu_vm *vpmu_vm; > > + pthread_t pt_vcpu, pt_sched; > > + struct vcpu_migrate_data migrate_data = { > > + .pt_vcpu = &pt_vcpu, > > + .vcpu_done = false, > > + }; > > + > > + __TEST_REQUIRE(get_nprocs() >= 2, "At least two pCPUs needed for vCPU migration test"); > > Considering that get_pcpu() chooses the target CPU from CPUs returned > from sched_getaffinity(), I would think the test should use the number of > the bits set in the returned cpu_set_t from sched_getaffinity() here > instead of get_nprocs(), as those numbers could be different (e.g. if the > test runs with taskset with a subset of the CPUs on the system). > I'm not familiar with tasksets, but if you feel the current approach could cause problems, I'll switch to your suggestion. Thanks. > > > + > > + guest_data.test_stage = TEST_STAGE_VCPU_MIGRATION; > > + guest_data.expected_pmcr_n = pmcr_n; > > + > > + migrate_data.vpmu_vm = vpmu_vm = create_vpmu_vm(guest_code, NULL); > > + > > + /* Initialize random number generation for migrating vCPUs to random pCPUs */ > > + srand(time(NULL)); > > + > > + /* Spawn a vCPU thread */ > > + ret = pthread_create(&pt_vcpu, NULL, run_vcpus_migrate_test_func, &migrate_data); > > + TEST_ASSERT(!ret, "Failed to create the vCPU thread"); > > + > > + /* Spawn a scheduler thread to force-migrate vCPUs to various pCPUs */ > > + ret = pthread_create(&pt_sched, NULL, vcpus_migrate_func, &migrate_data); > > Why do you want to spawn another thread to run vcpus_migrate_func(), > rather than calling that from the current thread ? > > I suppose it should be fine calling from the current thread (unless I'm forgetting a reason why I had a similar behavior in arch_timer test). Thank you. Raghavendra > > + TEST_ASSERT(!ret, "Failed to create the scheduler thread for migrating the vCPUs"); > > + > > + pthread_join(pt_sched, NULL); > > + pthread_join(pt_vcpu, NULL); > > + > > + destroy_vpmu_vm(vpmu_vm); > > +} > > + > > static void run_tests(uint64_t pmcr_n) > > { > > run_counter_access_tests(pmcr_n); > > run_kvm_event_filter_test(); > > run_kvm_evtype_filter_test(); > > + run_vcpu_migration_test(pmcr_n); > > } > > > > /* > > @@ -1121,12 +1271,53 @@ static uint64_t get_pmcr_n_limit(void) > > return FIELD_GET(ARMV8_PMU_PMCR_N, pmcr); > > } > > > > -int main(void) > > +static void print_help(char *name) > > +{ > > + pr_info("Usage: %s [-h] [-i vcpu_migration_test_iterations] [-m vcpu_migration_freq_ms]\n", > > + name); > > + pr_info("\t-i: Number of iterations of vCPU migrations test (default: %u)\n", > > + VCPU_MIGRATIONS_TEST_ITERS_DEF); > > + pr_info("\t-m: Frequency (in ms) of vCPUs to migrate to different pCPU. (default: %u)\n", > > + VCPU_MIGRATIONS_TEST_MIGRATION_FREQ_MS); > > + pr_info("\t-h: print this help screen\n"); > > +} > > + > > +static bool parse_args(int argc, char *argv[]) > > +{ > > + int opt; > > + > > + while ((opt = getopt(argc, argv, "hi:m:")) != -1) { > > + switch (opt) { > > + case 'i': > > + test_args.vcpu_migration_test_iter = > > + atoi_positive("Nr vCPU migration iterations", optarg); > > + break; > > + case 'm': > > + test_args.vcpu_migration_test_migrate_freq_ms = > > + atoi_positive("vCPU migration frequency", optarg); > > + break; > > + case 'h': > > + default: > > + goto err; > > + } > > + } > > + > > + return true; > > + > > +err: > > + print_help(argv[0]); > > + return false; > > +} > > + > > +int main(int argc, char *argv[]) > > { > > uint64_t pmcr_n; > > > > TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_PMU_V3)); > > > > + if (!parse_args(argc, argv)) > > + exit(KSFT_SKIP); > > + > > pmcr_n = get_pmcr_n_limit(); > > run_tests(pmcr_n); > > > > -- > > 2.39.1.581.gbfd45094c4-goog > > > > Thanks, > Reiji
diff --git a/tools/testing/selftests/kvm/aarch64/vpmu_test.c b/tools/testing/selftests/kvm/aarch64/vpmu_test.c index 5c166df245589..0c9d801f4e602 100644 --- a/tools/testing/selftests/kvm/aarch64/vpmu_test.c +++ b/tools/testing/selftests/kvm/aarch64/vpmu_test.c @@ -19,9 +19,15 @@ * higher exception levels (EL2, EL3). Verify this functionality by * configuring and trying to count the events for EL2 in the guest. * + * 4. Since the PMU registers are per-cpu, stress KVM by frequently + * migrating the guest vCPU to random pCPUs in the system, and check + * if the vPMU is still behaving as expected. + * * Copyright (c) 2022 Google LLC. * */ +#define _GNU_SOURCE + #include <kvm_util.h> #include <processor.h> #include <test_util.h> @@ -30,6 +36,11 @@ #include <linux/arm-smccc.h> #include <linux/bitfield.h> #include <linux/bitmap.h> +#include <stdlib.h> +#include <pthread.h> +#include <sys/sysinfo.h> + +#include "delay.h" /* The max number of the PMU event counters (excluding the cycle counter) */ #define ARMV8_PMU_MAX_GENERAL_COUNTERS (ARMV8_PMU_MAX_COUNTERS - 1) @@ -37,6 +48,8 @@ /* The max number of event numbers that's supported */ #define ARMV8_PMU_MAX_EVENTS 64 +#define msecs_to_usecs(msec) ((msec) * 1000LL) + /* * The macros and functions below for reading/writing PMEV{CNTR,TYPER}<n>_EL0 * were basically copied from arch/arm64/kernel/perf_event.c. @@ -265,6 +278,7 @@ enum test_stage { TEST_STAGE_COUNTER_ACCESS = 1, TEST_STAGE_KVM_EVENT_FILTER, TEST_STAGE_KVM_EVTYPE_FILTER, + TEST_STAGE_VCPU_MIGRATION, }; struct guest_data { @@ -275,6 +289,19 @@ struct guest_data { static struct guest_data guest_data; +#define VCPU_MIGRATIONS_TEST_ITERS_DEF 1000 +#define VCPU_MIGRATIONS_TEST_MIGRATION_FREQ_MS 2 + +struct test_args { + int vcpu_migration_test_iter; + int vcpu_migration_test_migrate_freq_ms; +}; + +static struct test_args test_args = { + .vcpu_migration_test_iter = VCPU_MIGRATIONS_TEST_ITERS_DEF, + .vcpu_migration_test_migrate_freq_ms = VCPU_MIGRATIONS_TEST_MIGRATION_FREQ_MS, +}; + static void guest_sync_handler(struct ex_regs *regs) { uint64_t esr, ec; @@ -352,7 +379,6 @@ static bool pmu_event_is_supported(uint64_t event) GUEST_ASSERT_3(!(_tval & mask), _tval, mask, set_expected);\ } - /* * Extra instructions inserted by the compiler would be difficult to compensate * for, so hand assemble everything between, and including, the PMCR accesses @@ -459,6 +485,13 @@ static void test_event_count(uint64_t event, int pmc_idx, bool expect_count) } } +static void test_basic_pmu_functionality(void) +{ + /* Test events on generic and cycle counters */ + test_instructions_count(0, true); + test_cycles_count(true); +} + /* * Check if @mask bits in {PMCNTEN,PMINTEN,PMOVS}{SET,CLR} registers * are set or cleared as specified in @set_expected. @@ -748,6 +781,16 @@ static void guest_evtype_filter_test(void) GUEST_ASSERT_2(cnt == 0, cnt, typer); } +static void guest_vcpu_migration_test(void) +{ + /* + * While the userspace continuously migrates this vCPU to random pCPUs, + * run basic PMU functionalities and verify the results. + */ + while (test_args.vcpu_migration_test_iter--) + test_basic_pmu_functionality(); +} + static void guest_code(void) { switch (guest_data.test_stage) { @@ -760,6 +803,9 @@ static void guest_code(void) case TEST_STAGE_KVM_EVTYPE_FILTER: guest_evtype_filter_test(); break; + case TEST_STAGE_VCPU_MIGRATION: + guest_vcpu_migration_test(); + break; default: GUEST_ASSERT_1(0, guest_data.test_stage); } @@ -837,6 +883,7 @@ create_vpmu_vm(void *guest_code, struct kvm_pmu_event_filter *pmu_event_filters) vpmu_vm->vm = vm = vm_create(1); vm_init_descriptor_tables(vm); + /* Catch exceptions for easier debugging */ for (ec = 0; ec < ESR_EC_NUM; ec++) { vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, ec, @@ -881,6 +928,8 @@ static void run_vcpu(struct kvm_vcpu *vcpu) struct ucall uc; sync_global_to_guest(vcpu->vm, guest_data); + sync_global_to_guest(vcpu->vm, test_args); + vcpu_run(vcpu); switch (get_ucall(vcpu, &uc)) { case UCALL_ABORT: @@ -1098,11 +1147,112 @@ static void run_kvm_evtype_filter_test(void) destroy_vpmu_vm(vpmu_vm); } +struct vcpu_migrate_data { + struct vpmu_vm *vpmu_vm; + pthread_t *pt_vcpu; + bool vcpu_done; +}; + +static void *run_vcpus_migrate_test_func(void *arg) +{ + struct vcpu_migrate_data *migrate_data = arg; + struct vpmu_vm *vpmu_vm = migrate_data->vpmu_vm; + + run_vcpu(vpmu_vm->vcpu); + migrate_data->vcpu_done = true; + + return NULL; +} + +static uint32_t get_pcpu(void) +{ + uint32_t pcpu; + unsigned int nproc_conf; + cpu_set_t online_cpuset; + + nproc_conf = get_nprocs_conf(); + sched_getaffinity(0, sizeof(cpu_set_t), &online_cpuset); + + /* Randomly find an available pCPU to place the vCPU on */ + do { + pcpu = rand() % nproc_conf; + } while (!CPU_ISSET(pcpu, &online_cpuset)); + + return pcpu; +} + +static int migrate_vcpu(struct vcpu_migrate_data *migrate_data) +{ + int ret; + cpu_set_t cpuset; + uint32_t new_pcpu = get_pcpu(); + + CPU_ZERO(&cpuset); + CPU_SET(new_pcpu, &cpuset); + + pr_debug("Migrating vCPU to pCPU: %u\n", new_pcpu); + + ret = pthread_setaffinity_np(*migrate_data->pt_vcpu, sizeof(cpuset), &cpuset); + + /* Allow the error where the vCPU thread is already finished */ + TEST_ASSERT(ret == 0 || ret == ESRCH, + "Failed to migrate the vCPU to pCPU: %u; ret: %d\n", new_pcpu, ret); + + return ret; +} + +static void *vcpus_migrate_func(void *arg) +{ + struct vcpu_migrate_data *migrate_data = arg; + + while (!migrate_data->vcpu_done) { + usleep(msecs_to_usecs(test_args.vcpu_migration_test_migrate_freq_ms)); + migrate_vcpu(migrate_data); + } + + return NULL; +} + +static void run_vcpu_migration_test(uint64_t pmcr_n) +{ + int ret; + struct vpmu_vm *vpmu_vm; + pthread_t pt_vcpu, pt_sched; + struct vcpu_migrate_data migrate_data = { + .pt_vcpu = &pt_vcpu, + .vcpu_done = false, + }; + + __TEST_REQUIRE(get_nprocs() >= 2, "At least two pCPUs needed for vCPU migration test"); + + guest_data.test_stage = TEST_STAGE_VCPU_MIGRATION; + guest_data.expected_pmcr_n = pmcr_n; + + migrate_data.vpmu_vm = vpmu_vm = create_vpmu_vm(guest_code, NULL); + + /* Initialize random number generation for migrating vCPUs to random pCPUs */ + srand(time(NULL)); + + /* Spawn a vCPU thread */ + ret = pthread_create(&pt_vcpu, NULL, run_vcpus_migrate_test_func, &migrate_data); + TEST_ASSERT(!ret, "Failed to create the vCPU thread"); + + /* Spawn a scheduler thread to force-migrate vCPUs to various pCPUs */ + ret = pthread_create(&pt_sched, NULL, vcpus_migrate_func, &migrate_data); + TEST_ASSERT(!ret, "Failed to create the scheduler thread for migrating the vCPUs"); + + pthread_join(pt_sched, NULL); + pthread_join(pt_vcpu, NULL); + + destroy_vpmu_vm(vpmu_vm); +} + static void run_tests(uint64_t pmcr_n) { run_counter_access_tests(pmcr_n); run_kvm_event_filter_test(); run_kvm_evtype_filter_test(); + run_vcpu_migration_test(pmcr_n); } /* @@ -1121,12 +1271,53 @@ static uint64_t get_pmcr_n_limit(void) return FIELD_GET(ARMV8_PMU_PMCR_N, pmcr); } -int main(void) +static void print_help(char *name) +{ + pr_info("Usage: %s [-h] [-i vcpu_migration_test_iterations] [-m vcpu_migration_freq_ms]\n", + name); + pr_info("\t-i: Number of iterations of vCPU migrations test (default: %u)\n", + VCPU_MIGRATIONS_TEST_ITERS_DEF); + pr_info("\t-m: Frequency (in ms) of vCPUs to migrate to different pCPU. (default: %u)\n", + VCPU_MIGRATIONS_TEST_MIGRATION_FREQ_MS); + pr_info("\t-h: print this help screen\n"); +} + +static bool parse_args(int argc, char *argv[]) +{ + int opt; + + while ((opt = getopt(argc, argv, "hi:m:")) != -1) { + switch (opt) { + case 'i': + test_args.vcpu_migration_test_iter = + atoi_positive("Nr vCPU migration iterations", optarg); + break; + case 'm': + test_args.vcpu_migration_test_migrate_freq_ms = + atoi_positive("vCPU migration frequency", optarg); + break; + case 'h': + default: + goto err; + } + } + + return true; + +err: + print_help(argv[0]); + return false; +} + +int main(int argc, char *argv[]) { uint64_t pmcr_n; TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_PMU_V3)); + if (!parse_args(argc, argv)) + exit(KSFT_SKIP); + pmcr_n = get_pmcr_n_limit(); run_tests(pmcr_n);
Implement a stress test for KVM by frequently force-migrating the vCPU to random pCPUs in the system. This would validate the save/restore functionality of KVM and starting/stopping of PMU counters as necessary. Signed-off-by: Raghavendra Rao Ananta <rananta@google.com> --- .../testing/selftests/kvm/aarch64/vpmu_test.c | 195 +++++++++++++++++- 1 file changed, 193 insertions(+), 2 deletions(-)