| Message ID | 20220817214146.3285106-3-coltonlewis@google.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | Randomize memory access of dirty_log_perf_test | expand |
On Wed, Aug 17, 2022 at 09:41:45PM +0000, Colton Lewis wrote: > Randomize which tables are written vs read using the random number > arrays. Change the variable wr_fract and associated function calls to > write_percent that now operates as a percentage from 0 to 100 where X > means each page has an X% chance of being written. Change the -f > argument to -w to reflect the new variable semantics. Keep the same > default of 100 percent writes. Doesn't the new option cause like a 1000x slowdown in "Dirty memory time"? I don't think we should merge this until that is understood and addressed (and it should be at least called out here so that reviewers can be made aware). > > Signed-off-by: Colton Lewis <coltonlewis@google.com> > --- > .../selftests/kvm/access_tracking_perf_test.c | 2 +- > .../selftests/kvm/dirty_log_perf_test.c | 28 ++++++++++--------- > .../selftests/kvm/include/perf_test_util.h | 4 +-- > .../selftests/kvm/lib/perf_test_util.c | 9 +++--- > 4 files changed, 23 insertions(+), 20 deletions(-) > > diff --git a/tools/testing/selftests/kvm/access_tracking_perf_test.c b/tools/testing/selftests/kvm/access_tracking_perf_test.c > index 1c2749b1481a..e87696b60e0f 100644 > --- a/tools/testing/selftests/kvm/access_tracking_perf_test.c > +++ b/tools/testing/selftests/kvm/access_tracking_perf_test.c > @@ -272,7 +272,7 @@ static void run_iteration(struct kvm_vm *vm, int nr_vcpus, const char *descripti > static void access_memory(struct kvm_vm *vm, int nr_vcpus, > enum access_type access, const char *description) > { > - perf_test_set_wr_fract(vm, (access == ACCESS_READ) ? INT_MAX : 1); > + perf_test_set_write_percent(vm, (access == ACCESS_READ) ? 0 : 100); > iteration_work = ITERATION_ACCESS_MEMORY; > run_iteration(vm, nr_vcpus, description); > } > diff --git a/tools/testing/selftests/kvm/dirty_log_perf_test.c b/tools/testing/selftests/kvm/dirty_log_perf_test.c > index 362b946019e9..9226eeea79bc 100644 > --- a/tools/testing/selftests/kvm/dirty_log_perf_test.c > +++ b/tools/testing/selftests/kvm/dirty_log_perf_test.c > @@ -128,10 +128,10 @@ static void vcpu_worker(struct perf_test_vcpu_args *vcpu_args) > struct test_params { > unsigned long iterations; > uint64_t phys_offset; > - int wr_fract; > bool partition_vcpu_memory_access; > enum vm_mem_backing_src_type backing_src; > int slots; > + uint32_t write_percent; > uint32_t random_seed; > }; > > @@ -227,7 +227,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) > p->partition_vcpu_memory_access); > > perf_test_set_random_seed(vm, p->random_seed); > - perf_test_set_wr_fract(vm, p->wr_fract); > + perf_test_set_write_percent(vm, p->write_percent); > > guest_num_pages = (nr_vcpus * guest_percpu_mem_size) >> vm->page_shift; > guest_num_pages = vm_adjust_num_guest_pages(mode, guest_num_pages); > @@ -355,7 +355,7 @@ static void help(char *name) > puts(""); > printf("usage: %s [-h] [-i iterations] [-p offset] [-g] " > "[-m mode] [-n] [-b vcpu bytes] [-v vcpus] [-o] [-r random seed ] [-s mem type]" > - "[-x memslots]\n", name); > + "[-x memslots] [-w percentage]\n", name); > puts(""); > printf(" -i: specify iteration counts (default: %"PRIu64")\n", > TEST_HOST_LOOP_N); > @@ -375,10 +375,6 @@ static void help(char *name) > printf(" -b: specify the size of the memory region which should be\n" > " dirtied by each vCPU. e.g. 10M or 3G.\n" > " (default: 1G)\n"); > - printf(" -f: specify the fraction of pages which should be written to\n" > - " as opposed to simply read, in the form\n" > - " 1/<fraction of pages to write>.\n" > - " (default: 1 i.e. all pages are written to.)\n"); > printf(" -v: specify the number of vCPUs to run.\n"); > printf(" -o: Overlap guest memory accesses instead of partitioning\n" > " them into a separate region of memory for each vCPU.\n"); > @@ -386,6 +382,11 @@ static void help(char *name) > backing_src_help("-s"); > printf(" -x: Split the memory region into this number of memslots.\n" > " (default: 1)\n"); > + printf(" -w: specify the percentage of pages which should be written to\n" > + " as an integer from 0-100 inclusive. This is probabalistic,\n" > + " so -w X means each page has an X%% chance of writing\n" > + " and a (100-X)%% chance of reading.\n" > + " (default: 100 i.e. all pages are written to.)\n"); > puts(""); > exit(0); > } > @@ -395,10 +396,10 @@ int main(int argc, char *argv[]) > int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS); > struct test_params p = { > .iterations = TEST_HOST_LOOP_N, > - .wr_fract = 1, > .partition_vcpu_memory_access = true, > .backing_src = DEFAULT_VM_MEM_SRC, > .slots = 1, > + .write_percent = 100, > .random_seed = time(NULL), > }; > int opt; > @@ -410,7 +411,7 @@ int main(int argc, char *argv[]) > > guest_modes_append_default(); > > - while ((opt = getopt(argc, argv, "eghi:p:m:nb:f:v:or:s:x:")) != -1) { > + while ((opt = getopt(argc, argv, "eghi:p:m:nb:v:or:s:x:w:")) != -1) { > switch (opt) { > case 'e': > /* 'e' is for evil. */ > @@ -433,10 +434,11 @@ int main(int argc, char *argv[]) > case 'b': > guest_percpu_mem_size = parse_size(optarg); > break; > - case 'f': > - p.wr_fract = atoi(optarg); > - TEST_ASSERT(p.wr_fract >= 1, > - "Write fraction cannot be less than one"); > + case 'w': > + perf_test_args.write_percent = atoi(optarg); > + TEST_ASSERT(perf_test_args.write_percent >= 0 > + && perf_test_args.write_percent <= 100, > + "Write percentage must be between 0 and 100"); perf_test_create_vm() overwrites this with 100. Did you mean p.write_percent? > break; > case 'v': > nr_vcpus = atoi(optarg); > diff --git a/tools/testing/selftests/kvm/include/perf_test_util.h b/tools/testing/selftests/kvm/include/perf_test_util.h > index 9517956424fc..8da4a839c585 100644 > --- a/tools/testing/selftests/kvm/include/perf_test_util.h > +++ b/tools/testing/selftests/kvm/include/perf_test_util.h > @@ -37,7 +37,7 @@ struct perf_test_args { > uint64_t size; > uint64_t guest_page_size; > uint32_t random_seed; > - int wr_fract; > + int write_percent; > > /* Run vCPUs in L2 instead of L1, if the architecture supports it. */ > bool nested; > @@ -53,7 +53,7 @@ struct kvm_vm *perf_test_create_vm(enum vm_guest_mode mode, int nr_vcpus, > bool partition_vcpu_memory_access); > void perf_test_destroy_vm(struct kvm_vm *vm); > > -void perf_test_set_wr_fract(struct kvm_vm *vm, int wr_fract); > +void perf_test_set_write_percent(struct kvm_vm *vm, uint32_t write_percent); > void perf_test_set_random_seed(struct kvm_vm *vm, uint32_t random_seed); > > void perf_test_start_vcpu_threads(int vcpus, void (*vcpu_fn)(struct perf_test_vcpu_args *)); > diff --git a/tools/testing/selftests/kvm/lib/perf_test_util.c b/tools/testing/selftests/kvm/lib/perf_test_util.c > index 8d85923acb4e..1a6b69713337 100644 > --- a/tools/testing/selftests/kvm/lib/perf_test_util.c > +++ b/tools/testing/selftests/kvm/lib/perf_test_util.c > @@ -48,6 +48,7 @@ void perf_test_guest_code(uint32_t vcpu_idx) > struct perf_test_vcpu_args *vcpu_args = &pta->vcpu_args[vcpu_idx]; > uint64_t gva; > uint64_t pages; > + uint32_t *rnd_arr = (uint32_t *)vcpu_args->random_array; > int i; > > gva = vcpu_args->gva; > @@ -60,7 +61,7 @@ void perf_test_guest_code(uint32_t vcpu_idx) > for (i = 0; i < pages; i++) { > uint64_t addr = gva + (i * pta->guest_page_size); > > - if (i % pta->wr_fract == 0) > + if (rnd_arr[i] % 100 < pta->write_percent) > *(uint64_t *)addr = 0x0123456789ABCDEF; > else > READ_ONCE(*(uint64_t *)addr); > @@ -150,7 +151,7 @@ struct kvm_vm *perf_test_create_vm(enum vm_guest_mode mode, int nr_vcpus, > pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode)); > > /* Set perf_test_args defaults. */ > - pta->wr_fract = 100; > + pta->write_percent = 100; > pta->random_seed = time(NULL); > > /* > @@ -258,9 +259,9 @@ void perf_test_destroy_vm(struct kvm_vm *vm) > kvm_vm_free(vm); > } > > -void perf_test_set_wr_fract(struct kvm_vm *vm, int wr_fract) > +void perf_test_set_write_percent(struct kvm_vm *vm, uint32_t write_percent) > { > - perf_test_args.wr_fract = wr_fract; > + perf_test_args.write_percent = write_percent; > sync_global_to_guest(vm, perf_test_args); > } > > -- > 2.37.1.595.g718a3a8f04-goog >
On Tue, Aug 30, 2022 at 07:02:10PM +0000, Colton Lewis wrote: > David Matlack <dmatlack@google.com> writes: > > > On Wed, Aug 17, 2022 at 09:41:45PM +0000, Colton Lewis wrote: > > > Randomize which tables are written vs read using the random number > > > arrays. Change the variable wr_fract and associated function calls to > > > write_percent that now operates as a percentage from 0 to 100 where X > > > means each page has an X% chance of being written. Change the -f > > > argument to -w to reflect the new variable semantics. Keep the same > > > default of 100 percent writes. > > > Doesn't the new option cause like a 1000x slowdown in "Dirty memory > > time"? I don't think we should merge this until that is understood and > > addressed (and it should be at least called out here so that reviewers > > can be made aware). > > > I'm guessing you got that from my internally posted tests. This option > itself does not cause the slowdown. If this option is set to 0% or 100% > (the default), there is no slowdown at all. The slowdown I measured was > at 50%, probably because that makes branch prediction impossible because > it has an equal chance of doing a read or a write each time. This is a > good thing. It's much more realistic than predictably alternating read > and write. I found it hard to believe that branch prediction could affect performance by 1000x (and why wouldn't random access order show the same effect?) so I looked into it further. The cause of the slowdown is actually MMU lock contention: - 82.62% [k] queued_spin_lock_slowpath - 82.09% queued_spin_lock_slowpath - 48.36% queued_write_lock_slowpath - _raw_write_lock - 22.18% kvm_mmu_notifier_invalidate_range_start __mmu_notifier_invalidate_range_start wp_page_copy do_wp_page __handle_mm_fault handle_mm_fault __get_user_pages get_user_pages_unlocked hva_to_pfn __gfn_to_pfn_memslot kvm_faultin_pfn direct_page_fault kvm_tdp_page_fault kvm_mmu_page_fault handle_ept_violation I think the bug is due to the following: 1. Randomized reads/writes were being applied to the Populate phase, which (when using anonymous memory) results in the guest memory being mapped to the Zero Page. 2. The random access order changed across each iteration (Population phase included) which means that some pages were written to during each iteration for the first time. Those pages resulted in a copy-on-write in the host MM fault handler, which invokes the invalidate range notifier and acquires the MMU lock in write-mode. 3. All vCPUs are doing this in parallel which results in a ton of lock contention. Your internal test results also showed that performance got better during each iteration. That's because more and more of the guest memory has been faulted in during each iteration (less and less copy-on-write faults that need to acquire the MMU lock in write-mode). I see this issue in your v1 patchset but not v3 (I did not check v2). But I think that's just because of the bug Ricardo pointed out (write_percent is always 100 no matter what the user passes in on the command line). The proper fix for v4 would be to set write-percent to 100 during the populate phase so all memory actually gets populated. Then only use the provided write-percent for testing dirty logging. Now you might wonder why the old version of the code didn't have this problem despite also doing reads during the populate phase. That's because the non-randomized version always read/wrote the same pages during every pass. With your code every iteratoin reads/writes to a random set of pages.
On Thu, Sep 8, 2022 at 12:46 PM Colton Lewis <coltonlewis@google.com> wrote: > > David Matlack <dmatlack@google.com> writes: > > > On Tue, Aug 30, 2022 at 07:02:10PM +0000, Colton Lewis wrote: > >> David Matlack <dmatlack@google.com> writes: > > >> > On Wed, Aug 17, 2022 at 09:41:45PM +0000, Colton Lewis wrote: > >> > > Randomize which tables are written vs read using the random number > >> > > arrays. Change the variable wr_fract and associated function calls to > >> > > write_percent that now operates as a percentage from 0 to 100 where X > >> > > means each page has an X% chance of being written. Change the -f > >> > > argument to -w to reflect the new variable semantics. Keep the same > >> > > default of 100 percent writes. > > >> > Doesn't the new option cause like a 1000x slowdown in "Dirty memory > >> > time"? I don't think we should merge this until that is understood and > >> > addressed (and it should be at least called out here so that reviewers > >> > can be made aware). > > > >> I'm guessing you got that from my internally posted tests. This option > >> itself does not cause the slowdown. If this option is set to 0% or 100% > >> (the default), there is no slowdown at all. The slowdown I measured was > >> at 50%, probably because that makes branch prediction impossible because > >> it has an equal chance of doing a read or a write each time. This is a > >> good thing. It's much more realistic than predictably alternating read > >> and write. > > > I found it hard to believe that branch prediction could affect > > performance by 1000x (and why wouldn't random access order show the same > > effect?) so I looked into it further. > > > The cause of the slowdown is actually MMU lock contention: > > > - 82.62% [k] queued_spin_lock_slowpath > > - 82.09% queued_spin_lock_slowpath > > - 48.36% queued_write_lock_slowpath > > - _raw_write_lock > > - 22.18% kvm_mmu_notifier_invalidate_range_start > > __mmu_notifier_invalidate_range_start > > wp_page_copy > > do_wp_page > > __handle_mm_fault > > handle_mm_fault > > __get_user_pages > > get_user_pages_unlocked > > hva_to_pfn > > __gfn_to_pfn_memslot > > kvm_faultin_pfn > > direct_page_fault > > kvm_tdp_page_fault > > kvm_mmu_page_fault > > handle_ept_violation > > > I think the bug is due to the following: > > > 1. Randomized reads/writes were being applied to the Populate phase, > > which (when using anonymous memory) results in the guest memory being > > mapped to the Zero Page. > > 2. The random access order changed across each iteration (Population > > phase included) which means that some pages were written to during > > each > > iteration for the first time. Those pages resulted in a copy-on-write > > in the host MM fault handler, which invokes the invalidate range > > notifier and acquires the MMU lock in write-mode. > > 3. All vCPUs are doing this in parallel which results in a ton of lock > > contention. > > > Your internal test results also showed that performance got better > > during each iteration. That's because more and more of the guest memory > > has been faulted in during each iteration (less and less copy-on-write > > faults that need to acquire the MMU lock in write-mode). > > > Thanks for the analysis David. I had wondered about the effects of > randomized reads/writes during the populate phase. > > > The proper fix for v4 would be to set write-percent to 100 during the > > populate phase so all memory actually gets populated. Then only use the > > provided write-percent for testing dirty logging. > > > Will do that. Sounds good, thanks. I do think that this is an interesting discovery though that we wouldn't have known about without your random access series. So I am somewhat tempted to say let's leave the behavior as is. But doing that will end up conflating two different tests. dirty_log_perf_test should really isolate the performance of dirty logging independent of memory population. It would be interesting though to create a generalized memory access test that randomizes reads, writes and even execution (to exercise NX Huge Pages) so that we do have a test that exercises the copy-on-write behavior. But I think that falls outside the scope of your series.
diff --git a/tools/testing/selftests/kvm/access_tracking_perf_test.c b/tools/testing/selftests/kvm/access_tracking_perf_test.c index 1c2749b1481a..e87696b60e0f 100644 --- a/tools/testing/selftests/kvm/access_tracking_perf_test.c +++ b/tools/testing/selftests/kvm/access_tracking_perf_test.c @@ -272,7 +272,7 @@ static void run_iteration(struct kvm_vm *vm, int nr_vcpus, const char *descripti static void access_memory(struct kvm_vm *vm, int nr_vcpus, enum access_type access, const char *description) { - perf_test_set_wr_fract(vm, (access == ACCESS_READ) ? INT_MAX : 1); + perf_test_set_write_percent(vm, (access == ACCESS_READ) ? 0 : 100); iteration_work = ITERATION_ACCESS_MEMORY; run_iteration(vm, nr_vcpus, description); } diff --git a/tools/testing/selftests/kvm/dirty_log_perf_test.c b/tools/testing/selftests/kvm/dirty_log_perf_test.c index 362b946019e9..9226eeea79bc 100644 --- a/tools/testing/selftests/kvm/dirty_log_perf_test.c +++ b/tools/testing/selftests/kvm/dirty_log_perf_test.c @@ -128,10 +128,10 @@ static void vcpu_worker(struct perf_test_vcpu_args *vcpu_args) struct test_params { unsigned long iterations; uint64_t phys_offset; - int wr_fract; bool partition_vcpu_memory_access; enum vm_mem_backing_src_type backing_src; int slots; + uint32_t write_percent; uint32_t random_seed; }; @@ -227,7 +227,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) p->partition_vcpu_memory_access); perf_test_set_random_seed(vm, p->random_seed); - perf_test_set_wr_fract(vm, p->wr_fract); + perf_test_set_write_percent(vm, p->write_percent); guest_num_pages = (nr_vcpus * guest_percpu_mem_size) >> vm->page_shift; guest_num_pages = vm_adjust_num_guest_pages(mode, guest_num_pages); @@ -355,7 +355,7 @@ static void help(char *name) puts(""); printf("usage: %s [-h] [-i iterations] [-p offset] [-g] " "[-m mode] [-n] [-b vcpu bytes] [-v vcpus] [-o] [-r random seed ] [-s mem type]" - "[-x memslots]\n", name); + "[-x memslots] [-w percentage]\n", name); puts(""); printf(" -i: specify iteration counts (default: %"PRIu64")\n", TEST_HOST_LOOP_N); @@ -375,10 +375,6 @@ static void help(char *name) printf(" -b: specify the size of the memory region which should be\n" " dirtied by each vCPU. e.g. 10M or 3G.\n" " (default: 1G)\n"); - printf(" -f: specify the fraction of pages which should be written to\n" - " as opposed to simply read, in the form\n" - " 1/<fraction of pages to write>.\n" - " (default: 1 i.e. all pages are written to.)\n"); printf(" -v: specify the number of vCPUs to run.\n"); printf(" -o: Overlap guest memory accesses instead of partitioning\n" " them into a separate region of memory for each vCPU.\n"); @@ -386,6 +382,11 @@ static void help(char *name) backing_src_help("-s"); printf(" -x: Split the memory region into this number of memslots.\n" " (default: 1)\n"); + printf(" -w: specify the percentage of pages which should be written to\n" + " as an integer from 0-100 inclusive. This is probabalistic,\n" + " so -w X means each page has an X%% chance of writing\n" + " and a (100-X)%% chance of reading.\n" + " (default: 100 i.e. all pages are written to.)\n"); puts(""); exit(0); } @@ -395,10 +396,10 @@ int main(int argc, char *argv[]) int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS); struct test_params p = { .iterations = TEST_HOST_LOOP_N, - .wr_fract = 1, .partition_vcpu_memory_access = true, .backing_src = DEFAULT_VM_MEM_SRC, .slots = 1, + .write_percent = 100, .random_seed = time(NULL), }; int opt; @@ -410,7 +411,7 @@ int main(int argc, char *argv[]) guest_modes_append_default(); - while ((opt = getopt(argc, argv, "eghi:p:m:nb:f:v:or:s:x:")) != -1) { + while ((opt = getopt(argc, argv, "eghi:p:m:nb:v:or:s:x:w:")) != -1) { switch (opt) { case 'e': /* 'e' is for evil. */ @@ -433,10 +434,11 @@ int main(int argc, char *argv[]) case 'b': guest_percpu_mem_size = parse_size(optarg); break; - case 'f': - p.wr_fract = atoi(optarg); - TEST_ASSERT(p.wr_fract >= 1, - "Write fraction cannot be less than one"); + case 'w': + perf_test_args.write_percent = atoi(optarg); + TEST_ASSERT(perf_test_args.write_percent >= 0 + && perf_test_args.write_percent <= 100, + "Write percentage must be between 0 and 100"); break; case 'v': nr_vcpus = atoi(optarg); diff --git a/tools/testing/selftests/kvm/include/perf_test_util.h b/tools/testing/selftests/kvm/include/perf_test_util.h index 9517956424fc..8da4a839c585 100644 --- a/tools/testing/selftests/kvm/include/perf_test_util.h +++ b/tools/testing/selftests/kvm/include/perf_test_util.h @@ -37,7 +37,7 @@ struct perf_test_args { uint64_t size; uint64_t guest_page_size; uint32_t random_seed; - int wr_fract; + int write_percent; /* Run vCPUs in L2 instead of L1, if the architecture supports it. */ bool nested; @@ -53,7 +53,7 @@ struct kvm_vm *perf_test_create_vm(enum vm_guest_mode mode, int nr_vcpus, bool partition_vcpu_memory_access); void perf_test_destroy_vm(struct kvm_vm *vm); -void perf_test_set_wr_fract(struct kvm_vm *vm, int wr_fract); +void perf_test_set_write_percent(struct kvm_vm *vm, uint32_t write_percent); void perf_test_set_random_seed(struct kvm_vm *vm, uint32_t random_seed); void perf_test_start_vcpu_threads(int vcpus, void (*vcpu_fn)(struct perf_test_vcpu_args *)); diff --git a/tools/testing/selftests/kvm/lib/perf_test_util.c b/tools/testing/selftests/kvm/lib/perf_test_util.c index 8d85923acb4e..1a6b69713337 100644 --- a/tools/testing/selftests/kvm/lib/perf_test_util.c +++ b/tools/testing/selftests/kvm/lib/perf_test_util.c @@ -48,6 +48,7 @@ void perf_test_guest_code(uint32_t vcpu_idx) struct perf_test_vcpu_args *vcpu_args = &pta->vcpu_args[vcpu_idx]; uint64_t gva; uint64_t pages; + uint32_t *rnd_arr = (uint32_t *)vcpu_args->random_array; int i; gva = vcpu_args->gva; @@ -60,7 +61,7 @@ void perf_test_guest_code(uint32_t vcpu_idx) for (i = 0; i < pages; i++) { uint64_t addr = gva + (i * pta->guest_page_size); - if (i % pta->wr_fract == 0) + if (rnd_arr[i] % 100 < pta->write_percent) *(uint64_t *)addr = 0x0123456789ABCDEF; else READ_ONCE(*(uint64_t *)addr); @@ -150,7 +151,7 @@ struct kvm_vm *perf_test_create_vm(enum vm_guest_mode mode, int nr_vcpus, pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode)); /* Set perf_test_args defaults. */ - pta->wr_fract = 100; + pta->write_percent = 100; pta->random_seed = time(NULL); /* @@ -258,9 +259,9 @@ void perf_test_destroy_vm(struct kvm_vm *vm) kvm_vm_free(vm); } -void perf_test_set_wr_fract(struct kvm_vm *vm, int wr_fract) +void perf_test_set_write_percent(struct kvm_vm *vm, uint32_t write_percent) { - perf_test_args.wr_fract = wr_fract; + perf_test_args.write_percent = write_percent; sync_global_to_guest(vm, perf_test_args); }
Randomize which tables are written vs read using the random number arrays. Change the variable wr_fract and associated function calls to write_percent that now operates as a percentage from 0 to 100 where X means each page has an X% chance of being written. Change the -f argument to -w to reflect the new variable semantics. Keep the same default of 100 percent writes. Signed-off-by: Colton Lewis <coltonlewis@google.com> --- .../selftests/kvm/access_tracking_perf_test.c | 2 +- .../selftests/kvm/dirty_log_perf_test.c | 28 ++++++++++--------- .../selftests/kvm/include/perf_test_util.h | 4 +-- .../selftests/kvm/lib/perf_test_util.c | 9 +++--- 4 files changed, 23 insertions(+), 20 deletions(-)