From patchwork Thu Nov 15 00:08:40 2012 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Marcelo Tosatti X-Patchwork-Id: 1744821 Return-Path: X-Original-To: patchwork-kvm@patchwork.kernel.org Delivered-To: patchwork-process-083081@patchwork1.kernel.org Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by patchwork1.kernel.org (Postfix) with ESMTP id E299A40E07 for ; Thu, 15 Nov 2012 01:28:01 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S2992537Ab2KOB16 (ORCPT ); Wed, 14 Nov 2012 20:27:58 -0500 Received: from mx1.redhat.com ([209.132.183.28]:42236 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S2992503Ab2KOB14 (ORCPT ); Wed, 14 Nov 2012 20:27:56 -0500 Received: from int-mx02.intmail.prod.int.phx2.redhat.com (int-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.12]) by mx1.redhat.com (8.14.4/8.14.4) with ESMTP id qAF1ROVk008687 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=OK); Wed, 14 Nov 2012 20:27:51 -0500 Received: from amt.cnet (vpn1-5-41.gru2.redhat.com [10.97.5.41]) by int-mx02.intmail.prod.int.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id qAF08q5A024607; Wed, 14 Nov 2012 19:08:52 -0500 Received: from amt.cnet (amt.cnet [127.0.0.1]) by amt.cnet (Postfix) with ESMTP id 047C066E189; Wed, 14 Nov 2012 22:07:38 -0200 (BRST) Received: (from marcelo@localhost) by amt.cnet (8.14.5/8.14.5/Submit) id qAF07bdc001721; Wed, 14 Nov 2012 22:07:37 -0200 Message-Id: <20121115000944.724933468@redhat.com> User-Agent: quilt/0.47-1 Date: Wed, 14 Nov 2012 22:08:40 -0200 From: Marcelo Tosatti To: kvm@vger.kernel.org Cc: johnstul@us.ibm.com, jeremy@goop.org, glommer@parallels.com, zamsden@gmail.com, gleb@redhat.com, avi@redhat.com, pbonzini@redhat.com, Marcelo Tosatti Subject: [patch 17/18] KVM: x86: require matched TSC offsets for master clock References: <20121115000823.285102321@redhat.com> Content-Disposition: inline; filename=17-masterclock-require-matched-tsc X-Scanned-By: MIMEDefang 2.67 on 10.5.11.12 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org With master clock, a pvclock clock read calculates: ret = system_timestamp + [ (rdtsc + tsc_offset) - tsc_timestamp ] Where 'rdtsc' is the host TSC. system_timestamp and tsc_timestamp are unique, one tuple per VM: the "master clock". Given a host with synchronized TSCs, its obvious that guest TSC must be matched for the above to guarantee monotonicity. Allow master clock usage only if guest TSCs are synchronized. Signed-off-by: Marcelo Tosatti --- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Index: vsyscall/arch/x86/include/asm/kvm_host.h =================================================================== --- vsyscall.orig/arch/x86/include/asm/kvm_host.h +++ vsyscall/arch/x86/include/asm/kvm_host.h @@ -560,6 +560,7 @@ struct kvm_arch { u64 cur_tsc_write; u64 cur_tsc_offset; u8 cur_tsc_generation; + int nr_vcpus_matched_tsc; spinlock_t pvclock_gtod_sync_lock; bool use_master_clock; Index: vsyscall/arch/x86/kvm/x86.c =================================================================== --- vsyscall.orig/arch/x86/kvm/x86.c +++ vsyscall/arch/x86/kvm/x86.c @@ -1097,12 +1097,38 @@ static u64 compute_guest_tsc(struct kvm_ return tsc; } +void kvm_track_tsc_matching(struct kvm_vcpu *vcpu) +{ + bool vcpus_matched; + bool do_request = false; + struct kvm_arch *ka = &vcpu->kvm->arch; + struct pvclock_gtod_data *gtod = &pvclock_gtod_data; + + vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 == + atomic_read(&vcpu->kvm->online_vcpus)); + + if (vcpus_matched && gtod->clock.vclock_mode == VCLOCK_TSC) + if (!ka->use_master_clock) + do_request = 1; + + if (!vcpus_matched && ka->use_master_clock) + do_request = 1; + + if (do_request) + kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); + + trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc, + atomic_read(&vcpu->kvm->online_vcpus), + ka->use_master_clock, gtod->clock.vclock_mode); +} + void kvm_write_tsc(struct kvm_vcpu *vcpu, u64 data) { struct kvm *kvm = vcpu->kvm; u64 offset, ns, elapsed; unsigned long flags; s64 usdiff; + bool matched; raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); offset = kvm_x86_ops->compute_tsc_offset(vcpu, data); @@ -1145,6 +1171,7 @@ void kvm_write_tsc(struct kvm_vcpu *vcpu offset = kvm_x86_ops->compute_tsc_offset(vcpu, data); pr_debug("kvm: adjusted tsc offset by %llu\n", delta); } + matched = true; } else { /* * We split periods of matched TSC writes into generations. @@ -1159,6 +1186,7 @@ void kvm_write_tsc(struct kvm_vcpu *vcpu kvm->arch.cur_tsc_nsec = ns; kvm->arch.cur_tsc_write = data; kvm->arch.cur_tsc_offset = offset; + matched = false; pr_debug("kvm: new tsc generation %u, clock %llu\n", kvm->arch.cur_tsc_generation, data); } @@ -1182,6 +1210,15 @@ void kvm_write_tsc(struct kvm_vcpu *vcpu kvm_x86_ops->write_tsc_offset(vcpu, offset); raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); + + spin_lock(&kvm->arch.pvclock_gtod_sync_lock); + if (matched) + kvm->arch.nr_vcpus_matched_tsc++; + else + kvm->arch.nr_vcpus_matched_tsc = 0; + + kvm_track_tsc_matching(vcpu); + spin_unlock(&kvm->arch.pvclock_gtod_sync_lock); } EXPORT_SYMBOL_GPL(kvm_write_tsc); @@ -1271,8 +1308,9 @@ static bool kvm_get_time_and_clockread(s /* * - * Assuming a stable TSC across physical CPUS, the following condition - * is possible. Each numbered line represents an event visible to both + * Assuming a stable TSC across physical CPUS, and a stable TSC + * across virtual CPUs, the following condition is possible. + * Each numbered line represents an event visible to both * CPUs at the next numbered event. * * "timespecX" represents host monotonic time. "tscX" represents @@ -1305,7 +1343,7 @@ static bool kvm_get_time_and_clockread(s * copy of host monotonic time values. Update that master copy * in lockstep. * - * Rely on synchronization of host TSCs for monotonicity. + * Rely on synchronization of host TSCs and guest TSCs for monotonicity. * */ @@ -1313,20 +1351,27 @@ static void pvclock_update_vm_gtod_copy( { struct kvm_arch *ka = &kvm->arch; int vclock_mode; + bool host_tsc_clocksource, vcpus_matched; + + vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 == + atomic_read(&kvm->online_vcpus)); /* * If the host uses TSC clock, then passthrough TSC as stable * to the guest. */ - ka->use_master_clock = kvm_get_time_and_clockread( + host_tsc_clocksource = kvm_get_time_and_clockread( &ka->master_kernel_ns, &ka->master_cycle_now); + ka->use_master_clock = host_tsc_clocksource & vcpus_matched; + if (ka->use_master_clock) atomic_set(&kvm_guest_has_master_clock, 1); vclock_mode = pvclock_gtod_data.clock.vclock_mode; - trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode); + trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode, + vcpus_matched); } static int kvm_guest_time_update(struct kvm_vcpu *v) Index: vsyscall/arch/x86/kvm/trace.h =================================================================== --- vsyscall.orig/arch/x86/kvm/trace.h +++ vsyscall/arch/x86/kvm/trace.h @@ -762,21 +762,54 @@ TRACE_EVENT( {VCLOCK_HPET, "hpet"} \ TRACE_EVENT(kvm_update_master_clock, - TP_PROTO(bool use_master_clock, unsigned int host_clock), - TP_ARGS(use_master_clock, host_clock), + TP_PROTO(bool use_master_clock, unsigned int host_clock, bool offset_matched), + TP_ARGS(use_master_clock, host_clock, offset_matched), TP_STRUCT__entry( __field( bool, use_master_clock ) __field( unsigned int, host_clock ) + __field( bool, offset_matched ) ), TP_fast_assign( __entry->use_master_clock = use_master_clock; __entry->host_clock = host_clock; + __entry->offset_matched = offset_matched; ), - TP_printk("masterclock %d hostclock %s", + TP_printk("masterclock %d hostclock %s offsetmatched %u", __entry->use_master_clock, + __print_symbolic(__entry->host_clock, host_clocks), + __entry->offset_matched) +); + +TRACE_EVENT(kvm_track_tsc, + TP_PROTO(unsigned int vcpu_id, unsigned int nr_matched, + unsigned int online_vcpus, bool use_master_clock, + unsigned int host_clock), + TP_ARGS(vcpu_id, nr_matched, online_vcpus, use_master_clock, + host_clock), + + TP_STRUCT__entry( + __field( unsigned int, vcpu_id ) + __field( unsigned int, nr_vcpus_matched_tsc ) + __field( unsigned int, online_vcpus ) + __field( bool, use_master_clock ) + __field( unsigned int, host_clock ) + ), + + TP_fast_assign( + __entry->vcpu_id = vcpu_id; + __entry->nr_vcpus_matched_tsc = nr_matched; + __entry->online_vcpus = online_vcpus; + __entry->use_master_clock = use_master_clock; + __entry->host_clock = host_clock; + ), + + TP_printk("vcpu_id %u masterclock %u offsetmatched %u nr_online %u" + " hostclock %s", + __entry->vcpu_id, __entry->use_master_clock, + __entry->nr_vcpus_matched_tsc, __entry->online_vcpus, __print_symbolic(__entry->host_clock, host_clocks)) );