| Message ID | 20170124171020.537965390@redhat.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
On Tue, Jan 24, 2017 at 03:09:42PM -0200, Marcelo Tosatti wrote: > Add a driver with gettime method returning hosts realtime clock. > This allows Chrony to synchronize host and guest clocks with > high precision (see results below). > > chronyc> sources > MS Name/IP address Stratum Poll Reach LastRx Last sample > =============================================================================== > #* PHC0 0 3 377 4 +162ns[ -683ns] +/- 11ns > > To configure Chronyd to use PHC refclock, add the > following line to its configuration file: > > refclock PHC /dev/ptpX poll 3 dpoll -2 offset 0 > > Where /dev/ptpX is the kvmclock PTP clock. > > Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com> Acked-by: Richard Cochran <richardcochran@gmail.com>
On 24/01/2017 18:09, Marcelo Tosatti wrote: > Add a driver with gettime method returning hosts realtime clock. > This allows Chrony to synchronize host and guest clocks with > high precision (see results below). > > chronyc> sources > MS Name/IP address Stratum Poll Reach LastRx Last sample > =============================================================================== > #* PHC0 0 3 377 4 +162ns[ -683ns] +/- 11ns > > To configure Chronyd to use PHC refclock, add the > following line to its configuration file: > > refclock PHC /dev/ptpX poll 3 dpoll -2 offset 0 > > Where /dev/ptpX is the kvmclock PTP clock. > > Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com> > > --- > drivers/ptp/Kconfig | 12 ++ > drivers/ptp/Makefile | 1 > drivers/ptp/ptp_kvm.c | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++ > 3 files changed, 215 insertions(+) > > v2: check for kvmclock (Radim) > initialize global variables before device registration (Radim) > v3: use cross timestamps callback (Paolo, Miroslav, Radim) > v4: remove gettime64 callback (Paolo) > v5: drop useless nsec assignment (Radim) > > Index: kvm-ptpdriver/drivers/ptp/Kconfig > =================================================================== > --- kvm-ptpdriver.orig/drivers/ptp/Kconfig 2017-01-24 09:03:47.604586098 -0200 > +++ kvm-ptpdriver/drivers/ptp/Kconfig 2017-01-24 09:05:06.321704124 -0200 > @@ -90,4 +90,16 @@ > To compile this driver as a module, choose M here: the module > will be called ptp_pch. > > +config PTP_1588_CLOCK_KVM > + tristate "KVM virtual PTP clock" > + depends on PTP_1588_CLOCK > + depends on KVM_GUEST > + default y > + help > + This driver adds support for using kvm infrastructure as a PTP > + clock. This clock is only useful if you are using KVM guests. > + > + To compile this driver as a module, choose M here: the module > + will be called ptp_kvm. > + > endmenu > Index: kvm-ptpdriver/drivers/ptp/Makefile > =================================================================== > --- kvm-ptpdriver.orig/drivers/ptp/Makefile 2017-01-24 09:03:47.604586098 -0200 > +++ kvm-ptpdriver/drivers/ptp/Makefile 2017-01-24 09:05:06.322704125 -0200 > @@ -6,3 +6,4 @@ > obj-$(CONFIG_PTP_1588_CLOCK) += ptp.o > obj-$(CONFIG_PTP_1588_CLOCK_IXP46X) += ptp_ixp46x.o > obj-$(CONFIG_PTP_1588_CLOCK_PCH) += ptp_pch.o > +obj-$(CONFIG_PTP_1588_CLOCK_KVM) += ptp_kvm.o > Index: kvm-ptpdriver/drivers/ptp/ptp_kvm.c > =================================================================== > --- /dev/null 1970-01-01 00:00:00.000000000 +0000 > +++ kvm-ptpdriver/drivers/ptp/ptp_kvm.c 2017-01-24 09:05:06.322704125 -0200 > @@ -0,0 +1,202 @@ > +/* > + * Virtual PTP 1588 clock for use with KVM guests > + * > + * Copyright (C) 2017 Red Hat Inc. > + * > + * This program is free software; you can redistribute it and/or modify > + * it under the terms of the GNU General Public License as published by > + * the Free Software Foundation; either version 2 of the License, or > + * (at your option) any later version. > + * > + * This program is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + * GNU General Public License for more details. > + * > + */ > +#include <linux/device.h> > +#include <linux/err.h> > +#include <linux/init.h> > +#include <linux/kernel.h> > +#include <linux/module.h> > +#include <uapi/linux/kvm_para.h> > +#include <asm/kvm_para.h> > +#include <asm/pvclock.h> > +#include <asm/kvmclock.h> > +#include <uapi/asm/kvm_para.h> > + > +#include <linux/ptp_clock_kernel.h> > + > +struct kvm_ptp_clock { > + struct ptp_clock *ptp_clock; > + struct ptp_clock_info caps; > +}; > + > +DEFINE_SPINLOCK(kvm_ptp_lock); > + > +static struct pvclock_vsyscall_time_info *hv_clock; > + > +static struct kvm_clock_offset clock_off; > +static phys_addr_t clock_off_gpa; > + > +static int ptp_kvm_get_time_fn(ktime_t *device_time, > + struct system_counterval_t *system_counter, > + void *ctx) > +{ > + unsigned long ret; > + struct timespec64 tspec; > + unsigned version; > + u8 flags; > + int cpu; > + struct pvclock_vcpu_time_info *src; > + > + preempt_disable_notrace(); > + cpu = smp_processor_id(); > + src = &hv_clock[cpu].pvti; > + > + spin_lock(&kvm_ptp_lock); Any reason to take the spinlock in the non-preemptable region? This would be worse for RT kernels than taking it outside. > + do { > + /* > + * We are using a TSC value read in the hosts > + * kvm_hc_clock_pairing handling. > + * So any changes to tsc_to_system_mul > + * and tsc_shift or any other pvclock > + * data invalidate that measurement. > + */ > + version = pvclock_read_begin(src); > + > + ret = kvm_hypercall2(KVM_HC_CLOCK_PAIRING, > + clock_off_gpa, > + KVM_CLOCK_PAIRING_WALLCLOCK); > + if (ret != 0) { > + pr_err("clock offset hypercall ret %lu\n", ret); Need ratelimiting in case something goes wrong and the host stops using the TSC clocksource? > + spin_unlock(&kvm_ptp_lock); > + preempt_enable_notrace(); > + return -EOPNOTSUPP; > + } > + > + tspec.tv_sec = clock_off.sec; > + tspec.tv_nsec = clock_off.nsec; > + ret = __pvclock_read_cycles(src, clock_off.tsc); > + flags = src->flags; Flags written and not read. Also, I think I disagree with Radim, gettime64 should use the TSC correction. It's the right thing to do for the POSIX clock interface. It's a bit arbitrary for PTP_SYS_OFFSET (though you cannot really argue with better precision), but chrony is now using PTP_SYS_OFFSET_PRECISE (and again you cannot really argue with better precision anyway). I'll test this a bit and then push it to kvm/queue. Please take a look! Paolo
On Wed, Jan 25, 2017 at 01:56:45PM +0100, Paolo Bonzini wrote: > > > On 24/01/2017 18:09, Marcelo Tosatti wrote: > > Add a driver with gettime method returning hosts realtime clock. > > This allows Chrony to synchronize host and guest clocks with > > high precision (see results below). > > > > chronyc> sources > > MS Name/IP address Stratum Poll Reach LastRx Last sample > > =============================================================================== > > #* PHC0 0 3 377 4 +162ns[ -683ns] +/- 11ns > > > > To configure Chronyd to use PHC refclock, add the > > following line to its configuration file: > > > > refclock PHC /dev/ptpX poll 3 dpoll -2 offset 0 > > > > Where /dev/ptpX is the kvmclock PTP clock. > > > > Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com> > > > > --- > > drivers/ptp/Kconfig | 12 ++ > > drivers/ptp/Makefile | 1 > > drivers/ptp/ptp_kvm.c | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++ > > 3 files changed, 215 insertions(+) > > > > v2: check for kvmclock (Radim) > > initialize global variables before device registration (Radim) > > v3: use cross timestamps callback (Paolo, Miroslav, Radim) > > v4: remove gettime64 callback (Paolo) > > v5: drop useless nsec assignment (Radim) > > > > Index: kvm-ptpdriver/drivers/ptp/Kconfig > > =================================================================== > > --- kvm-ptpdriver.orig/drivers/ptp/Kconfig 2017-01-24 09:03:47.604586098 -0200 > > +++ kvm-ptpdriver/drivers/ptp/Kconfig 2017-01-24 09:05:06.321704124 -0200 > > @@ -90,4 +90,16 @@ > > To compile this driver as a module, choose M here: the module > > will be called ptp_pch. > > > > +config PTP_1588_CLOCK_KVM > > + tristate "KVM virtual PTP clock" > > + depends on PTP_1588_CLOCK > > + depends on KVM_GUEST > > + default y > > + help > > + This driver adds support for using kvm infrastructure as a PTP > > + clock. This clock is only useful if you are using KVM guests. > > + > > + To compile this driver as a module, choose M here: the module > > + will be called ptp_kvm. > > + > > endmenu > > Index: kvm-ptpdriver/drivers/ptp/Makefile > > =================================================================== > > --- kvm-ptpdriver.orig/drivers/ptp/Makefile 2017-01-24 09:03:47.604586098 -0200 > > +++ kvm-ptpdriver/drivers/ptp/Makefile 2017-01-24 09:05:06.322704125 -0200 > > @@ -6,3 +6,4 @@ > > obj-$(CONFIG_PTP_1588_CLOCK) += ptp.o > > obj-$(CONFIG_PTP_1588_CLOCK_IXP46X) += ptp_ixp46x.o > > obj-$(CONFIG_PTP_1588_CLOCK_PCH) += ptp_pch.o > > +obj-$(CONFIG_PTP_1588_CLOCK_KVM) += ptp_kvm.o > > Index: kvm-ptpdriver/drivers/ptp/ptp_kvm.c > > =================================================================== > > --- /dev/null 1970-01-01 00:00:00.000000000 +0000 > > +++ kvm-ptpdriver/drivers/ptp/ptp_kvm.c 2017-01-24 09:05:06.322704125 -0200 > > @@ -0,0 +1,202 @@ > > +/* > > + * Virtual PTP 1588 clock for use with KVM guests > > + * > > + * Copyright (C) 2017 Red Hat Inc. > > + * > > + * This program is free software; you can redistribute it and/or modify > > + * it under the terms of the GNU General Public License as published by > > + * the Free Software Foundation; either version 2 of the License, or > > + * (at your option) any later version. > > + * > > + * This program is distributed in the hope that it will be useful, > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > + * GNU General Public License for more details. > > + * > > + */ > > +#include <linux/device.h> > > +#include <linux/err.h> > > +#include <linux/init.h> > > +#include <linux/kernel.h> > > +#include <linux/module.h> > > +#include <uapi/linux/kvm_para.h> > > +#include <asm/kvm_para.h> > > +#include <asm/pvclock.h> > > +#include <asm/kvmclock.h> > > +#include <uapi/asm/kvm_para.h> > > + > > +#include <linux/ptp_clock_kernel.h> > > + > > +struct kvm_ptp_clock { > > + struct ptp_clock *ptp_clock; > > + struct ptp_clock_info caps; > > +}; > > + > > +DEFINE_SPINLOCK(kvm_ptp_lock); > > + > > +static struct pvclock_vsyscall_time_info *hv_clock; > > + > > +static struct kvm_clock_offset clock_off; > > +static phys_addr_t clock_off_gpa; > > + > > +static int ptp_kvm_get_time_fn(ktime_t *device_time, > > + struct system_counterval_t *system_counter, > > + void *ctx) > > +{ > > + unsigned long ret; > > + struct timespec64 tspec; > > + unsigned version; > > + u8 flags; > > + int cpu; > > + struct pvclock_vcpu_time_info *src; > > + > > + preempt_disable_notrace(); > > + cpu = smp_processor_id(); > > + src = &hv_clock[cpu].pvti; > > + > > + spin_lock(&kvm_ptp_lock); > > Any reason to take the spinlock in the non-preemptable region? > This would be worse for RT kernels than taking it outside. Not really, can move it before. > > > + do { > > + /* > > + * We are using a TSC value read in the hosts > > + * kvm_hc_clock_pairing handling. > > + * So any changes to tsc_to_system_mul > > + * and tsc_shift or any other pvclock > > + * data invalidate that measurement. > > + */ > > + version = pvclock_read_begin(src); > > + > > + ret = kvm_hypercall2(KVM_HC_CLOCK_PAIRING, > > + clock_off_gpa, > > + KVM_CLOCK_PAIRING_WALLCLOCK); > > + if (ret != 0) { > > + pr_err("clock offset hypercall ret %lu\n", ret); > > Need ratelimiting in case something goes wrong and the host stops using > the TSC clocksource? Yes wise. > > + spin_unlock(&kvm_ptp_lock); > > + preempt_enable_notrace(); > > + return -EOPNOTSUPP; > > + } > > + > > + tspec.tv_sec = clock_off.sec; > > + tspec.tv_nsec = clock_off.nsec; > > + ret = __pvclock_read_cycles(src, clock_off.tsc); > > + flags = src->flags; > > Flags written and not read. > > Also, I think I disagree with Radim, gettime64 should use the TSC > correction. It's the right thing to do for the POSIX clock interface. > It's a bit arbitrary for PTP_SYS_OFFSET (though you cannot really argue > with better precision), but chrony is now using PTP_SYS_OFFSET_PRECISE > (and again you cannot really argue with better precision anyway). > > I'll test this a bit and then push it to kvm/queue. Please take a > look! > > Paolo Ok cool thanks Paolo, will review and ping if necessary.
=============================================================================== #* PHC0 0 3 377 4 +162ns[ -683ns] +/- 11ns To configure Chronyd to use PHC refclock, add the following line to its configuration file: refclock PHC /dev/ptpX poll 3 dpoll -2 offset 0 Where /dev/ptpX is the kvmclock PTP clock. Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com> --- drivers/ptp/Kconfig | 12 ++ drivers/ptp/Makefile | 1 drivers/ptp/ptp_kvm.c | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 215 insertions(+) v2: check for kvmclock (Radim) initialize global variables before device registration (Radim) v3: use cross timestamps callback (Paolo, Miroslav, Radim) v4: remove gettime64 callback (Paolo) v5: drop useless nsec assignment (Radim) Index: kvm-ptpdriver/drivers/ptp/Kconfig =================================================================== --- kvm-ptpdriver.orig/drivers/ptp/Kconfig 2017-01-24 09:03:47.604586098 -0200 +++ kvm-ptpdriver/drivers/ptp/Kconfig 2017-01-24 09:05:06.321704124 -0200 @@ -90,4 +90,16 @@ To compile this driver as a module, choose M here: the module will be called ptp_pch. +config PTP_1588_CLOCK_KVM + tristate "KVM virtual PTP clock" + depends on PTP_1588_CLOCK + depends on KVM_GUEST + default y + help + This driver adds support for using kvm infrastructure as a PTP + clock. This clock is only useful if you are using KVM guests. + + To compile this driver as a module, choose M here: the module + will be called ptp_kvm. + endmenu Index: kvm-ptpdriver/drivers/ptp/Makefile =================================================================== --- kvm-ptpdriver.orig/drivers/ptp/Makefile 2017-01-24 09:03:47.604586098 -0200 +++ kvm-ptpdriver/drivers/ptp/Makefile 2017-01-24 09:05:06.322704125 -0200 @@ -6,3 +6,4 @@ obj-$(CONFIG_PTP_1588_CLOCK) += ptp.o obj-$(CONFIG_PTP_1588_CLOCK_IXP46X) += ptp_ixp46x.o obj-$(CONFIG_PTP_1588_CLOCK_PCH) += ptp_pch.o +obj-$(CONFIG_PTP_1588_CLOCK_KVM) += ptp_kvm.o Index: kvm-ptpdriver/drivers/ptp/ptp_kvm.c =================================================================== --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ kvm-ptpdriver/drivers/ptp/ptp_kvm.c 2017-01-24 09:05:06.322704125 -0200 @@ -0,0 +1,202 @@ +/* + * Virtual PTP 1588 clock for use with KVM guests + * + * Copyright (C) 2017 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include <linux/device.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <uapi/linux/kvm_para.h> +#include <asm/kvm_para.h> +#include <asm/pvclock.h> +#include <asm/kvmclock.h> +#include <uapi/asm/kvm_para.h> + +#include <linux/ptp_clock_kernel.h> + +struct kvm_ptp_clock { + struct ptp_clock *ptp_clock; + struct ptp_clock_info caps; +}; + +DEFINE_SPINLOCK(kvm_ptp_lock); + +static struct pvclock_vsyscall_time_info *hv_clock; + +static struct kvm_clock_offset clock_off; +static phys_addr_t clock_off_gpa; + +static int ptp_kvm_get_time_fn(ktime_t *device_time, + struct system_counterval_t *system_counter, + void *ctx) +{ + unsigned long ret; + struct timespec64 tspec; + unsigned version; + u8 flags; + int cpu; + struct pvclock_vcpu_time_info *src; + + preempt_disable_notrace(); + cpu = smp_processor_id(); + src = &hv_clock[cpu].pvti; + + spin_lock(&kvm_ptp_lock); + + do { + /* + * We are using a TSC value read in the hosts + * kvm_hc_clock_pairing handling. + * So any changes to tsc_to_system_mul + * and tsc_shift or any other pvclock + * data invalidate that measurement. + */ + version = pvclock_read_begin(src); + + ret = kvm_hypercall2(KVM_HC_CLOCK_PAIRING, + clock_off_gpa, + KVM_CLOCK_PAIRING_WALLCLOCK); + if (ret != 0) { + pr_err("clock offset hypercall ret %lu\n", ret); + spin_unlock(&kvm_ptp_lock); + preempt_enable_notrace(); + return -EOPNOTSUPP; + } + + tspec.tv_sec = clock_off.sec; + tspec.tv_nsec = clock_off.nsec; + ret = __pvclock_read_cycles(src, clock_off.tsc); + flags = src->flags; + } while (pvclock_read_retry(src, version)); + + preempt_enable_notrace(); + + system_counter->cycles = ret; + system_counter->cs = &kvm_clock; + + *device_time = timespec_to_ktime(tspec); + + spin_unlock(&kvm_ptp_lock); + + return 0; +} + +static int ptp_kvm_getcrosststamp(struct ptp_clock_info *ptp, + struct system_device_crosststamp *xtstamp) +{ + return get_device_system_crosststamp(ptp_kvm_get_time_fn, NULL, + NULL, xtstamp); +} + +/* + * PTP clock operations + */ + +static int ptp_kvm_adjfreq(struct ptp_clock_info *ptp, s32 ppb) +{ + return -EOPNOTSUPP; +} + +static int ptp_kvm_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + return -EOPNOTSUPP; +} + +static int ptp_kvm_settime(struct ptp_clock_info *ptp, + const struct timespec64 *ts) +{ + return -EOPNOTSUPP; +} + +static int ptp_kvm_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts) +{ + unsigned long ret; + struct timespec64 tspec; + + spin_lock(&kvm_ptp_lock); + + ret = kvm_hypercall2(KVM_HC_CLOCK_PAIRING, + clock_off_gpa, + KVM_CLOCK_PAIRING_WALLCLOCK); + if (ret != 0) { + pr_err("clock offset hypercall ret %lu\n", ret); + spin_unlock(&kvm_ptp_lock); + return -EOPNOTSUPP; + } + + tspec.tv_sec = clock_off.sec; + tspec.tv_nsec = clock_off.nsec; + spin_unlock(&kvm_ptp_lock); + + memcpy(ts, &tspec, sizeof(struct timespec64)); + + return 0; +} + +static int ptp_kvm_enable(struct ptp_clock_info *ptp, + struct ptp_clock_request *rq, int on) +{ + return -EOPNOTSUPP; +} + +static struct ptp_clock_info ptp_kvm_caps = { + .owner = THIS_MODULE, + .name = "KVM virtual PTP", + .max_adj = 0, + .n_ext_ts = 0, + .n_pins = 0, + .pps = 0, + .adjfreq = ptp_kvm_adjfreq, + .adjtime = ptp_kvm_adjtime, + .gettime64 = ptp_kvm_gettime, + .settime64 = ptp_kvm_settime, + .enable = ptp_kvm_enable, + .getcrosststamp = ptp_kvm_getcrosststamp, +}; + +/* module operations */ + +static struct kvm_ptp_clock kvm_ptp_clock; + +static void __exit ptp_kvm_exit(void) +{ + ptp_clock_unregister(kvm_ptp_clock.ptp_clock); +} + +static int __init ptp_kvm_init(void) +{ + clock_off_gpa = slow_virt_to_phys(&clock_off); + hv_clock = pvclock_pvti_cpu0_va(); + + if (!hv_clock) + return -ENODEV; + + kvm_ptp_clock.caps = ptp_kvm_caps; + + kvm_ptp_clock.ptp_clock = ptp_clock_register(&kvm_ptp_clock.caps, NULL); + + if (IS_ERR(kvm_ptp_clock.ptp_clock)) + return PTR_ERR(kvm_ptp_clock.ptp_clock); + + return 0; +} + +module_init(ptp_kvm_init); +module_exit(ptp_kvm_exit); + +MODULE_AUTHOR("Marcelo Tosatti <mtosatti@redhat.com>"); +MODULE_DESCRIPTION("PTP clock using KVMCLOCK"); +MODULE_LICENSE("GPL");