@@ -36,7 +36,7 @@ struct ms_hyperv_tsc_page *hv_get_tsc_page(void)
return tsc_pg;
}
-static u64 read_hv_clock_tsc(struct clocksource *arg)
+static u64 read_hv_clock_tsc(struct clocksource *arg, u64 *cycle_stamp)
{
u64 current_tick = hv_read_tsc_page(tsc_pg);
@@ -55,7 +55,7 @@ static struct clocksource hyperv_cs_tsc = {
};
#endif
-static u64 read_hv_clock_msr(struct clocksource *arg)
+static u64 read_hv_clock_msr(struct clocksource *arg, u64 *cycles_stamp)
{
u64 current_tick;
/*
@@ -14,7 +14,8 @@ static inline struct pvclock_vsyscall_time_info *pvclock_pvti_cpu0_va(void)
#endif
/* some helper functions for xen and kvm pv clock sources */
-u64 pvclock_clocksource_read(struct pvclock_vcpu_time_info *src);
+u64 pvclock_clocksource_read(struct pvclock_vcpu_time_info *src,
+ u64 *cycles_stamp);
u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src);
void pvclock_set_flags(u8 flags);
unsigned long pvclock_tsc_khz(struct pvclock_vcpu_time_info *src);
@@ -792,7 +792,7 @@ static union hpet_lock hpet __cacheline_aligned = {
{ .lock = __ARCH_SPIN_LOCK_UNLOCKED, },
};
-static u64 read_hpet(struct clocksource *cs)
+static u64 read_hpet(struct clocksource *cs, u64 *cycles_stamp)
{
unsigned long flags;
union hpet_lock old, new;
@@ -850,7 +850,7 @@ static u64 read_hpet(struct clocksource *cs)
/*
* For UP or 32-bit.
*/
-static u64 read_hpet(struct clocksource *cs)
+static u64 read_hpet(struct clocksource *cs, u64 *cycles_stamp)
{
return (u64)hpet_readl(HPET_COUNTER);
}
@@ -91,12 +91,12 @@ static u64 kvm_clock_read(void)
preempt_disable_notrace();
cpu = smp_processor_id();
src = &hv_clock[cpu].pvti;
- ret = pvclock_clocksource_read(src);
+ ret = pvclock_clocksource_read(src, NULL);
preempt_enable_notrace();
return ret;
}
-static u64 kvm_clock_get_cycles(struct clocksource *cs)
+static u64 kvm_clock_get_cycles(struct clocksource *cs, u64 *cycles_stamp)
{
return kvm_clock_read();
}
@@ -73,7 +73,8 @@ u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src)
return flags & valid_flags;
}
-u64 pvclock_clocksource_read(struct pvclock_vcpu_time_info *src)
+u64 pvclock_clocksource_read(struct pvclock_vcpu_time_info *src,
+ u64 *cycles_stamp)
{
unsigned version;
u64 ret;
@@ -136,7 +137,8 @@ void pvclock_read_wallclock(struct pvclock_wall_clock *wall_clock,
rmb(); /* fetch time before checking version */
} while ((wall_clock->version & 1) || (version != wall_clock->version));
- delta = pvclock_clocksource_read(vcpu_time); /* time since system boot */
+ /* time since system boot */
+ delta = pvclock_clocksource_read(vcpu_time, NULL);
delta += now.tv_sec * (u64)NSEC_PER_SEC + now.tv_nsec;
now.tv_nsec = do_div(delta, NSEC_PER_SEC);
@@ -1110,7 +1110,7 @@ static void tsc_resume(struct clocksource *cs)
* checking the result of read_tsc() - cycle_last for being negative.
* That works because CLOCKSOURCE_MASK(64) does not mask out any bit.
*/
-static u64 read_tsc(struct clocksource *cs)
+static u64 read_tsc(struct clocksource *cs, u64 *cycles_stamp)
{
return (u64)rdtsc_ordered();
}
@@ -916,7 +916,7 @@ static unsigned long lguest_tsc_khz(void)
* If we can't use the TSC, the kernel falls back to our lower-priority
* "lguest_clock", where we read the time value given to us by the Host.
*/
-static u64 lguest_clock_read(struct clocksource *cs)
+static u64 lguest_clock_read(struct clocksource *cs, u64 *cycles_stamp)
{
unsigned long sec, nsec;
@@ -30,7 +30,7 @@
#define RTC_NAME "sgi_rtc"
-static u64 uv_read_rtc(struct clocksource *cs);
+static u64 uv_read_rtc(struct clocksource *cs, u64 *cycles_stamp);
static int uv_rtc_next_event(unsigned long, struct clock_event_device *);
static int uv_rtc_shutdown(struct clock_event_device *evt);
@@ -133,7 +133,7 @@ static int uv_setup_intr(int cpu, u64 expires)
/* Initialize comparator value */
uv_write_global_mmr64(pnode, UVH_INT_CMPB, expires);
- if (uv_read_rtc(NULL) <= expires)
+ if (uv_read_rtc(NULL, NULL) <= expires)
return 0;
return !uv_intr_pending(pnode);
@@ -269,7 +269,7 @@ static int uv_rtc_unset_timer(int cpu, int force)
spin_lock_irqsave(&head->lock, flags);
- if ((head->next_cpu == bcpu && uv_read_rtc(NULL) >= *t) || force)
+ if ((head->next_cpu == bcpu && uv_read_rtc(NULL, NULL) >= *t) || force)
rc = 1;
if (rc) {
@@ -296,7 +296,7 @@ static int uv_rtc_unset_timer(int cpu, int force)
* cachelines of it's own page. This allows faster simultaneous reads
* from a given socket.
*/
-static u64 uv_read_rtc(struct clocksource *cs)
+static u64 uv_read_rtc(struct clocksource *cs, u64 *cycles_stamp)
{
unsigned long offset;
@@ -316,7 +316,7 @@ static int uv_rtc_next_event(unsigned long delta,
{
int ced_cpu = cpumask_first(ced->cpumask);
- return uv_rtc_set_timer(ced_cpu, delta + uv_read_rtc(NULL));
+ return uv_rtc_set_timer(ced_cpu, delta + uv_read_rtc(NULL, NULL));
}
/*
@@ -46,12 +46,12 @@ u64 xen_clocksource_read(void)
preempt_disable_notrace();
src = &__this_cpu_read(xen_vcpu)->time;
- ret = pvclock_clocksource_read(src);
+ ret = pvclock_clocksource_read(src, NULL);
preempt_enable_notrace();
return ret;
}
-static u64 xen_clocksource_get_cycles(struct clocksource *cs)
+static u64 xen_clocksource_get_cycles(struct clocksource *cs, u64 *cycles_stamp)
{
return xen_clocksource_read();
}
@@ -70,7 +70,7 @@ static u32 hpet_nhpet, hpet_max_freq = HPET_USER_FREQ;
#ifdef CONFIG_IA64
static void __iomem *hpet_mctr;
-static u64 read_hpet(struct clocksource *cs)
+static u64 read_hpet(struct clocksource *cs, u64 *cycles_stamp)
{
return (u64)read_counter((void __iomem *)hpet_mctr);
}
@@ -58,7 +58,7 @@ u32 acpi_pm_read_verified(void)
return v2;
}
-static u64 acpi_pm_read(struct clocksource *cs)
+static u64 acpi_pm_read(struct clocksource *cs, u64 *cycles_stamp)
{
return (u64)read_pmtmr();
}
@@ -81,7 +81,7 @@ static int __init acpi_pm_good_setup(char *__str)
}
__setup("acpi_pm_good", acpi_pm_good_setup);
-static u64 acpi_pm_read_slow(struct clocksource *cs)
+static u64 acpi_pm_read_slow(struct clocksource *cs, u64 *cycles_stamp)
{
return (u64)acpi_pm_read_verified();
}
@@ -149,9 +149,9 @@ static int verify_pmtmr_rate(void)
unsigned long count, delta;
mach_prepare_counter();
- value1 = clocksource_acpi_pm.read(&clocksource_acpi_pm);
+ value1 = clocksource_acpi_pm.read(&clocksource_acpi_pm, NULL);
mach_countup(&count);
- value2 = clocksource_acpi_pm.read(&clocksource_acpi_pm);
+ value2 = clocksource_acpi_pm.read(&clocksource_acpi_pm, NULL);
delta = (value2 - value1) & ACPI_PM_MASK;
/* Check that the PMTMR delta is within 5% of what we expect */
@@ -184,9 +184,10 @@ static int __init init_acpi_pm_clocksource(void)
/* "verify" this timing source: */
for (j = 0; j < ACPI_PM_MONOTONICITY_CHECKS; j++) {
udelay(100 * j);
- value1 = clocksource_acpi_pm.read(&clocksource_acpi_pm);
+ value1 = clocksource_acpi_pm.read(&clocksource_acpi_pm, NULL);
for (i = 0; i < ACPI_PM_READ_CHECKS; i++) {
- value2 = clocksource_acpi_pm.read(&clocksource_acpi_pm);
+ value2 = clocksource_acpi_pm.read(
+ &clocksource_acpi_pm, NULL);
if (value2 == value1)
continue;
if (value2 > value1)
@@ -218,7 +218,7 @@ static void hv_set_host_time(struct work_struct *work)
wrk = container_of(work, struct adj_time_work, work);
- reftime = hyperv_cs->read(hyperv_cs);
+ reftime = hyperv_cs->read(hyperv_cs, NULL);
newtime = wrk->host_time + (reftime - wrk->ref_time);
host_ts = ns_to_timespec64((newtime - WLTIMEDELTA) * 100);
@@ -278,7 +278,7 @@ static inline void adj_guesttime(u64 hosttime, u64 reftime, u8 adj_flags)
*/
spin_lock_irqsave(&host_ts.lock, flags);
- cur_reftime = hyperv_cs->read(hyperv_cs);
+ cur_reftime = hyperv_cs->read(hyperv_cs, NULL);
host_ts.host_time = hosttime;
host_ts.ref_time = cur_reftime;
ktime_get_snapshot(&host_ts.snap);
@@ -530,7 +530,7 @@ static int hv_ptp_gettime(struct ptp_clock_info *info, struct timespec64 *ts)
u64 newtime, reftime;
spin_lock_irqsave(&host_ts.lock, flags);
- reftime = hyperv_cs->read(hyperv_cs);
+ reftime = hyperv_cs->read(hyperv_cs, NULL);
newtime = host_ts.host_time + (reftime - host_ts.ref_time);
*ts = ns_to_timespec64((newtime - WLTIMEDELTA) * 100);
spin_unlock_irqrestore(&host_ts.lock, flags);
@@ -48,7 +48,10 @@ struct module;
* 400-499: Perfect
* The ideal clocksource. A must-use where
* available.
- * @read: returns a cycle value, passes clocksource as argument
+ * @read: returns a cycle value, passes as arguments clocksource
+ * and a pointer where the cycles "stamp" is stored which
+ * was used in calcualtion of the returning cycle value,
+ * if any, otherwise the pointer value is untouched.
* @enable: optional function to enable the clocksource
* @disable: optional function to disable the clocksource
* @mask: bitmask for two's complement
@@ -77,7 +80,7 @@ struct module;
* structure.
*/
struct clocksource {
- u64 (*read)(struct clocksource *cs);
+ u64 (*read)(struct clocksource *cs, u64 *cycles_stamp);
u64 mask;
u32 mult;
u32 shift;
@@ -194,8 +194,8 @@ static void clocksource_watchdog(unsigned long data)
}
local_irq_disable();
- csnow = cs->read(cs);
- wdnow = watchdog->read(watchdog);
+ csnow = cs->read(cs, NULL);
+ wdnow = watchdog->read(watchdog, NULL);
local_irq_enable();
/* Clocksource initialized ? */
@@ -48,7 +48,7 @@
#define JIFFIES_SHIFT 8
#endif
-static u64 jiffies_read(struct clocksource *cs)
+static u64 jiffies_read(struct clocksource *cs, u64 *cycles_stamp)
{
return (u64) jiffies;
}
@@ -131,11 +131,11 @@ static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
* a read of the fast-timekeeper tkrs (which is protected by its own locking
* and update logic).
*/
-static inline u64 tk_clock_read(struct tk_read_base *tkr)
+static inline u64 tk_clock_read(struct tk_read_base *tkr, u64 *cycles_stamp)
{
struct clocksource *clock = READ_ONCE(tkr->clock);
- return clock->read(clock);
+ return clock->read(clock, cycles_stamp);
}
#ifdef CONFIG_DEBUG_TIMEKEEPING
@@ -195,7 +195,7 @@ static inline u64 timekeeping_get_delta(struct tk_read_base *tkr)
*/
do {
seq = read_seqcount_begin(&tk_core.seq);
- now = tk_clock_read(tkr);
+ now = tk_clock_read(tkr, NULL);
last = tkr->cycle_last;
mask = tkr->mask;
max = tkr->clock->max_cycles;
@@ -229,7 +229,7 @@ static inline u64 timekeeping_get_delta(struct tk_read_base *tkr)
u64 cycle_now, delta;
/* read clocksource */
- cycle_now = tk_clock_read(tkr);
+ cycle_now = tk_clock_read(tkr, NULL);
/* calculate the delta since the last update_wall_time */
delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask);
@@ -259,7 +259,7 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
old_clock = tk->tkr_mono.clock;
tk->tkr_mono.clock = clock;
tk->tkr_mono.mask = clock->mask;
- tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono);
+ tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono, NULL);
tk->tkr_raw.clock = clock;
tk->tkr_raw.mask = clock->mask;
@@ -422,7 +422,7 @@ static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
now += timekeeping_delta_to_ns(tkr,
clocksource_delta(
- tk_clock_read(tkr),
+ tk_clock_read(tkr, NULL),
tkr->cycle_last,
tkr->mask));
} while (read_seqcount_retry(&tkf->seq, seq));
@@ -474,7 +474,7 @@ EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns);
/* Suspend-time cycles value for halted fast timekeeper. */
static u64 cycles_at_suspend;
-static u64 dummy_clock_read(struct clocksource *cs)
+static u64 dummy_clock_read(struct clocksource *cs, u64 *cycles_stamp)
{
return cycles_at_suspend;
}
@@ -499,7 +499,7 @@ static void halt_fast_timekeeper(struct timekeeper *tk)
struct tk_read_base *tkr = &tk->tkr_mono;
memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
- cycles_at_suspend = tk_clock_read(tkr);
+ cycles_at_suspend = tk_clock_read(tkr, NULL);
tkr_dummy.clock = &dummy_clock;
update_fast_timekeeper(&tkr_dummy, &tk_fast_mono);
@@ -674,7 +674,7 @@ static void timekeeping_forward_now(struct timekeeper *tk)
u64 cycle_now, delta;
u64 nsec;
- cycle_now = tk_clock_read(&tk->tkr_mono);
+ cycle_now = tk_clock_read(&tk->tkr_mono, NULL);
delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
tk->tkr_mono.cycle_last = cycle_now;
tk->tkr_raw.cycle_last = cycle_now;
@@ -950,7 +950,7 @@ void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
do {
seq = read_seqcount_begin(&tk_core.seq);
- now = tk_clock_read(&tk->tkr_mono);
+ now = tk_clock_read(&tk->tkr_mono, NULL);
systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
base_real = ktime_add(tk->tkr_mono.base,
@@ -1128,7 +1128,7 @@ int get_device_system_crosststamp(int (*get_time_fn)
* Check whether the system counter value provided by the
* device driver is on the current timekeeping interval.
*/
- now = tk_clock_read(&tk->tkr_mono);
+ now = tk_clock_read(&tk->tkr_mono, NULL);
interval_start = tk->tkr_mono.cycle_last;
if (!cycle_between(interval_start, cycles, now)) {
clock_was_set_seq = tk->clock_was_set_seq;
@@ -1649,7 +1649,7 @@ void timekeeping_resume(void)
* The less preferred source will only be tried if there is no better
* usable source. The rtc part is handled separately in rtc core code.
*/
- cycle_now = tk_clock_read(&tk->tkr_mono);
+ cycle_now = tk_clock_read(&tk->tkr_mono, NULL);
if ((clock->flags & CLOCK_SOURCE_SUSPEND_NONSTOP) &&
cycle_now > tk->tkr_mono.cycle_last) {
u64 nsec, cyc_delta;
@@ -2051,7 +2051,7 @@ void update_wall_time(void)
#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
offset = real_tk->cycle_interval;
#else
- offset = clocksource_delta(tk_clock_read(&tk->tkr_mono),
+ offset = clocksource_delta(tk_clock_read(&tk->tkr_mono, NULL),
tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
#endif
When using timekeepeing API in some cases it is useful to return cycles stamp value which has been used used along with the time calculated to use that cycles stamp value for other purpuses (e.g. in KVM master clock) Signed-off-by: Denis Plotnikov <dplotnikov@virtuozzo.com> --- arch/x86/hyperv/hv_init.c | 4 ++-- arch/x86/include/asm/pvclock.h | 3 ++- arch/x86/kernel/hpet.c | 4 ++-- arch/x86/kernel/kvmclock.c | 4 ++-- arch/x86/kernel/pvclock.c | 6 ++++-- arch/x86/kernel/tsc.c | 2 +- arch/x86/lguest/boot.c | 2 +- arch/x86/platform/uv/uv_time.c | 10 +++++----- arch/x86/xen/time.c | 4 ++-- drivers/char/hpet.c | 2 +- drivers/clocksource/acpi_pm.c | 13 +++++++------ drivers/hv/hv_util.c | 6 +++--- include/linux/clocksource.h | 7 +++++-- kernel/time/clocksource.c | 4 ++-- kernel/time/jiffies.c | 2 +- kernel/time/timekeeping.c | 26 +++++++++++++------------- 16 files changed, 53 insertions(+), 46 deletions(-)