// SPDX-License-Identifier: GPL-2.0

/*

* Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>

* Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar

* Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner

*

* High-resolution kernel timers

*

* In contrast to the low-resolution timeout API, aka timer wheel,

* hrtimers provide finer resolution and accuracy depending on system

* configuration and capabilities.

*

* Started by: Thomas Gleixner and Ingo Molnar

*

* Credits:

* Based on the original timer wheel code

*

* Help, testing, suggestions, bugfixes, improvements were

* provided by:

*

* George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel

* et. al.

*/

#include <linux/cpu.h>

#include <linux/export.h>

#include <linux/percpu.h>

#include <linux/hrtimer.h>

#include <linux/notifier.h>

#include <linux/syscalls.h>

#include <linux/interrupt.h>

#include <linux/tick.h>

#include <linux/err.h>

#include <linux/debugobjects.h>

#include <linux/sched/signal.h>

#include <linux/sched/sysctl.h>

#include <linux/sched/rt.h>

#include <linux/sched/deadline.h>

#include <linux/sched/nohz.h>

#include <linux/sched/debug.h>

#include <linux/sched/isolation.h>

#include <linux/timer.h>

#include <linux/freezer.h>

#include <linux/compat.h>

#include <linux/uaccess.h>

#include <trace/events/timer.h>

#include "tick-internal.h"

/*

* Masks for selecting the soft and hard context timers from

* cpu_base->active

*/

#define MASK_SHIFT (HRTIMER_BASE_MONOTONIC_SOFT)

#define HRTIMER_ACTIVE_HARD ((1U << MASK_SHIFT) - 1)

#define HRTIMER_ACTIVE_SOFT (HRTIMER_ACTIVE_HARD << MASK_SHIFT)

#define HRTIMER_ACTIVE_ALL (HRTIMER_ACTIVE_SOFT | HRTIMER_ACTIVE_HARD)

static void retrigger_next_event(void *arg);

/*

* The timer bases:

*

* There are more clockids than hrtimer bases. Thus, we index

* into the timer bases by the hrtimer_base_type enum. When trying

* to reach a base using a clockid, hrtimer_clockid_to_base()

* is used to convert from clockid to the proper hrtimer_base_type.

*/

DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =

{

.lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),

.clock_base =

{

{

.index = HRTIMER_BASE_MONOTONIC,

.clockid = CLOCK_MONOTONIC,

.get_time = &ktime_get,

},

{

.index = HRTIMER_BASE_REALTIME,

.clockid = CLOCK_REALTIME,

.get_time = &ktime_get_real,

},

{

.index = HRTIMER_BASE_BOOTTIME,

.clockid = CLOCK_BOOTTIME,

.get_time = &ktime_get_boottime,

},

{

.index = HRTIMER_BASE_TAI,

.clockid = CLOCK_TAI,

.get_time = &ktime_get_clocktai,

},

{

.index = HRTIMER_BASE_MONOTONIC_SOFT,

.clockid = CLOCK_MONOTONIC,

.get_time = &ktime_get,

},

{

.index = HRTIMER_BASE_REALTIME_SOFT,

.clockid = CLOCK_REALTIME,

.get_time = &ktime_get_real,

},

{

.index = HRTIMER_BASE_BOOTTIME_SOFT,

.clockid = CLOCK_BOOTTIME,

.get_time = &ktime_get_boottime,

},

{

.index = HRTIMER_BASE_TAI_SOFT,

.clockid = CLOCK_TAI,

.get_time = &ktime_get_clocktai,

},

},

.csd = CSD_INIT(retrigger_next_event, NULL)

};

static const int hrtimer_clock_to_base_table[MAX_CLOCKS] = {

/* Make sure we catch unsupported clockids */

[0 ... MAX_CLOCKS - 1] = HRTIMER_MAX_CLOCK_BASES,

[CLOCK_REALTIME] = HRTIMER_BASE_REALTIME,

[CLOCK_MONOTONIC] = HRTIMER_BASE_MONOTONIC,

[CLOCK_BOOTTIME] = HRTIMER_BASE_BOOTTIME,

[CLOCK_TAI] = HRTIMER_BASE_TAI,

};

static inline bool hrtimer_base_is_online(struct hrtimer_cpu_base *base)

{

if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))

return true;

else

return likely(base->online);

}

/*

* Functions and macros which are different for UP/SMP systems are kept in a

* single place

*/

#ifdef CONFIG_SMP

/*

* We require the migration_base for lock_hrtimer_base()/switch_hrtimer_base()

* such that hrtimer_callback_running() can unconditionally dereference

* timer->base->cpu_base

*/

static struct hrtimer_cpu_base migration_cpu_base = {

.clock_base = { {

.cpu_base = &migration_cpu_base,

.seq = SEQCNT_RAW_SPINLOCK_ZERO(migration_cpu_base.seq,

&migration_cpu_base.lock),

}, },

};

#define migration_base migration_cpu_base.clock_base[0]

/*

* We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock

* means that all timers which are tied to this base via timer->base are

* locked, and the base itself is locked too.

*

* So __run_timers/migrate_timers can safely modify all timers which could

* be found on the lists/queues.

*

* When the timer's base is locked, and the timer removed from list, it is

* possible to set timer->base = &migration_base and drop the lock: the timer

* remains locked.

*/

static

struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,

unsigned long *flags)

__acquires(&timer->base->lock)

{

struct hrtimer_clock_base *base;

for (;;) {

base = READ_ONCE(timer->base);

if (likely(base != &migration_base)) {

raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);

if (likely(base == timer->base))

return base;

/* The timer has migrated to another CPU: */

raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);

}

cpu_relax();

}

}

/*

* Check if the elected target is suitable considering its next

* event and the hotplug state of the current CPU.

*

* If the elected target is remote and its next event is after the timer

* to queue, then a remote reprogram is necessary. However there is no

* guarantee the IPI handling the operation would arrive in time to meet

* the high resolution deadline. In this case the local CPU becomes a

* preferred target, unless it is offline.

*

* High and low resolution modes are handled the same way for simplicity.

*

* Called with cpu_base->lock of target cpu held.

*/

static bool hrtimer_suitable_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base,

struct hrtimer_cpu_base *new_cpu_base,

struct hrtimer_cpu_base *this_cpu_base)

{

ktime_t expires;

/*

* The local CPU clockevent can be reprogrammed. Also get_target_base()

* guarantees it is online.

*/

if (new_cpu_base == this_cpu_base)

return true;

/*

* The offline local CPU can't be the default target if the

* next remote target event is after this timer. Keep the

* elected new base. An IPI will we issued to reprogram

* it as a last resort.

*/

if (!hrtimer_base_is_online(this_cpu_base))

return true;

expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);

return expires >= new_base->cpu_base->expires_next;

}

static inline struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base, int pinned)

{

if (!hrtimer_base_is_online(base)) {

int cpu = cpumask_any_and(cpu_online_mask, housekeeping_cpumask(HK_TYPE_TIMER));

return &per_cpu(hrtimer_bases, cpu);

}

#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)

if (static_branch_likely(&timers_migration_enabled) && !pinned)

return &per_cpu(hrtimer_bases, get_nohz_timer_target());

#endif

return base;

}

/*

* We switch the timer base to a power-optimized selected CPU target,

* if:

* - NO_HZ_COMMON is enabled

* - timer migration is enabled

* - the timer callback is not running

* - the timer is not the first expiring timer on the new target

*

* If one of the above requirements is not fulfilled we move the timer

* to the current CPU or leave it on the previously assigned CPU if

* the timer callback is currently running.

*/

static inline struct hrtimer_clock_base *

switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,

int pinned)

{

struct hrtimer_cpu_base *new_cpu_base, *this_cpu_base;

struct hrtimer_clock_base *new_base;

int basenum = base->index;

this_cpu_base = this_cpu_ptr(&hrtimer_bases);

new_cpu_base = get_target_base(this_cpu_base, pinned);

again:

new_base = &new_cpu_base->clock_base[basenum];

if (base != new_base) {

/*

* We are trying to move timer to new_base.

* However we can't change timer's base while it is running,

* so we keep it on the same CPU. No hassle vs. reprogramming

* the event source in the high resolution case. The softirq

* code will take care of this when the timer function has

* completed. There is no conflict as we hold the lock until

* the timer is enqueued.

*/

if (unlikely(hrtimer_callback_running(timer)))

return base;

/* See the comment in lock_hrtimer_base() */

WRITE_ONCE(timer->base, &migration_base);

raw_spin_unlock(&base->cpu_base->lock);

raw_spin_lock(&new_base->cpu_base->lock);

if (!hrtimer_suitable_target(timer, new_base, new_cpu_base,

this_cpu_base)) {

raw_spin_unlock(&new_base->cpu_base->lock);

raw_spin_lock(&base->cpu_base->lock);

new_cpu_base = this_cpu_base;

WRITE_ONCE(timer->base, base);

goto again;

}

WRITE_ONCE(timer->base, new_base);

} else {

if (!hrtimer_suitable_target(timer, new_base, new_cpu_base, this_cpu_base)) {

new_cpu_base = this_cpu_base;

goto again;

}

}

return new_base;

}

#else /* CONFIG_SMP */

static inline struct hrtimer_clock_base *

lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)

__acquires(&timer->base->cpu_base->lock)

{

struct hrtimer_clock_base *base = timer->base;

raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);

return base;

}

# define switch_hrtimer_base(t, b, p) (b)

#endif /* !CONFIG_SMP */

/*

* Functions for the union type storage format of ktime_t which are

* too large for inlining:

*/

#if BITS_PER_LONG < 64

/*

* Divide a ktime value by a nanosecond value

*/

s64 __ktime_divns(const ktime_t kt, s64 div)

{

int sft = 0;

s64 dclc;

u64 tmp;

dclc = ktime_to_ns(kt);

tmp = dclc < 0 ? -dclc : dclc;

/* Make sure the divisor is less than 2^32: */

while (div >> 32) {

sft++;

div >>= 1;

}

tmp >>= sft;

do_div(tmp, (u32) div);

return dclc < 0 ? -tmp : tmp;

}

EXPORT_SYMBOL_GPL(__ktime_divns);

#endif /* BITS_PER_LONG >= 64 */

/*

* Add two ktime values and do a safety check for overflow:

*/

ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs)

{

ktime_t res = ktime_add_unsafe(lhs, rhs);

/*

* We use KTIME_SEC_MAX here, the maximum timeout which we can

* return to user space in a timespec:

*/

if (res < 0 || res < lhs || res < rhs)

res = ktime_set(KTIME_SEC_MAX, 0);

return res;

}

EXPORT_SYMBOL_GPL(ktime_add_safe);

#ifdef CONFIG_DEBUG_OBJECTS_TIMERS

static const struct debug_obj_descr hrtimer_debug_descr;

static void *hrtimer_debug_hint(void *addr)

{

return ((struct hrtimer *) addr)->function;

}

/*

* fixup_init is called when:

* - an active object is initialized

*/

static bool hrtimer_fixup_init(void *addr, enum debug_obj_state state)

{

struct hrtimer *timer = addr;

switch (state) {

case ODEBUG_STATE_ACTIVE:

hrtimer_cancel(timer);

debug_object_init(timer, &hrtimer_debug_descr);

return true;

default:

return false;

}

}

/*

* fixup_activate is called when:

* - an active object is activated

* - an unknown non-static object is activated

*/

static bool hrtimer_fixup_activate(void *addr, enum debug_obj_state state)

{

switch (state) {

case ODEBUG_STATE_ACTIVE:

WARN_ON(1);

fallthrough;

default:

return false;

}

}

/*

* fixup_free is called when:

* - an active object is freed

*/

static bool hrtimer_fixup_free(void *addr, enum debug_obj_state state)

{

struct hrtimer *timer = addr;

switch (state) {

case ODEBUG_STATE_ACTIVE:

hrtimer_cancel(timer);

debug_object_free(timer, &hrtimer_debug_descr);

return true;

default:

return false;

}

}

static const struct debug_obj_descr hrtimer_debug_descr = {

.name = "hrtimer",

.debug_hint = hrtimer_debug_hint,

.fixup_init = hrtimer_fixup_init,

.fixup_activate = hrtimer_fixup_activate,

.fixup_free = hrtimer_fixup_free,

};

static inline void debug_hrtimer_init(struct hrtimer *timer)

{

debug_object_init(timer, &hrtimer_debug_descr);

}

static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer)

{

debug_object_init_on_stack(timer, &hrtimer_debug_descr);

}

static inline void debug_hrtimer_activate(struct hrtimer *timer,

enum hrtimer_mode mode)

{

debug_object_activate(timer, &hrtimer_debug_descr);

}

static inline void debug_hrtimer_deactivate(struct hrtimer *timer)

{

debug_object_deactivate(timer, &hrtimer_debug_descr);

}

void destroy_hrtimer_on_stack(struct hrtimer *timer)

{

debug_object_free(timer, &hrtimer_debug_descr);

}

EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);

#else

static inline void debug_hrtimer_init(struct hrtimer *timer) { }

static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer) { }

static inline void debug_hrtimer_activate(struct hrtimer *timer,

enum hrtimer_mode mode) { }

static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }

#endif

static inline void

debug_init(struct hrtimer *timer, clockid_t clockid,

enum hrtimer_mode mode)

{

debug_hrtimer_init(timer);

trace_hrtimer_init(timer, clockid, mode);

}

static inline void debug_init_on_stack(struct hrtimer *timer, clockid_t clockid,

enum hrtimer_mode mode)

{

debug_hrtimer_init_on_stack(timer);

trace_hrtimer_init(timer, clockid, mode);

}

static inline void debug_activate(struct hrtimer *timer,

enum hrtimer_mode mode)

{

debug_hrtimer_activate(timer, mode);

trace_hrtimer_start(timer, mode);

}

static inline void debug_deactivate(struct hrtimer *timer)

{

debug_hrtimer_deactivate(timer);

trace_hrtimer_cancel(timer);

}

static struct hrtimer_clock_base *

__next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)

{

unsigned int idx;

if (!*active)

return NULL;

idx = __ffs(*active);

*active &= ~(1U << idx);

return &cpu_base->clock_base[idx];

}

#define for_each_active_base(base, cpu_base, active) \

while ((base = __next_base((cpu_base), &(active))))

static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,

const struct hrtimer *exclude,

unsigned int active,

ktime_t expires_next)

{

struct hrtimer_clock_base *base;

ktime_t expires;

for_each_active_base(base, cpu_base, active) {

struct timerqueue_node *next;

struct hrtimer *timer;

next = timerqueue_getnext(&base->active);

timer = container_of(next, struct hrtimer, node);

if (timer == exclude) {

/* Get to the next timer in the queue. */

next = timerqueue_iterate_next(next);

if (!next)

continue;

timer = container_of(next, struct hrtimer, node);

}

expires = ktime_sub(hrtimer_get_expires(timer), base->offset);

if (expires < expires_next) {

expires_next = expires;

/* Skip cpu_base update if a timer is being excluded. */

if (exclude)

continue;

if (timer->is_soft)

cpu_base->softirq_next_timer = timer;

else

cpu_base->next_timer = timer;

}

}

/*

* clock_was_set() might have changed base->offset of any of

* the clock bases so the result might be negative. Fix it up

* to prevent a false positive in clockevents_program_event().

*/

if (expires_next < 0)

expires_next = 0;

return expires_next;

}

/*

* Recomputes cpu_base::*next_timer and returns the earliest expires_next

* but does not set cpu_base::*expires_next, that is done by

* hrtimer[_force]_reprogram and hrtimer_interrupt only. When updating

* cpu_base::*expires_next right away, reprogramming logic would no longer

* work.

*

* When a softirq is pending, we can ignore the HRTIMER_ACTIVE_SOFT bases,

* those timers will get run whenever the softirq gets handled, at the end of

* hrtimer_run_softirq(), hrtimer_update_softirq_timer() will re-add these bases.

*

* Therefore softirq values are those from the HRTIMER_ACTIVE_SOFT clock bases.

* The !softirq values are the minima across HRTIMER_ACTIVE_ALL, unless an actual

* softirq is pending, in which case they're the minima of HRTIMER_ACTIVE_HARD.

*

* @active_mask must be one of:

* - HRTIMER_ACTIVE_ALL,

* - HRTIMER_ACTIVE_SOFT, or

* - HRTIMER_ACTIVE_HARD.

*/

static ktime_t

__hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)

{

unsigned int active;

struct hrtimer *next_timer = NULL;

ktime_t expires_next = KTIME_MAX;

if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {

active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;

cpu_base->softirq_next_timer = NULL;

expires_next = __hrtimer_next_event_base(cpu_base, NULL,

active, KTIME_MAX);

next_timer = cpu_base->softirq_next_timer;

}

if (active_mask & HRTIMER_ACTIVE_HARD) {

active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;

cpu_base->next_timer = next_timer;

expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,

expires_next);

}

return expires_next;

}

static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)

{

ktime_t expires_next, soft = KTIME_MAX;

/*

* If the soft interrupt has already been activated, ignore the

* soft bases. They will be handled in the already raised soft

* interrupt.

*/

if (!cpu_base->softirq_activated) {

soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);

/*

* Update the soft expiry time. clock_settime() might have

* affected it.

*/

cpu_base->softirq_expires_next = soft;

}

expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD);

/*

* If a softirq timer is expiring first, update cpu_base->next_timer

* and program the hardware with the soft expiry time.

*/

if (expires_next > soft) {

cpu_base->next_timer = cpu_base->softirq_next_timer;

expires_next = soft;

}

return expires_next;

}

static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)

{

ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;

ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;

ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;

ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq,

offs_real, offs_boot, offs_tai);

base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real;

base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot;

base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai;

return now;

}

/*

* Is the high resolution mode active ?

*/

static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)

{

return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?

cpu_base->hres_active : 0;

}

static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,

struct hrtimer *next_timer,

ktime_t expires_next)

{

cpu_base->expires_next = expires_next;

/*

* If hres is not active, hardware does not have to be

* reprogrammed yet.

*

* If a hang was detected in the last timer interrupt then we

* leave the hang delay active in the hardware. We want the

* system to make progress. That also prevents the following

* scenario:

* T1 expires 50ms from now

* T2 expires 5s from now

*

* T1 is removed, so this code is called and would reprogram

* the hardware to 5s from now. Any hrtimer_start after that

* will not reprogram the hardware due to hang_detected being

* set. So we'd effectively block all timers until the T2 event

* fires.

*/

if (!hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)

return;

tick_program_event(expires_next, 1);

}

/*

* Reprogram the event source with checking both queues for the

* next event

* Called with interrupts disabled and base->lock held

*/

static void

hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)

{

ktime_t expires_next;

expires_next = hrtimer_update_next_event(cpu_base);

if (skip_equal && expires_next == cpu_base->expires_next)

return;

__hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);

}

/* High resolution timer related functions */

#ifdef CONFIG_HIGH_RES_TIMERS

/*

* High resolution timer enabled ?

*/

static bool hrtimer_hres_enabled __read_mostly = true;

unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;

EXPORT_SYMBOL_GPL(hrtimer_resolution);

/*

* Enable / Disable high resolution mode

*/

static int __init setup_hrtimer_hres(char *str)

{

return (kstrtobool(str, &hrtimer_hres_enabled) == 0);

}

__setup("highres=", setup_hrtimer_hres);

/*

* hrtimer_high_res_enabled - query, if the highres mode is enabled

*/

static inline int hrtimer_is_hres_enabled(void)

{

return hrtimer_hres_enabled;

}

/*

* Switch to high resolution mode

*/

static void hrtimer_switch_to_hres(void)

{

struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);

if (tick_init_highres()) {

pr_warn("Could not switch to high resolution mode on CPU %u\n",

base->cpu);

return;

}

base->hres_active = 1;

hrtimer_resolution = HIGH_RES_NSEC;

tick_setup_sched_timer(true);

/* "Retrigger" the interrupt to get things going */

retrigger_next_event(NULL);

}

#else

static inline int hrtimer_is_hres_enabled(void) { return 0; }

static inline void hrtimer_switch_to_hres(void) { }

#endif /* CONFIG_HIGH_RES_TIMERS */

/*

* Retrigger next event is called after clock was set with interrupts

* disabled through an SMP function call or directly from low level

* resume code.

*

* This is only invoked when:

* - CONFIG_HIGH_RES_TIMERS is enabled.

* - CONFIG_NOHZ_COMMON is enabled

*

* For the other cases this function is empty and because the call sites

* are optimized out it vanishes as well, i.e. no need for lots of

* #ifdeffery.

*/

static void retrigger_next_event(void *arg)

{

struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);

/*

* When high resolution mode or nohz is active, then the offsets of

* CLOCK_REALTIME/TAI/BOOTTIME have to be updated. Otherwise the

* next tick will take care of that.

*

* If high resolution mode is active then the next expiring timer

* must be reevaluated and the clock event device reprogrammed if

* necessary.

*

* In the NOHZ case the update of the offset and the reevaluation

* of the next expiring timer is enough. The return from the SMP

* function call will take care of the reprogramming in case the

* CPU was in a NOHZ idle sleep.

*/

if (!hrtimer_hres_active(base) && !tick_nohz_active)

return;

raw_spin_lock(&base->lock);

hrtimer_update_base(base);

if (hrtimer_hres_active(base))

hrtimer_force_reprogram(base, 0);

else

hrtimer_update_next_event(base);

raw_spin_unlock(&base->lock);

}

/*

* When a timer is enqueued and expires earlier than the already enqueued

* timers, we have to check, whether it expires earlier than the timer for

* which the clock event device was armed.

*

* Called with interrupts disabled and base->cpu_base.lock held

*/

static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)

{

struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);

struct hrtimer_clock_base *base = timer->base;

ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);

WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);

/*

* CLOCK_REALTIME timer might be requested with an absolute

* expiry time which is less than base->offset. Set it to 0.

*/

if (expires < 0)

expires = 0;

if (timer->is_soft) {

/*

* soft hrtimer could be started on a remote CPU. In this

* case softirq_expires_next needs to be updated on the

* remote CPU. The soft hrtimer will not expire before the

* first hard hrtimer on the remote CPU -

* hrtimer_check_target() prevents this case.

*/

struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;

if (timer_cpu_base->softirq_activated)

return;

if (!ktime_before(expires, timer_cpu_base->softirq_expires_next))

return;

timer_cpu_base->softirq_next_timer = timer;

timer_cpu_base->softirq_expires_next = expires;

if (!ktime_before(expires, timer_cpu_base->expires_next) ||

!reprogram)

return;

}

/*

* If the timer is not on the current cpu, we cannot reprogram

* the other cpus clock event device.

*/

if (base->cpu_base != cpu_base)

return;

if (expires >= cpu_base->expires_next)

return;

/*

* If the hrtimer interrupt is running, then it will reevaluate the

* clock bases and reprogram the clock event device.

*/

if (cpu_base->in_hrtirq)

return;

cpu_base->next_timer = timer;

__hrtimer_reprogram(cpu_base, timer, expires);

}

static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base,

unsigned int active)

{

struct hrtimer_clock_base *base;

unsigned int seq;

ktime_t expires;

/*

* Update the base offsets unconditionally so the following

* checks whether the SMP function call is required works.

*

* The update is safe even when the remote CPU is in the hrtimer

* interrupt or the hrtimer soft interrupt and expiring affected

* bases. Either it will see the update before handling a base or

* it will see it when it finishes the processing and reevaluates

* the next expiring timer.

*/

seq = cpu_base->clock_was_set_seq;

hrtimer_update_base(cpu_base);

/*

* If the sequence did not change over the update then the

* remote CPU already handled it.

*/

if (seq == cpu_base->clock_was_set_seq)

return false;

/*

* If the remote CPU is currently handling an hrtimer interrupt, it

* will reevaluate the first expiring timer of all clock bases

* before reprogramming. Nothing to do here.

*/

if (cpu_base->in_hrtirq)

return false;

/*

* Walk the affected clock bases and check whether the first expiring

* timer in a clock base is moving ahead of the first expiring timer of

* @cpu_base. If so, the IPI must be invoked because per CPU clock

* event devices cannot be remotely reprogrammed.

*/

active &= cpu_base->active_bases;

for_each_active_base(base, cpu_base, active) {

struct timerqueue_node *next;

next = timerqueue_getnext(&base->active);

expires = ktime_sub(next->expires, base->offset);

if (expires < cpu_base->expires_next)

return true;

/* Extra check for softirq clock bases */

if (base->clockid < HRTIMER_BASE_MONOTONIC_SOFT)

continue;

if (cpu_base->softirq_activated)

continue;

if (expires < cpu_base->softirq_expires_next)

return true;

}

return false;

}

/*

* Clock was set. This might affect CLOCK_REALTIME, CLOCK_TAI and

* CLOCK_BOOTTIME (for late sleep time injection).

*

* This requires to update the offsets for these clocks

* vs. CLOCK_MONOTONIC. When high resolution timers are enabled, then this

* also requires to eventually reprogram the per CPU clock event devices

* when the change moves an affected timer ahead of the first expiring

* timer on that CPU. Obviously remote per CPU clock event devices cannot

* be reprogrammed. The other reason why an IPI has to be sent is when the

* system is in !HIGH_RES and NOHZ mode. The NOHZ mode updates the offsets

* in the tick, which obviously might be stopped, so this has to bring out

* the remote CPU which might sleep in idle to get this sorted.

*/

void clock_was_set(unsigned int bases)

{

struct hrtimer_cpu_base *cpu_base = raw_cpu_ptr(&hrtimer_bases);

cpumask_var_t mask;

int cpu;

if (!hrtimer_hres_active(cpu_base) && !tick_nohz_active)

goto out_timerfd;

if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {

on_each_cpu(retrigger_next_event, NULL, 1);

goto out_timerfd;

}

/* Avoid interrupting CPUs if possible */

cpus_read_lock();

for_each_online_cpu(cpu) {

unsigned long flags;

cpu_base = &per_cpu(hrtimer_bases, cpu);

raw_spin_lock_irqsave(&cpu_base->lock, flags);

if (update_needs_ipi(cpu_base, bases))

cpumask_set_cpu(cpu, mask);

raw_spin_unlock_irqrestore(&cpu_base->lock, flags);

}

preempt_disable();

smp_call_function_many(mask, retrigger_next_event, NULL, 1);

preempt_enable();

cpus_read_unlock();

free_cpumask_var(mask);

out_timerfd:

timerfd_clock_was_set();

}

static void clock_was_set_work(struct work_struct *work)

{

clock_was_set(CLOCK_SET_WALL);

}