Skip to content

Commit fe1d482

Browse files
fbqfbq
committed
Merge branch 'rcu-misc.20260111a'
* rcu-misc.20260111a: rcu: Reduce synchronize_rcu() latency by reporting GP kthread's CPU QS early srcu: Use suitable gfp_flags for the init_srcu_struct_nodes() rcu: Fix rcu_read_unlock() deadloop due to softirq rcutorture: Correctly compute probability to invoke ->exp_current() rcu: Make expedited RCU CPU stall warnings detect stall-end races
2 parents acb0b2f + bc3705e commit fe1d482

6 files changed

Lines changed: 31 additions & 11 deletions

File tree

‎kernel/rcu/rcutorture.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1749,7 +1749,7 @@ rcu_torture_writer(void *arg)
17491749
ulo[i] = cur_ops->get_comp_state();
17501750
gp_snap = cur_ops->start_gp_poll();
17511751
rcu_torture_writer_state = RTWS_POLL_WAIT;
1752-
if (cur_ops->exp_current && !torture_random(&rand) % 0xff)
1752+
if (cur_ops->exp_current && !(torture_random(&rand) & 0xff))
17531753
cur_ops->exp_current();
17541754
while (!cur_ops->poll_gp_state(gp_snap)) {
17551755
gp_snap1 = cur_ops->get_gp_state();
@@ -1771,7 +1771,7 @@ rcu_torture_writer(void *arg)
17711771
cur_ops->get_comp_state_full(&rgo[i]);
17721772
cur_ops->start_gp_poll_full(&gp_snap_full);
17731773
rcu_torture_writer_state = RTWS_POLL_WAIT_FULL;
1774-
if (cur_ops->exp_current && !torture_random(&rand) % 0xff)
1774+
if (cur_ops->exp_current && !(torture_random(&rand) & 0xff))
17751775
cur_ops->exp_current();
17761776
while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
17771777
cur_ops->get_gp_state_full(&gp_snap1_full);

‎kernel/rcu/srcutree.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
262262
ssp->srcu_sup->srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL;
263263
ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns();
264264
if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) {
265-
if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC))
265+
if (!init_srcu_struct_nodes(ssp, is_static ? GFP_ATOMIC : GFP_KERNEL))
266266
goto err_free_sda;
267267
WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG);
268268
}

‎kernel/rcu/tree.c‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
160160
unsigned long gps, unsigned long flags);
161161
static void invoke_rcu_core(void);
162162
static void rcu_report_exp_rdp(struct rcu_data *rdp);
163+
static void rcu_report_qs_rdp(struct rcu_data *rdp);
163164
static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);
164165
static bool rcu_rdp_is_offloaded(struct rcu_data *rdp);
165166
static bool rcu_rdp_cpu_online(struct rcu_data *rdp);
@@ -1983,6 +1984,17 @@ static noinline_for_stack bool rcu_gp_init(void)
19831984
if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
19841985
on_each_cpu(rcu_strict_gp_boundary, NULL, 0);
19851986

1987+
/*
1988+
* Immediately report QS for the GP kthread's CPU. The GP kthread
1989+
* cannot be in an RCU read-side critical section while running
1990+
* the FQS scan. This eliminates the need for a second FQS wait
1991+
* when all CPUs are idle.
1992+
*/
1993+
preempt_disable();
1994+
rcu_qs();
1995+
rcu_report_qs_rdp(this_cpu_ptr(&rcu_data));
1996+
preempt_enable();
1997+
19861998
return true;
19871999
}
19882000

‎kernel/rcu/tree.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ struct rcu_data {
203203
/* during and after the last grace */
204204
/* period it is aware of. */
205205
struct irq_work defer_qs_iw; /* Obtain later scheduler attention. */
206-
int defer_qs_iw_pending; /* Scheduler attention pending? */
206+
int defer_qs_pending; /* irqwork or softirq pending? */
207207
struct work_struct strict_work; /* Schedule readers for strict GPs. */
208208

209209
/* 2) batch handling */

‎kernel/rcu/tree_exp.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -589,7 +589,12 @@ static void synchronize_rcu_expedited_stall(unsigned long jiffies_start, unsigne
589589
pr_cont(" } %lu jiffies s: %lu root: %#lx/%c\n",
590590
j - jiffies_start, rcu_state.expedited_sequence, data_race(rnp_root->expmask),
591591
".T"[!!data_race(rnp_root->exp_tasks)]);
592-
if (ndetected) {
592+
if (!ndetected) {
593+
// This is invoked from the grace-period worker, so
594+
// a new grace period cannot have started. And if this
595+
// worker were stalled, we would not get here. ;-)
596+
pr_err("INFO: Expedited stall ended before state dump start\n");
597+
} else {
593598
pr_err("blocking rcu_node structures (internal RCU debug):");
594599
rcu_for_each_node_breadth_first(rnp) {
595600
if (rnp == rnp_root)

‎kernel/rcu/tree_plugin.h‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -487,8 +487,8 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags)
487487
union rcu_special special;
488488

489489
rdp = this_cpu_ptr(&rcu_data);
490-
if (rdp->defer_qs_iw_pending == DEFER_QS_PENDING)
491-
rdp->defer_qs_iw_pending = DEFER_QS_IDLE;
490+
if (rdp->defer_qs_pending == DEFER_QS_PENDING)
491+
rdp->defer_qs_pending = DEFER_QS_IDLE;
492492

493493
/*
494494
* If RCU core is waiting for this CPU to exit its critical section,
@@ -645,7 +645,7 @@ static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp)
645645
* 5. Deferred QS reporting does not happen.
646646
*/
647647
if (rcu_preempt_depth() > 0)
648-
WRITE_ONCE(rdp->defer_qs_iw_pending, DEFER_QS_IDLE);
648+
WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE);
649649
}
650650

651651
/*
@@ -747,19 +747,22 @@ static void rcu_read_unlock_special(struct task_struct *t)
747747
// Using softirq, safe to awaken, and either the
748748
// wakeup is free or there is either an expedited
749749
// GP in flight or a potential need to deboost.
750-
raise_softirq_irqoff(RCU_SOFTIRQ);
750+
if (rdp->defer_qs_pending != DEFER_QS_PENDING) {
751+
rdp->defer_qs_pending = DEFER_QS_PENDING;
752+
raise_softirq_irqoff(RCU_SOFTIRQ);
753+
}
751754
} else {
752755
// Enabling BH or preempt does reschedule, so...
753756
// Also if no expediting and no possible deboosting,
754757
// slow is OK. Plus nohz_full CPUs eventually get
755758
// tick enabled.
756759
set_need_resched_current();
757760
if (IS_ENABLED(CONFIG_IRQ_WORK) && irqs_were_disabled &&
758-
needs_exp && rdp->defer_qs_iw_pending != DEFER_QS_PENDING &&
761+
needs_exp && rdp->defer_qs_pending != DEFER_QS_PENDING &&
759762
cpu_online(rdp->cpu)) {
760763
// Get scheduler to re-evaluate and call hooks.
761764
// If !IRQ_WORK, FQS scan will eventually IPI.
762-
rdp->defer_qs_iw_pending = DEFER_QS_PENDING;
765+
rdp->defer_qs_pending = DEFER_QS_PENDING;
763766
irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu);
764767
}
765768
}

0 commit comments

Comments
 (0)