Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- // SPDX-License-Identifier: GPL-2.0
- //! Recursive read acquisition of `cpufreq_driver_lock`, via the in-tree Rust cpufreq
- //! abstraction and the in-tree `rcpufreq_dt` driver.
- use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
- use kernel::bindings;
- use kernel::cpu::CpuId;
- use kernel::cpumask::CpumaskVar;
- use kernel::prelude::*;
- use kernel::time::delay::udelay;
- use kernel::time::Delta;
- use kernel::workqueue;
- module! {
- type: PocRace,
- name: "poc_race",
- authors: ["spidermana"],
- description: "recursive read_lock on cpufreq_driver_lock via rcpufreq_dt",
- license: "GPL",
- params: {
- run: u32 {
- default: 0,
- description: "1 = hammer both sides (HANGS THE GUEST)",
- },
- },
- }
- struct PocRace;
- const NO_CPU: u32 = u32::MAX;
- static READS: AtomicU64 = AtomicU64::new(0);
- static WRITES: AtomicU64 = AtomicU64::new(0);
- static READER_CPU: AtomicU32 = AtomicU32::new(NO_CPU);
- static WRITER_CPU: AtomicU32 = AtomicU32::new(NO_CPU);
- static STOP: AtomicBool = AtomicBool::new(false);
- unsafe extern "C" fn dummy_verify(_: *mut bindings::cpufreq_policy_data) -> kernel::ffi::c_int {
- 0
- }
- unsafe extern "C" fn dummy_init(_: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int {
- -1
- }
- unsafe extern "C" fn dummy_setpolicy(_: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int {
- 0
- }
- /// Busy-waits for `ms` milliseconds instead of sleeping.
- ///
- /// `msleep()` arms a timer on the current CPU and blocks. If a worker then wedges that CPU
- /// spinning with interrupts masked, the timer never fires and the sleeper never wakes.
- fn spin_ms(ms: u32) {
- // `udelay()` is only valid up to MAX_UDELAY_MS (5ms), so step in 1ms units.
- for _ in 0..ms {
- udelay(Delta::from_millis(1));
- }
- }
- /// Waits for `rcpufreq_dt` to bind;
- fn wait_for_policy() -> bool {
- let mut ms = 0;
- loop {
- // SAFETY: FFI call, valid for any CPU number.
- let p = unsafe { bindings::cpufreq_cpu_get(0) };
- if !p.is_null() {
- // SAFETY: `p` is the policy `cpufreq_cpu_get()` just referenced.
- unsafe { bindings::cpufreq_cpu_put(p) };
- return true;
- }
- if ms >= 5000 {
- return false;
- }
- // SAFETY: FFI call, process context.
- unsafe { bindings::msleep(50) };
- ms += 50;
- }
- }
- /// NMI-backtraces exactly one CPU, so the reader's stack and the writer's stack land in the
- /// log as two separately labelled chains instead of one dump of everything online.
- fn backtrace_cpu(cpu: u32) {
- let Some(id) = CpuId::from_u32(cpu) else {
- pr_err!(" (no CPU recorded for this side; nothing to backtrace)\n");
- return;
- };
- let Ok(mut mask) = CpumaskVar::new_zero(GFP_KERNEL) else {
- pr_err!(" (out of memory, cannot build the cpumask)\n");
- return;
- };
- mask.set(id);
- // SAFETY: FFI call; `mask` is a valid cpumask and the callee only reads it.
- unsafe { bindings::arch_trigger_cpumask_backtrace(mask.as_raw() as *const _, -1) };
- // `nmi_trigger_cpumask_backtrace()` waits for the NMI to be taken, but the printk of the
- // stack itself can still be in flight. Keep this short: see the sampling comment below.
- spin_ms(200);
- }
- fn report(reader_cpu: u32, writer_cpu: u32, reads: u64, writes: u64, ms: u32) {
- pr_err!("*** DEADLOCK on cpufreq_driver_lock ***\n");
- // The verdict can land before the first periodic progress line, so carry the evidence
- // it would have shown: both counters stopped, and stayed stopped.
- pr_err!(
- " reads={} writes={}, both flat for the last {}ms\n",
- reads,
- writes,
- ms
- );
- pr_err!(
- " reader cpu{} holds it for read and is blocked taking it for read again;\n",
- reader_cpu
- );
- pr_err!(
- " writer cpu{} is queued for write between the two acquisitions.\n",
- writer_cpu
- );
- pr_err!("--- [reader] cpu{}, expected chain: ---\n", reader_cpu);
- pr_err!(" cpufreq_quick_get -> read_lock #1 -> ->get() -> get_callback\n");
- pr_err!(" -> PolicyCpu::from_cpu -> cpufreq_cpu_get -> read_lock #2 (wedged)\n");
- backtrace_cpu(reader_cpu);
- pr_err!("--- [writer] cpu{}, expected chain: ---\n", writer_cpu);
- pr_err!(" cpufreq_register_driver -> write_lock (wedged)\n");
- backtrace_cpu(writer_cpu);
- }
- impl kernel::Module for PocRace {
- fn init(_module: &'static ThisModule) -> Result<Self> {
- pr_info!("poc_race: hammering cpufreq_driver_lock with recursive read_lock\n");
- if !wait_for_policy() {
- pr_err!("rcpufreq_dt did not bind within 5s; cannot hammer cpufreq_driver_lock\n");
- return Ok(PocRace);
- }
- if module_parameters::run.value() != 1 {
- pr_info!("pass poc_race.run=1 to hammer (this WILL hang the guest)\n");
- return Ok(PocRace);
- }
- // Reader: cpufreq_quick_get -> read_lock -> ->get() -> cpufreq_cpu_get -> read_lock.
- workqueue::system_unbound().try_spawn(GFP_KERNEL, || {
- pr_info!("[reader] entered\n");
- while !STOP.load(Ordering::Relaxed) {
- // Recorded before the call, so once wedged it names the CPU spinning in
- // `queued_read_lock_slowpath()`. Preemption cannot move a task that is
- // spinning with interrupts off, so the last value stays true.
- READER_CPU.store(CpuId::current().as_u32(), Ordering::Relaxed);
- // SAFETY: FFI call, valid for any CPU number.
- unsafe { bindings::cpufreq_quick_get(0) };
- READS.fetch_add(1, Ordering::Relaxed);
- }
- })?;
- // Writer.
- workqueue::system_unbound().try_spawn(GFP_KERNEL, || {
- pr_info!("[writer] entered\n");
- // SAFETY: a plain C struct of pointers; all-zero is a valid starting value.
- let mut drv = unsafe { core::mem::zeroed::<bindings::cpufreq_driver>() };
- drv.verify = Some(dummy_verify);
- drv.init = Some(dummy_init);
- drv.setpolicy = Some(dummy_setpolicy);
- for (d, s) in drv.name.iter_mut().zip(b"poc-race-dummy\0") {
- *d = *s as _;
- }
- while !STOP.load(Ordering::Relaxed) {
- WRITER_CPU.store(CpuId::current().as_u32(), Ordering::Relaxed);
- // SAFETY: `drv` is a valid vtable; the call returns -EEXIST after taking
- // the write lock, because rcpufreq_dt is already registered.
- unsafe { bindings::cpufreq_register_driver(&mut drv) };
- WRITES.fetch_add(1, Ordering::Relaxed);
- }
- })?;
- // a wedged CPU freezes the whole guest, this thread included, before it can report anything.
- // Sampling at 50ms reaches a verdict
- const TICK_MS: u32 = 50;
- let (mut lr, mut lw, mut stuck) = (0u64, 0u64, 0u32);
- for i in 1..=(20_000 / TICK_MS) {
- spin_ms(TICK_MS);
- let (r, w) = (
- READS.load(Ordering::Relaxed),
- WRITES.load(Ordering::Relaxed),
- );
- let (rc, wc) = (
- READER_CPU.load(Ordering::Relaxed),
- WRITER_CPU.load(Ordering::Relaxed),
- );
- // One line per 100ms
- if i % (100 / TICK_MS) == 0 {
- pr_info!(
- "t={}ms reads={} (cpu{}) writes={} (cpu{})\n",
- i * TICK_MS,
- r,
- rc as i32,
- w,
- wc as i32
- );
- }
- // Both sides must have entered their loop: counters alone cannot tell a lock
- // cycle apart from a worker that never started.
- if rc == NO_CPU || wc == NO_CPU || r != lr || w != lw {
- stuck = 0;
- (lr, lw) = (r, w);
- continue;
- }
- stuck += 1;
- if stuck < 3 {
- continue;
- }
- if rc == wc {
- // Two workers on one CPU wedge each other by starving the runqueue, which
- // looks identical in the counters and is not this bug.
- pr_err!(
- "both workers landed on cpu{}; inconclusive, not the AA deadlock\n",
- rc
- );
- STOP.store(true, Ordering::Relaxed);
- return Ok(PocRace);
- }
- report(rc, wc, r, w, stuck * TICK_MS);
- return Ok(PocRace);
- }
- pr_info!("no deadlock within 20s\n");
- STOP.store(true, Ordering::Relaxed);
- Ok(PocRace)
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment