From 9199ac4e7ac30919223d562c9b132fab8f8812c3 Mon Sep 17 00:00:00 2001 From: nokosaaan Date: Fri, 16 Oct 2026 13:03:37 +0900 Subject: [PATCH 1/3] feat(usb): enumerate devices on xHCI root hub ports Power the root hub ports, reset every connected port, then Enable Slot, Address Device and read the Device Descriptor of each device over EP0 control transfers, logging its VID/PID and class. Device Contexts and EP0 rings stay allocated for the controller's lifetime because the controller keeps pointers to them; the control transfer data buffer is per-device and persistent so a timed-out transfer can never DMA into freed memory. Co-Authored-By: Claude Opus 5.5 --- awkernel_drivers/src/pcie/usb/xhci/mod.rs | 431 ++++++++++++++++++++- awkernel_drivers/src/pcie/usb/xhci/regs.rs | 45 +++ awkernel_drivers/src/pcie/usb/xhci/ring.rs | 7 + 3 files changed, 474 insertions(+), 9 deletions(-) diff --git a/awkernel_drivers/src/pcie/usb/xhci/mod.rs b/awkernel_drivers/src/pcie/usb/xhci/mod.rs index 05b4010b4..62c279c8c 100644 --- a/awkernel_drivers/src/pcie/usb/xhci/mod.rs +++ b/awkernel_drivers/src/pcie/usb/xhci/mod.rs @@ -1,14 +1,14 @@ -//! xHCI host controller driver: controller bring-up (reset, rings, start). +//! xHCI host controller driver: boot-time USB enumeration. //! //! The driver is fully polled (USBCMD.INTE = 0). //! //! Path classification: -//! - Non-RT: [`attach`] and everything it calls (controller reset and start). -//! Runs once at boot; allocates DMA pages and busy-waits with bounded polling -//! budgets. +//! - Non-RT: [`attach`] and everything it calls (controller reset, port reset, +//! enumeration). Runs once at boot; allocates DMA pages and busy-waits with +//! bounded polling budgets. use alloc::{borrow::Cow, format, string::String, sync::Arc, vec::Vec}; -use core::cell::UnsafeCell; +use core::{cell::UnsafeCell, mem::ManuallyDrop, ptr::read_volatile}; use awkernel_lib::{addr::Addr, dma_pool::DMAPool}; @@ -21,8 +21,8 @@ use super::super::{ mod regs; mod ring; -use regs::{completion, trb_type}; -use ring::{publish, CommandRing, Dcbaa, EventRing, Trb}; +use regs::{completion, trb_flag, trb_type}; +use ring::{publish, CommandRing, Dcbaa, EventRing, TransferRing, Trb}; // --------------------------------------------------------------------------- // Constants @@ -32,6 +32,11 @@ use ring::{publish, CommandRing, Dcbaa, EventRing, Trb}; const STATE_POLL_ITERS: u32 = 1_000_000; /// Polling budget for one command or transfer completion event. const COMPLETION_POLL_ITERS: u32 = 2_000_000; +/// A control transfer reports at most a Short Packet (Data Stage) followed by +/// the Status Stage event; allow one spare event. +const MAX_CTRL_EVENTS: usize = 3; +/// VBUS stabilization / device connect wait before scanning ports. +const PORT_SETTLE_USEC: u64 = 500_000; /// All DMA allocations go to NUMA node 0. const NUMA_NODE: usize = 0; @@ -41,6 +46,19 @@ type Page = [u8; PAGE_SIZE]; /// Scratchpad pointer array: one 8-byte entry per page, in one DMA page. const MAX_SCRATCHPAD: usize = PAGE_SIZE / 8; +/// Device Context Index of the default control endpoint (EP0). +const EP0_DCI: u8 = 1; + +/// Endpoint Context DW1: CErr = 3, EP Type = Control (4). +const EP_CTX_CONTROL: u8 = (3 << 1) | (4 << 3); + +/// PORTSC speed values (xHCI spec §7.2.2.1.1 default PSI). +const SPEED_LOW: u32 = 2; +const SPEED_HIGH: u32 = 3; +const SPEED_SUPER: u32 = 4; +const SPEED_SUPER_PLUS: u32 = 5; +const PORTSC_SPEED_SHIFT: u32 = 10; + // --------------------------------------------------------------------------- // Errors // --------------------------------------------------------------------------- @@ -56,6 +74,8 @@ pub(super) enum XhciErr { /// A command or transfer completed with this (non-success) completion code. Completion(u8), RingFull, + InvalidPort, + InvalidSlot, } impl From for PCIeDeviceErr { @@ -93,12 +113,94 @@ fn hi32(v: u64) -> u32 { u32::from_le_bytes([a, b, c, d]) } +/// TRB ctrl bits for Slot ID (ctrl[31:24]) and Endpoint ID (ctrl[20:16]). +fn ctrl_slot_ep(slot: u8, dci: u8) -> u32 { + u32::from_le_bytes([0, 0, dci, slot]) +} + +/// Write a little-endian u32 into an Input Context page. +fn ctx_write32(page: &mut Page, off: usize, val: u32) -> Result<(), XhciErr> { + let end = off.checked_add(4).ok_or(XhciErr::Overflow)?; + page.get_mut(off..end) + .ok_or(XhciErr::Overflow)? + .copy_from_slice(&val.to_le_bytes()); + Ok(()) +} + fn new_page() -> Result, XhciErr> { let mut page = DMAPool::::new(NUMA_NODE, 1).ok_or(XhciErr::DmaAlloc)?; page.as_mut().fill(0); Ok(page) } +// --------------------------------------------------------------------------- +// USB control requests +// --------------------------------------------------------------------------- + +/// USB SETUP packet (USB 2.0 spec §9.3). +#[derive(Clone, Copy)] +struct SetupPacket { + request_type: u8, + request: u8, + value: u16, + index: u16, + length: u16, +} + +impl SetupPacket { + /// Pack into the 8-byte immediate data of a Setup Stage TRB. + fn to_param(self) -> u64 { + let [rt, rq] = [self.request_type, self.request]; + let [v0, v1] = self.value.to_le_bytes(); + let [i0, i1] = self.index.to_le_bytes(); + let [l0, l1] = self.length.to_le_bytes(); + u64::from_le_bytes([rt, rq, v0, v1, i0, i1, l0, l1]) + } + + /// True when the data stage (if any) moves data from device to host. + fn is_in(&self) -> bool { + self.request_type & 0x80 != 0 + } +} + +const RT_DEV_IN: u8 = 0x80; // D→H, Standard, Device + +const REQ_GET_DESCRIPTOR: u8 = 6; + +const DESC_DEVICE: u16 = 0x0100; +const DEVICE_DESC_LEN: u16 = 18; + +// --------------------------------------------------------------------------- +// Device state +// --------------------------------------------------------------------------- + +/// Device Descriptor fields kept after enumeration. +#[derive(Clone, Copy, Default)] +struct DeviceDescriptor { + vendor: u16, + product: u16, + bcd_device: u16, +} + +/// One addressed USB device. +/// +/// Always held in `ManuallyDrop`: once a slot is addressed, the controller keeps +/// pointers to the Device Context and the rings, so freeing them would let the +/// controller DMA into reused memory. A few pages per enumerated device are +/// leaked at boot instead. +struct UsbDevice { + slot: u8, + port: u8, + /// PORTSC speed value (bits [13:10], already shifted down). + speed: u32, + ep0: TransferRing, + /// Output Device Context, referenced by DCBAA[slot]. + _ctx: DMAPool, + /// Data-stage buffer shared by all control transfers of this device. + ctrl_buf: DMAPool, + desc: DeviceDescriptor, +} + // --------------------------------------------------------------------------- // Controller core: command ring, event ring and doorbells // --------------------------------------------------------------------------- @@ -142,6 +244,17 @@ impl HcCore { Err(XhciErr::Timeout) } + /// Consume and discard every pending event (at most one ring's worth). + fn drain_events(&mut self) -> Result<(), XhciErr> { + for _ in 0..ring::RING_SIZE { + if self.evt_ring.dequeue()?.is_none() { + break; + } + self.update_erdp()?; + } + Ok(()) + } + /// Submit a command and wait for its Command Completion event. fn command(&mut self, trb: Trb) -> Result { self.cmd_ring.enqueue(trb)?; @@ -152,6 +265,58 @@ impl HcCore { code => Err(XhciErr::Completion(code)), } } + + /// Wait for the Status Stage of a control transfer (Short Packet on the + /// Data Stage is skipped). + fn wait_transfer(&mut self, max_events: usize) -> Result<(), XhciErr> { + for _ in 0..max_events { + match self.wait_event(trb_type::TRANSFER_EVENT)?.completion_code() { + completion::SUCCESS => return Ok(()), + completion::SHORT_PACKET => continue, + code => return Err(XhciErr::Completion(code)), + } + } + Err(XhciErr::Completion(completion::SHORT_PACKET)) + } + + /// Run one control transfer on EP0 of `dev`. The data stage (if + /// `setup.length > 0`) uses `dev.ctrl_buf`. + fn control(&mut self, dev: &mut UsbDevice, setup: SetupPacket) -> Result<(), XhciErr> { + let is_in = setup.is_in(); + let has_data = setup.length > 0; + let trt = match (has_data, is_in) { + (false, _) => trb_flag::TRT_NO_DATA, + (true, true) => trb_flag::TRT_IN, + (true, false) => trb_flag::TRT_OUT, + }; + + dev.ep0.enqueue(Trb { + param: setup.to_param(), + status: 8, // setup packet size + ctrl: trb_type::SETUP_STAGE | trb_flag::IDT | trt, + })?; + if has_data { + dev.ep0.enqueue(Trb { + param: phys_of(&dev.ctrl_buf)?, + status: u32::from(setup.length), + ctrl: trb_type::DATA_STAGE | if is_in { trb_flag::DIR_IN } else { 0 }, + })?; + } + // Status Stage runs opposite to the data stage (IN when there is none). + let status_dir = if has_data && is_in { + 0 + } else { + trb_flag::DIR_IN + }; + dev.ep0.enqueue(Trb { + param: 0, + status: 0, + ctrl: trb_type::STATUS_STAGE | trb_flag::IOC | status_dir, + })?; + + self.ring_doorbell(dev.slot, EP0_DCI)?; + self.wait_transfer(MAX_CTRL_EVENTS) + } } // --------------------------------------------------------------------------- @@ -163,6 +328,9 @@ struct Controller { core: HcCore, dcbaa: Dcbaa, max_slots: u8, + max_ports: u8, + /// Context size in bytes: 32 (CSZ=0) or 64 (CSZ=1). + ctx_size: usize, max_scratchpad: usize, /// Scratchpad pointer array and pages; owned by the controller for its lifetime. _scratchpad: Vec>, @@ -181,13 +349,18 @@ impl Controller { usize::try_from(cap.read32(regs::cap::RTSOFF)?).map_err(|_| XhciErr::Overflow)?; // HCSPARAMS1: MaxSlots[7:0], MaxIntrs[18:8], MaxPorts[31:24]. - let [max_slots, ..] = hcsparams1.to_le_bytes(); + let [max_slots, _, _, max_ports] = hcsparams1.to_le_bytes(); // HCSPARAMS2 scratchpad count (§5.3.4): Hi = bits[25:21], Lo = bits[31:27]; // total = (Hi << 5) | Lo. (Reversing Hi/Lo is a common mistake.) let sp_hi = (hcsparams2 >> 21) & 0x1f; let sp_lo = (hcsparams2 >> 27) & 0x1f; let max_scratchpad = usize::try_from((sp_hi << 5) | sp_lo).map_err(|_| XhciErr::Overflow)?; + let ctx_size: usize = if hccparams1 & regs::cap::HCC_CSZ != 0 { + 64 + } else { + 32 + }; log::info!( "xHCI: {name}: HCIVERSION={hci_version:#06x} CAPLENGTH={caplength:#04x} \ @@ -208,6 +381,8 @@ impl Controller { core, dcbaa: Dcbaa::new(NUMA_NODE)?, max_slots, + max_ports, + ctx_size, max_scratchpad, _scratchpad: Vec::new(), }) @@ -332,12 +507,74 @@ impl Controller { Ok(()) } + // ------------------------------------------------------------------------- + // Ports + // ------------------------------------------------------------------------- + + /// PORTSC offset for 1-based port number `port`. + fn portsc_offset(&self, port: u8) -> Result { + if port == 0 || port > self.max_ports { + return Err(XhciErr::InvalidPort); + } + usize::from(port) + .checked_sub(1) + .and_then(|p| p.checked_mul(regs::port::STRIDE)) + .and_then(|o| o.checked_add(regs::port::BASE)) + .ok_or(XhciErr::Overflow) + } + + fn read_portsc(&self, port: u8) -> Result { + Ok(self.core.op.read32(self.portsc_offset(port)?)?) + } + + fn write_portsc(&self, port: u8, val: u32) -> Result<(), XhciErr> { + Ok(self.core.op.write32(self.portsc_offset(port)?, val)?) + } + + /// Reset `port` and wait for Port Reset Change (RW1CS bits handled per spec). + fn reset_port(&mut self, port: u8) -> Result<(), XhciErr> { + // Pre-clear stale change bits (write 1) while setting PR; keep PP, avoid PED. + let portsc = self.read_portsc(port)?; + self.write_portsc( + port, + (portsc & !regs::port::PED) | (portsc & regs::port::CHANGE_BITS) | regs::port::PR, + )?; + self.core.op.poll32( + self.portsc_offset(port)?, + regs::port::PRC, + regs::port::PRC, + COMPLETION_POLL_ITERS, + )?; + // Clear PRC (and any other change bits) by writing 1 to them. + let portsc = self.read_portsc(port)?; + self.write_portsc( + port, + (portsc & !(regs::port::PED | regs::port::PR)) | (portsc & regs::port::CHANGE_BITS), + )?; + self.core.drain_events() + } + // ------------------------------------------------------------------------- // Enumeration // ------------------------------------------------------------------------- - /// Check that the command ring works before touching any port. + /// Scan all ports and enumerate every connected device. fn boot_enumerate(&mut self) -> Result<(), XhciErr> { + // xHCI §4.19.1.1: with a port power controller, software must set PP=1; + // PR=1 on an unpowered port is silently ignored. + for port in 1..=self.max_ports { + let portsc = self.read_portsc(port)?; + if portsc & regs::port::PP == 0 { + self.write_portsc( + port, + (portsc & !(regs::port::PED | regs::port::PR)) + | (portsc & regs::port::CHANGE_BITS) + | regs::port::PP, + )?; + log::info!("xHCI: {}: port {port} PP was 0, powered on", self.name); + } + } + // If a No-Op times out, every later command would too. self.core .command(Trb { @@ -350,10 +587,186 @@ impl Controller { "xHCI: {}: NOOP command OK — command ring is alive", self.name ); + + awkernel_lib::delay::wait_microsec(PORT_SETTLE_USEC); + self.core.drain_events()?; + + // Snapshot connected ports first: resetting one port must not change + // which ports we visit. + let mut connected = Vec::new(); + for port in 1..=self.max_ports { + if self.read_portsc(port)? & regs::port::CCS != 0 { + connected.push(port); + } + } + if connected.is_empty() { + log::info!("xHCI: {}: no device found at boot", self.name); + return Ok(()); + } + + for port in connected { + log::info!("xHCI: {}: device detected on port {port}", self.name); + if let Err(e) = self.enumerate_port(port) { + log::error!("xHCI: {}: port {port} enumeration failed: {e:?}", self.name); + } + // The device is leaked (see `UsbDevice`). Stale transfer events + // would corrupt the next port's enumeration. + self.core.drain_events()?; + } + Ok(()) + } + + /// Reset `port`, enable a slot, address the device and read its Device Descriptor. + fn enumerate_port(&mut self, port: u8) -> Result, XhciErr> { + self.reset_port(port)?; + + let slot = self + .core + .command(Trb { + param: 0, + status: 0, + ctrl: trb_type::ENABLE_SLOT, + })? + .slot_id(); + if slot == 0 || slot > self.max_slots { + return Err(XhciErr::InvalidSlot); + } + log::info!("xHCI: {}: slot {slot} assigned for port {port}", self.name); + + let speed = (self.read_portsc(port)? & regs::port::SPEED_MASK) >> PORTSC_SPEED_SHIFT; + let ctx = new_page()?; + self.dcbaa.set(slot, phys_of(&ctx)?)?; + let mut dev = ManuallyDrop::new(UsbDevice { + slot, + port, + speed, + ep0: TransferRing::new(NUMA_NODE)?, + _ctx: ctx, + ctrl_buf: new_page()?, + desc: DeviceDescriptor::default(), + }); + + self.address_device(&mut dev)?; + log::info!("xHCI: {}: slot {slot} addressed", self.name); + self.read_device_descriptor(&mut dev)?; + Ok(dev) + } + + /// Issue Address Device (BSR=0, so the controller sends SET_ADDRESS). + fn address_device(&mut self, dev: &mut UsbDevice) -> Result<(), XhciErr> { + let ctx = self.ctx_size; + // EP0 max packet size by speed. Full Speed devices may use 8..=64; 64 + // avoids Babble Detected on FS devices whose real MaxPacketSize0 is 64 + // (e.g. PL2303HX/HXN) and is accepted for smaller ones as well. + let max_pkt: u16 = match dev.speed { + SPEED_LOW => 8, + SPEED_HIGH => 64, + SPEED_SUPER | SPEED_SUPER_PLUS => 512, + _ => 64, + }; + let ep0_phys = dev.ep0.phys_base()?; + + let mut input = ManuallyDrop::new(new_page()?); + { + let page = input.as_mut(); + // Input Control Context: Add A0 (slot) + A1 (EP0). + ctx_write32(page, 4, 0b11)?; + // Slot Context: Speed[23:20], Context Entries[31:27] = 1, Root Hub Port[23:16]. + let speed = u8::try_from(dev.speed).map_err(|_| XhciErr::Overflow)?; + ctx_write32(page, ctx, u32::from_le_bytes([0, 0, speed << 4, 1 << 3]))?; + ctx_write32( + page, + ctx.checked_add(4).ok_or(XhciErr::Overflow)?, + u32::from_le_bytes([0, 0, dev.port, 0]), + )?; + // EP0 Context (DCI 1). + let ep = ctx.checked_mul(2).ok_or(XhciErr::Overflow)?; + write_ep_context(page, ep, EP_CTX_CONTROL, max_pkt, ep0_phys, 8)?; + } + + let result = self.core.command(Trb { + param: phys_of(&input)?, + status: 0, + ctrl: trb_type::ADDRESS_DEVICE | ctrl_slot_ep(dev.slot, 0), + }); + release_unless_timeout(input, &result); + result.map(|_| ()) + } + + /// GET_DESCRIPTOR(Device) and cache the fields used later. + fn read_device_descriptor(&mut self, dev: &mut UsbDevice) -> Result<(), XhciErr> { + self.core.control( + dev, + SetupPacket { + request_type: RT_DEV_IN, + request: REQ_GET_DESCRIPTOR, + value: DESC_DEVICE, + index: 0, + length: DEVICE_DESC_LEN, + }, + )?; + let [_, _, usb_lo, usb_hi, class, sub, proto, _, v_lo, v_hi, p_lo, p_hi, d_lo, d_hi, ..] = + read_ctrl_buf::<18>(dev)?; + dev.desc = DeviceDescriptor { + vendor: u16::from_le_bytes([v_lo, v_hi]), + product: u16::from_le_bytes([p_lo, p_hi]), + bcd_device: u16::from_le_bytes([d_lo, d_hi]), + }; + log::info!( + "xHCI: {}: slot {} — USB {:x}.{:02x} VID={:#06x} PID={:#06x} \ + class={class:#04x}/{sub:#04x}/{proto:#04x} bcdDevice={:#06x}", + self.name, + dev.slot, + usb_hi, + usb_lo, + dev.desc.vendor, + dev.desc.product, + dev.desc.bcd_device, + ); Ok(()) } } +/// Write one Endpoint Context (DW1..DW4) at byte offset `off` of an Input Context. +fn write_ep_context( + page: &mut Page, + off: usize, + ep_type: u8, + max_pkt: u16, + ring_phys: u64, + avg_trb_len: u32, +) -> Result<(), XhciErr> { + let at = |delta: usize| off.checked_add(delta).ok_or(XhciErr::Overflow); + let [mp_lo, mp_hi] = max_pkt.to_le_bytes(); + // DW1: CErr / EP Type in the low byte, Max Packet Size in [31:16]. + ctx_write32(page, at(4)?, u32::from_le_bytes([ep_type, 0, mp_lo, mp_hi]))?; + // DW2/DW3: TR Dequeue Pointer with DCS = 1 (matches the ring's initial cycle). + ctx_write32(page, at(8)?, lo32(ring_phys) | trb_flag::CYCLE)?; + ctx_write32(page, at(12)?, hi32(ring_phys))?; + // DW4: Average TRB Length. + ctx_write32(page, at(16)?, avg_trb_len) +} + +/// Free a transient Input Context unless the command timed out (in which case +/// the controller may still read it later). +fn release_unless_timeout(buf: ManuallyDrop>, result: &Result) { + if !matches!(result, Err(XhciErr::Timeout)) { + drop(ManuallyDrop::into_inner(buf)); + } +} + +/// Copy the first `N` bytes of `dev.ctrl_buf` written by the controller. +fn read_ctrl_buf(dev: &UsbDevice) -> Result<[u8; N], XhciErr> { + let bytes = dev + .ctrl_buf + .as_ref() + .first_chunk::() + .ok_or(XhciErr::Overflow)?; + // SAFETY: `bytes` points into the live DMA page; volatile because the + // controller wrote it via DMA. + Ok(unsafe { read_volatile(bytes) }) +} + // --------------------------------------------------------------------------- // PCIe device // --------------------------------------------------------------------------- diff --git a/awkernel_drivers/src/pcie/usb/xhci/regs.rs b/awkernel_drivers/src/pcie/usb/xhci/regs.rs index eab2b480f..ea4527594 100644 --- a/awkernel_drivers/src/pcie/usb/xhci/regs.rs +++ b/awkernel_drivers/src/pcie/usb/xhci/regs.rs @@ -7,6 +7,9 @@ pub mod cap { pub const HCCPARAMS1: usize = 0x10; // u32 — AC64, CSZ, xECP pointer, … pub const DBOFF: usize = 0x14; // u32 — Doorbell Array offset from BAR[0] pub const RTSOFF: usize = 0x18; // u32 — Runtime Registers offset from BAR[0] + + /// HCCPARAMS1.CSZ: 0 = 32-byte contexts, 1 = 64-byte contexts. + pub const HCC_CSZ: u32 = 1 << 2; } /// Operational Register offsets — relative to `op_base` (= BAR[0] + CAPLENGTH). xHCI spec §5.4. @@ -58,8 +61,14 @@ pub const TRB_TYPE_MASK: u32 = 0x3f << TRB_TYPE_SHIFT; pub mod trb_type { use super::TRB_TYPE_SHIFT; + pub const SETUP_STAGE: u32 = 2 << TRB_TYPE_SHIFT; + pub const DATA_STAGE: u32 = 3 << TRB_TYPE_SHIFT; + pub const STATUS_STAGE: u32 = 4 << TRB_TYPE_SHIFT; pub const LINK: u32 = 6 << TRB_TYPE_SHIFT; + pub const ENABLE_SLOT: u32 = 9 << TRB_TYPE_SHIFT; + pub const ADDRESS_DEVICE: u32 = 11 << TRB_TYPE_SHIFT; pub const NOOP_CMD: u32 = 23 << TRB_TYPE_SHIFT; + pub const TRANSFER_EVENT: u32 = 32 << TRB_TYPE_SHIFT; pub const CMD_COMPLETION: u32 = 33 << TRB_TYPE_SHIFT; } @@ -69,9 +78,45 @@ pub mod trb_flag { pub const CYCLE: u32 = 1 << 0; /// Link TRB: Toggle Cycle — flips the Ring Cycle State when traversed. pub const LINK_TC: u32 = 1 << 1; + /// Interrupt On Completion. + pub const IOC: u32 = 1 << 5; + /// Setup Stage TRB: Immediate Data (setup packet is inside the TRB). + pub const IDT: u32 = 1 << 6; + /// Data/Status Stage TRB: direction IN. + pub const DIR_IN: u32 = 1 << 16; + /// Setup Stage TRB: Transfer Type = No Data Stage. + pub const TRT_NO_DATA: u32 = 0 << 16; + /// Setup Stage TRB: Transfer Type = OUT Data Stage. + pub const TRT_OUT: u32 = 2 << 16; + /// Setup Stage TRB: Transfer Type = IN Data Stage. + pub const TRT_IN: u32 = 3 << 16; } /// Completion codes reported in event TRBs (status[31:24]). xHCI spec §6.4.5. pub mod completion { pub const SUCCESS: u8 = 1; + pub const SHORT_PACKET: u8 = 13; +} + +/// Port Status and Control Register offsets — relative to op_base. xHCI spec §5.4.8. +/// Port n (1-based) is at BASE + (n-1) * STRIDE. +pub mod port { + pub const BASE: usize = 0x400; + pub const STRIDE: usize = 0x10; + + pub const CCS: u32 = 1 << 0; // Current Connect Status (RO) + pub const PED: u32 = 1 << 1; // Port Enabled/Disabled (RW1C — write 1 to disable) + pub const PR: u32 = 1 << 4; // Port Reset (RWS) + pub const PP: u32 = 1 << 9; // Port Power (RWS) + pub const SPEED_MASK: u32 = 0xf << 10; // Port Speed (RO) — PORTSC[13:10] + pub const CSC: u32 = 1 << 17; // Connect Status Change (RW1CS) + pub const PEC: u32 = 1 << 18; // Port Enabled/Disabled Change (RW1CS) + pub const WRC: u32 = 1 << 19; // Warm Port Reset Change (RW1CS) + pub const OCC: u32 = 1 << 20; // Over-current Change (RW1CS) + pub const PRC: u32 = 1 << 21; // Port Reset Change (RW1CS) + pub const PLC: u32 = 1 << 22; // Port Link State Change (RW1CS) + pub const CEC: u32 = 1 << 23; // Port Config Error Change (RW1CS) + + /// All RW1CS change bits. + pub const CHANGE_BITS: u32 = CSC | PEC | WRC | OCC | PRC | PLC | CEC; } diff --git a/awkernel_drivers/src/pcie/usb/xhci/ring.rs b/awkernel_drivers/src/pcie/usb/xhci/ring.rs index 2f0d4a765..6e9c48a45 100644 --- a/awkernel_drivers/src/pcie/usb/xhci/ring.rs +++ b/awkernel_drivers/src/pcie/usb/xhci/ring.rs @@ -57,6 +57,12 @@ impl Trb { let [_, _, _, code] = self.status.to_le_bytes(); code } + + /// Slot ID of a Command Completion event (ctrl[31:24]). + pub fn slot_id(&self) -> u8 { + let [_, _, _, slot] = self.ctrl.to_le_bytes(); + slot + } } type RingMem = [Trb; RING_SIZE]; @@ -90,6 +96,7 @@ pub struct ProducerRing { } pub type CommandRing = ProducerRing; +pub type TransferRing = ProducerRing; impl ProducerRing { /// Allocate a zeroed ring with its Link TRB pointing back to the start. From 6264200444e1b2959c02a29a4842e7931ac33476 Mon Sep 17 00:00:00 2001 From: nokosaaan Date: Tue, 20 Oct 2026 18:16:34 +0900 Subject: [PATCH 2/3] add: Makefile argument Signed-off-by: nokosaaan --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 28a00f6b5..96539cb02 100644 --- a/Makefile +++ b/Makefile @@ -162,6 +162,7 @@ QEMU_X86_ARGS+= -drive if=pflash,format=raw,file=${OVMF_PATH}/vars_qemu.fd QEMU_X86_ARGS+= -drive format=raw,file=x86_64_uefi.img QEMU_X86_ARGS+= -machine q35 QEMU_X86_ARGS+= -device qemu-xhci,id=xhci +QEMU_X86_ARGS+= -device usb-kbd,bus=xhci.0 QEMU_X86_ARGS+= -serial stdio -smp 4 -monitor telnet::5556,server,nowait QEMU_X86_ARGS+= -m 4G -smp cpus=16 QEMU_X86_ARGS+= -object memory-backend-ram,size=1G,id=m0 From bb0357fabab0340c03e9f8d859f2460b16ad48ad Mon Sep 17 00:00:00 2001 From: nokosaaan Date: Fri, 23 Oct 2026 17:18:29 +0900 Subject: [PATCH 3/3] fix(usb): match xHCI completion events by TRB address and recover on timeout --- awkernel_drivers/src/pcie/usb/xhci/mod.rs | 226 ++++++++++++++++++--- awkernel_drivers/src/pcie/usb/xhci/regs.rs | 3 + awkernel_drivers/src/pcie/usb/xhci/ring.rs | 53 ++++- 3 files changed, 248 insertions(+), 34 deletions(-) diff --git a/awkernel_drivers/src/pcie/usb/xhci/mod.rs b/awkernel_drivers/src/pcie/usb/xhci/mod.rs index 62c279c8c..4bfea6f3d 100644 --- a/awkernel_drivers/src/pcie/usb/xhci/mod.rs +++ b/awkernel_drivers/src/pcie/usb/xhci/mod.rs @@ -37,6 +37,9 @@ const COMPLETION_POLL_ITERS: u32 = 2_000_000; const MAX_CTRL_EVENTS: usize = 3; /// VBUS stabilization / device connect wait before scanning ports. const PORT_SETTLE_USEC: u64 = 500_000; +/// USB 2.0 spec §7.1.7.3 TRSTRCY: minimum 10 ms reset recovery before the +/// device is required to answer SET_ADDRESS. +const PORT_RESET_RECOVERY_USEC: u64 = 10_000; /// All DMA allocations go to NUMA node 0. const NUMA_NODE: usize = 0; @@ -211,6 +214,8 @@ struct HcCore { db: Mmio, cmd_ring: CommandRing, evt_ring: EventRing, + /// Set if Command Abort fails (`CRCR.CRR` stays 1); refuses further commands. + cmd_ring_broken: bool, } impl HcCore { @@ -228,14 +233,15 @@ impl HcCore { Ok(self.ir0.write32(regs::ir::ERDP_HI, hi32(erdp))?) } - /// Wait for the next event of type `kind`, discarding other events. + /// Wait for the next event for which `accept` returns true, discarding + /// every other event. /// /// WCET: at most `COMPLETION_POLL_ITERS` event-ring polls. - fn wait_event(&mut self, kind: u32) -> Result { + fn wait_event(&mut self, mut accept: impl FnMut(&Trb) -> bool) -> Result { for _ in 0..COMPLETION_POLL_ITERS { if let Some(trb) = self.evt_ring.dequeue()? { self.update_erdp()?; - if trb.trb_type() == kind { + if accept(&trb) { return Ok(trb); } } @@ -255,25 +261,148 @@ impl HcCore { Ok(()) } + /// Stop the Command Ring after a command timeout and re-synchronize it at + /// the ring base (xHCI spec §4.6.1.2, §5.4.5). + /// + /// Writing `CRCR_CA` aborts the running command (if any) and stops the + /// Command Ring; the controller clears `CRCR_CRR` once any in-flight command + /// DMA and its `Command Aborted` / `Command Ring Stopped` events have been + /// posted. Once `CRR == 0`, the ring is reset to index 0 and `CRCR` is + /// reprogrammed. + /// + /// WCET: at most `STATE_POLL_ITERS` polls of `CRCR_LO` plus `RING_SIZE` + /// TRB writes on reset. + fn abort_command_ring(&mut self) -> Result<(), XhciErr> { + // CRCR[63:6] and RCS/CS/CA read as 0 (§5.4.5); write CA alone. + self.op.write64( + regs::op::CRCR_LO, + regs::op::CRCR_HI, + u64::from(regs::op::CRCR_CA), + )?; + + let mut stopped = false; + for _ in 0..STATE_POLL_ITERS { + if self.evt_ring.dequeue()?.is_some() { + self.update_erdp()?; + } + if self.op.read32(regs::op::CRCR_LO)? & regs::op::CRCR_CRR == 0 { + stopped = true; + break; + } + core::hint::spin_loop(); + } + if !stopped { + self.cmd_ring_broken = true; + log::error!("xHCI: command ring did not stop after Command Abort (CRR=1)"); + return Err(XhciErr::Timeout); + } + + self.drain_events()?; + self.cmd_ring.reset()?; + let crcr = self.cmd_ring.phys_base()?; + publish(); + self.op.write64( + regs::op::CRCR_LO, + regs::op::CRCR_HI, + crcr | u64::from(regs::op::CRCR_RCS), + )?; + Ok(()) + } + /// Submit a command and wait for its Command Completion event. + /// + /// The event is matched against the Command TRB it reports (event.param = + /// Command TRB Pointer, xHCI spec §6.4.2.2). A completion for any other + /// TRB — e.g. one that arrives after an earlier command timed out — is + /// discarded, so it cannot be taken as this command's completion. On + /// timeout, the Command Ring is aborted and reset before returning. fn command(&mut self, trb: Trb) -> Result { - self.cmd_ring.enqueue(trb)?; + if self.cmd_ring_broken { + return Err(XhciErr::Timeout); + } + let cmd_phys = self.cmd_ring.enqueue(trb)?; self.ring_doorbell(0, 0)?; - let event = self.wait_event(trb_type::CMD_COMPLETION)?; + let event = match self.wait_event(|ev| { + if ev.trb_type() != trb_type::CMD_COMPLETION { + return false; + } + if ev.param != cmd_phys { + log::warn!( + "xHCI: discarded stale completion for command TRB {:#x} \ + (code {}, slot {}) while waiting for {cmd_phys:#x}", + ev.param, + ev.completion_code(), + ev.slot_id() + ); + return false; + } + true + }) { + Ok(ev) => ev, + Err(XhciErr::Timeout) => { + log::warn!("xHCI: command TRB {cmd_phys:#x} timed out; aborting command ring"); + if let Err(e) = self.abort_command_ring() { + self.cmd_ring_broken = true; + log::error!("xHCI: command ring abort failed: {e:?}"); + } + return Err(XhciErr::Timeout); + } + Err(e) => return Err(e), + }; match event.completion_code() { completion::SUCCESS => Ok(event), code => Err(XhciErr::Completion(code)), } } - /// Wait for the Status Stage of a control transfer (Short Packet on the - /// Data Stage is skipped). - fn wait_transfer(&mut self, max_events: usize) -> Result<(), XhciErr> { - for _ in 0..max_events { - match self.wait_event(trb_type::TRANSFER_EVENT)?.completion_code() { - completion::SUCCESS => return Ok(()), - completion::SHORT_PACKET => continue, - code => return Err(XhciErr::Completion(code)), + /// Wait for the Status Stage of a control transfer on (`slot`, `dci`) + /// (Short Packet on the Data Stage is skipped). + /// + /// Events are matched against (`slot`, `dci`) and the TRB physical addresses + /// of this transfer (`setup_phys`, `data_phys`, `status_phys`), so stale + /// events from earlier transfers or other endpoints are discarded. + fn wait_transfer( + &mut self, + slot: u8, + dci: u8, + setup_phys: u64, + data_phys: Option, + status_phys: u64, + ) -> Result<(), XhciErr> { + for _ in 0..MAX_CTRL_EVENTS { + let event = self.wait_event(|ev| { + if ev.trb_type() != trb_type::TRANSFER_EVENT { + return false; + } + let in_td = ev.param == status_phys + || ev.param == setup_phys + || data_phys == Some(ev.param); + if ev.slot_id() != slot || ev.endpoint_id() != dci || !in_td { + log::warn!( + "xHCI: discarded stale transfer event (slot {}, ep {}, trb {:#x}, code {}) \ + while waiting for slot {slot} ep {dci}", + ev.slot_id(), + ev.endpoint_id(), + ev.param, + ev.completion_code() + ); + return false; + } + true + })?; + let code = event.completion_code(); + if event.param == status_phys { + return match code { + completion::SUCCESS | completion::SHORT_PACKET => Ok(()), + _ => Err(XhciErr::Completion(code)), + }; + } + // Event from Setup or Data stage: Short Packet / Success on Data + // Stage is followed by the Status Stage event; any error halts EP0 + // immediately without executing Status Stage. + match code { + completion::SHORT_PACKET | completion::SUCCESS => continue, + _ => return Err(XhciErr::Completion(code)), } } Err(XhciErr::Completion(completion::SHORT_PACKET)) @@ -290,32 +419,34 @@ impl HcCore { (true, false) => trb_flag::TRT_OUT, }; - dev.ep0.enqueue(Trb { + let setup_phys = dev.ep0.enqueue(Trb { param: setup.to_param(), status: 8, // setup packet size ctrl: trb_type::SETUP_STAGE | trb_flag::IDT | trt, })?; - if has_data { - dev.ep0.enqueue(Trb { + let data_phys = if has_data { + Some(dev.ep0.enqueue(Trb { param: phys_of(&dev.ctrl_buf)?, status: u32::from(setup.length), ctrl: trb_type::DATA_STAGE | if is_in { trb_flag::DIR_IN } else { 0 }, - })?; - } + })?) + } else { + None + }; // Status Stage runs opposite to the data stage (IN when there is none). let status_dir = if has_data && is_in { 0 } else { trb_flag::DIR_IN }; - dev.ep0.enqueue(Trb { + let status_phys = dev.ep0.enqueue(Trb { param: 0, status: 0, ctrl: trb_type::STATUS_STAGE | trb_flag::IOC | status_dir, })?; self.ring_doorbell(dev.slot, EP0_DCI)?; - self.wait_transfer(MAX_CTRL_EVENTS) + self.wait_transfer(dev.slot, EP0_DCI, setup_phys, data_phys, status_phys) } } @@ -374,6 +505,7 @@ impl Controller { db: cap.window(dboff)?, cmd_ring: CommandRing::new(NUMA_NODE)?, evt_ring: EventRing::new(NUMA_NODE)?, + cmd_ring_broken: false, }; Ok(Self { @@ -551,6 +683,12 @@ impl Controller { port, (portsc & !(regs::port::PED | regs::port::PR)) | (portsc & regs::port::CHANGE_BITS), )?; + // A successful reset leaves the port Enabled (PED=1, §4.3.2). + if self.read_portsc(port)? & regs::port::PED == 0 { + return Err(XhciErr::InvalidPort); + } + // USB 2.0 §7.1.7.3 TRSTRCY: allow the device to recover before SET_ADDRESS. + awkernel_lib::delay::wait_microsec(PORT_RESET_RECOVERY_USEC); self.core.drain_events() } @@ -608,6 +746,13 @@ impl Controller { log::info!("xHCI: {}: device detected on port {port}", self.name); if let Err(e) = self.enumerate_port(port) { log::error!("xHCI: {}: port {port} enumeration failed: {e:?}", self.name); + if self.core.cmd_ring_broken { + log::error!( + "xHCI: {}: command ring is unresponsive; aborting enumeration", + self.name + ); + return Err(e); + } } // The device is leaked (see `UsbDevice`). Stale transfer events // would corrupt the next port's enumeration. @@ -646,12 +791,47 @@ impl Controller { desc: DeviceDescriptor::default(), }); - self.address_device(&mut dev)?; - log::info!("xHCI: {}: slot {slot} addressed", self.name); - self.read_device_descriptor(&mut dev)?; + if let Err(e) = self.address_and_read_descriptor(&mut dev) { + self.cleanup_failed_slot(dev); + return Err(e); + } Ok(dev) } + fn address_and_read_descriptor(&mut self, dev: &mut UsbDevice) -> Result<(), XhciErr> { + self.address_device(dev)?; + log::info!("xHCI: {}: slot {} addressed", self.name, dev.slot); + self.read_device_descriptor(dev) + } + + /// Disable `dev.slot` after a failed enumeration attempt so the controller + /// stops any pending EP0 transfer and releases `DCBAA[slot]` (xHCI spec §4.6.4). + /// + /// `dev`'s DMA pages (`_ctx`, `ep0`, `ctrl_buf`) and `DCBAA[slot]` are only + /// reclaimed if `DISABLE_SLOT` completes; otherwise they remain leaked so + /// the controller cannot DMA into freed memory. + fn cleanup_failed_slot(&mut self, dev: ManuallyDrop) { + let slot = dev.slot; + match self.core.command(Trb { + param: 0, + status: 0, + ctrl: trb_type::DISABLE_SLOT | ctrl_slot_ep(slot, 0), + }) { + Ok(_) => { + if self.dcbaa.set(slot, 0).is_ok() { + drop(ManuallyDrop::into_inner(dev)); + } + log::info!("xHCI: {}: slot {slot} disabled after error", self.name); + } + Err(e) => { + log::warn!( + "xHCI: {}: DISABLE_SLOT {slot} failed ({e:?}); leaking slot DMA pages", + self.name + ); + } + } + } + /// Issue Address Device (BSR=0, so the controller sends SET_ADDRESS). fn address_device(&mut self, dev: &mut UsbDevice) -> Result<(), XhciErr> { let ctx = self.ctx_size; diff --git a/awkernel_drivers/src/pcie/usb/xhci/regs.rs b/awkernel_drivers/src/pcie/usb/xhci/regs.rs index ea4527594..f8cde130d 100644 --- a/awkernel_drivers/src/pcie/usb/xhci/regs.rs +++ b/awkernel_drivers/src/pcie/usb/xhci/regs.rs @@ -34,6 +34,8 @@ pub mod op { // CRCR bits (written to low word; high bits are ring base address) pub const CRCR_RCS: u32 = 1 << 0; // Ring Cycle State + pub const CRCR_CA: u32 = 1 << 2; // Command Abort (RW1S) + pub const CRCR_CRR: u32 = 1 << 3; // Command Ring Running (RO) } /// Interrupter Register Set offsets — relative to `ir0_base` (= rt_base + BASE). xHCI spec §5.5.2. @@ -66,6 +68,7 @@ pub mod trb_type { pub const STATUS_STAGE: u32 = 4 << TRB_TYPE_SHIFT; pub const LINK: u32 = 6 << TRB_TYPE_SHIFT; pub const ENABLE_SLOT: u32 = 9 << TRB_TYPE_SHIFT; + pub const DISABLE_SLOT: u32 = 10 << TRB_TYPE_SHIFT; pub const ADDRESS_DEVICE: u32 = 11 << TRB_TYPE_SHIFT; pub const NOOP_CMD: u32 = 23 << TRB_TYPE_SHIFT; pub const TRANSFER_EVENT: u32 = 32 << TRB_TYPE_SHIFT; diff --git a/awkernel_drivers/src/pcie/usb/xhci/ring.rs b/awkernel_drivers/src/pcie/usb/xhci/ring.rs index 6e9c48a45..6fd8359ec 100644 --- a/awkernel_drivers/src/pcie/usb/xhci/ring.rs +++ b/awkernel_drivers/src/pcie/usb/xhci/ring.rs @@ -63,6 +63,12 @@ impl Trb { let [_, _, _, slot] = self.ctrl.to_le_bytes(); slot } + + /// Endpoint ID (DCI) of a Transfer Event (ctrl[20:16]). + pub fn endpoint_id(&self) -> u8 { + let [_, _, ep, _] = self.ctrl.to_le_bytes(); + ep & 0x1f + } } type RingMem = [Trb; RING_SIZE]; @@ -101,14 +107,31 @@ pub type TransferRing = ProducerRing; impl ProducerRing { /// Allocate a zeroed ring with its Link TRB pointing back to the start. pub fn new(numa_id: usize) -> Result { - let mut mem = DMAPool::::new(numa_id, 1).ok_or(XhciErr::DmaAlloc)?; - let base = phys_of(&mem)?; + let mem = DMAPool::::new(numa_id, 1).ok_or(XhciErr::DmaAlloc)?; + let mut ring = Self { + mem, + enqueue_idx: 0, + cycle_bit: trb_flag::CYCLE, + }; + ring.reset()?; + Ok(ring) + } - for slot in mem.as_mut().iter_mut() { + /// Clear all TRBs, reinstall the Link TRB and rewind to index 0 (cycle = 1). + /// + /// Caller must ensure the controller is not actively fetching from this ring + /// (e.g. `CRCR.CRR == 0` for the Command Ring). + pub fn reset(&mut self) -> Result<(), XhciErr> { + let base = self.phys_base()?; + for slot in self.mem.as_mut().iter_mut() { // SAFETY: `slot` is a valid, aligned &mut Trb inside the DMA page. unsafe { write_volatile(slot, Trb::default()) }; } - let link = mem.as_mut().get_mut(LINK_IDX).ok_or(XhciErr::Overflow)?; + let link = self + .mem + .as_mut() + .get_mut(LINK_IDX) + .ok_or(XhciErr::Overflow)?; let link_trb = Trb { param: base, status: 0, @@ -118,21 +141,29 @@ impl ProducerRing { // SAFETY: `link` is a valid, aligned &mut Trb inside the DMA page. unsafe { write_volatile(link, link_trb) }; - Ok(Self { - mem, - enqueue_idx: 0, - cycle_bit: trb_flag::CYCLE, - }) + self.enqueue_idx = 0; + self.cycle_bit = trb_flag::CYCLE; + Ok(()) } /// Write `trb` at the enqueue position, stamping the Producer Cycle State. /// Wraps through the Link TRB when the last data slot is used. /// /// The caller must call [`publish`] and then ring the doorbell. - pub fn enqueue(&mut self, mut trb: Trb) -> Result<(), XhciErr> { + /// + /// Returns the physical address of the written TRB. The controller reports + /// this address in the event it generates for the TRB (Command Completion + /// Event / Transfer Event, xHCI spec §6.4.2), so callers can match the event + /// against the TRB that produced it. + pub fn enqueue(&mut self, mut trb: Trb) -> Result { let idx = self.enqueue_idx; let pcs = self.cycle_bit; trb.ctrl = (trb.ctrl & !trb_flag::CYCLE) | pcs; + let trb_phys = u64::try_from(idx) + .ok() + .and_then(|i| i.checked_mul(TRB_BYTES)) + .and_then(|off| self.phys_base().ok()?.checked_add(off)) + .ok_or(XhciErr::Overflow)?; let slot = self.mem.as_mut().get_mut(idx).ok_or(XhciErr::RingFull)?; // SAFETY: `slot` is a valid, aligned &mut Trb inside the DMA page. @@ -157,7 +188,7 @@ impl ProducerRing { } else { self.enqueue_idx = next; } - Ok(()) + Ok(trb_phys) } /// Physical base address of the ring (CRCR / EP Context TR Dequeue Pointer).