iris-hostgl: run command buffers on a GL thread of their own

CHANGELOG.md

@@ -135,6 +135,22 @@ hashes are given where a change is easiest to understand by reading the commit.
   `nvram` ones, so both chips get a real MAC regardless of machine profile.
 ### Graphics and host OpenGL
 
+- **Host GL no longer stops the machine while it draws.** A host call is
+  answered on the emulated CPU's thread, so while host GL replayed a
+  command buffer (2-5 ms for a big one) no instruction ran and no interrupt
+  was taken: IRIX acknowledged its 1 kHz clock late ~15 times a second, its
+  audio driver refilled HAL2's ring late, and sound crackled under any busy
+  GL program. The service now runs on a thread of its own (`threaded.rs`).
+  A new op, `OP_BATCH_ASYNC`, queues a self-contained buffer and answers at
+  once; a call from a program with buffers still queued is answered with
+  the new `ENOTYET` (0x3001) and the library yields and asks again, so IRIX
+  keeps running. Against this tree without it (IP28, Max IMPACT, macOS): a
+  GL demo with synthesized sound went from 19.7 to 3.7 dropouts a second
+  (31 to 4 ms of sound lost a second) and Quake 3's timedemo from 30.4 to
+  33.9 fps; boot, idle CPU, sound without GL and SGI's own libGL on the
+  emulated board are unchanged. Libraries without `OP_BATCH_ASYNC` work
+  unchanged. `IRIS_HOSTGL_INLINE=1` keeps the service on the CPU thread.
+  See `rules/hostgl/host-calls-hold-the-cpu-thread.md`.
 - **Maximum Impact draws correctly, on two raster engines.** Its two RE4s
   each own every other scanline in their own 12MB, so a buffer takes half
   the pages per engine. The model assumed one engine; buffers overlapped,

FEATURES.md

@@ -417,6 +417,7 @@ and offline-tool controls are included so they are not mistaken for GUI settings
 | `IRIS_GUI_GL` | 1 selects the GUI GL capture path. | [iris-gui/src/framebuffer.rs](iris-gui/src/framebuffer.rs) |
 | `IRIS_HAL2_CAPTURE` | HAL2 host audio capture output path. | [src/dev/hal2.rs](src/dev/hal2.rs) |
 | `IRIS_HOSTGL_CALLS` | Substring filter for HostGL call logging (bounded log). | [iris-hostgl/src/service.rs](iris-hostgl/src/service.rs) |
+| `IRIS_HOSTGL_INLINE` | 1 runs the HostGL service on the CPU thread instead of its own (for comparison; the CPU stalls while a buffer replays). | [iris-hostgl/src/lib.rs](iris-hostgl/src/lib.rs) |
 | `IRIS_HOSTGL_NOSURFACE` | 1 disables CGL IOSurface presentation. | [iris-hostgl/src/backend/cgl.rs](iris-hostgl/src/backend/cgl.rs) |
 | `IRIS_HOSTGL_QOS` | default disables HostGL thread QoS adjustment. | [iris-hostgl/src/qos.rs](iris-hostgl/src/qos.rs) |
 | `IRIS_HOSTGL_STRINGS` | host passes through host GL strings instead of compatibility strings. | [iris-hostgl/src/service.rs](iris-hostgl/src/service.rs) |

iris-hostcall/src/lib.rs

@@ -20,6 +20,7 @@
 //! | success   | 0    | result            | second result                |
 //! | error     | 1    | errno             | -                            |
 //! | need page | 1    | [`ENEEDPAGE`]     | page address, bit 0 = write  |
+//! | not yet   | 1    | [`ENOTYET`]       | -                            |
 //!
 //! Arguments are in `$4`..`$11` whatever the program's ABI (the caller's stub
 //! loads them). Callers should use IRIX's **indirect** form -- `v0` = 1000,
@@ -63,6 +64,12 @@ pub const SYS_SYSCALL: u32 = 1000;
 /// does not know about it reports an unknown error rather than a wrong one.
 pub const ENEEDPAGE: u64 = 0x3000;
 
+/// `v0` when the call cannot run yet and the caller should make it again
+/// later, letting its machine run meanwhile (yielding, say). A service gives
+/// it only to callers that asked for work it has not finished -- host GL's
+/// unanswered buffers -- so a caller that never does need not know it.
+pub const ENOTYET: u64 = 0x3001;
+
 /// IRIX's page size on the Indy.
 pub const PAGE: u64 = 0x1000;
 

iris-hostgl/src/lib.rs

@@ -26,6 +26,8 @@
 //! - `emul`: the SGI extensions the host has not got, done in the fragment
 //!   stage with shaders over the fixed-function state.
 //! - `draw`: drawables and the swap.
+//! - `threaded`: the service on its own thread, so the emulated CPU is not
+//!   held while a command buffer runs.
 //! - `backend`: what is platform-specific -- contexts, entry point lookup, a
 //!   CPU-addressable render target. Apple's CGL today.
 //! - `calls`, `ext`: generated tables.
@@ -55,6 +57,8 @@ mod present;
 pub mod qos;
 #[cfg(target_os = "macos")]
 mod service;
+#[cfg(target_os = "macos")]
+mod threaded;
 
 #[cfg(all(test, target_os = "macos"))]
 mod tests;
@@ -65,23 +69,30 @@ use exec::Exec;
 
 #[cfg(target_os = "macos")]
 pub use service::{
-    GlService, OP_BATCH, OP_CREATE, OP_DESTROY, OP_DRAWABLE_GONE, OP_FINISH, OP_GETSTRING, OP_GLX_STRING, OP_GOODBYE, OP_HELLO,
+    GlService, OP_BATCH, OP_BATCH_ASYNC, OP_CREATE, OP_DESTROY, OP_DRAWABLE_GONE, OP_FINISH, OP_GETSTRING, OP_GLX_STRING, OP_GOODBYE, OP_HELLO,
     OP_MAKECURRENT, OP_PBUFFER, OP_RELEASE, OP_SWAP, OP_SWAP_INTERVAL, OP_VIDEO_SYNC, PROTOCOL,
 };
 
 /// Answer host call [`iris_hostcall::GL`] with a new host GL service.
 ///
 /// Registering again replaces the service, destroying every context the old
 /// one held -- which is what a guest reboot wants, since no process that
-/// made them is left. Call it from the thread that will make the calls (the
-/// CPU thread) or before that thread starts: contexts are made current on the
-/// thread that uses them, whichever it is.
+/// made them is left. The service runs on a thread of its own, which makes
+/// and uses every context (threaded.rs); with `IRIS_HOSTGL_INLINE=1` it runs
+/// on the thread that makes the calls (the CPU thread), so call this from that
+/// thread or before it starts.
 pub fn register() {
     #[cfg(target_os = "macos")]
     {
         let backend = Box::new(backend::cgl::Cgl::new());
-        iris_hostcall::register(iris_hostcall::GL, Box::new(GlService::new(backend)));
-        log::info!("host GL: service registered (CGL backend)");
+        let svc = GlService::new(backend);
+        if std::env::var_os("IRIS_HOSTGL_INLINE").is_some() {
+            iris_hostcall::register(iris_hostcall::GL, Box::new(svc));
+            log::info!("host GL: service registered (CGL backend, on the CPU thread)");
+        } else {
+            iris_hostcall::register(iris_hostcall::GL, Box::new(threaded::Threaded::spawn(svc)));
+            log::info!("host GL: service registered (CGL backend, on its own thread)");
+        }
     }
     #[cfg(not(target_os = "macos"))]
     log::info!("host GL: no backend for this platform; host call {} stays unanswered", iris_hostcall::GL);

iris-hostgl/src/service.rs

@@ -72,6 +72,12 @@ pub const OP_SWAP_INTERVAL: u64 = 0x1_000B;
 pub const OP_VIDEO_SYNC: u64 = 0x1_000C;
 pub const OP_GLX_STRING: u64 = 0x1_000D;
 pub const OP_GOODBYE: u64 = 0x1_000E;
+/// A command buffer the program does not wait for: answered as soon as its
+/// bytes are read, run afterwards on the GL thread (threaded.rs), in order
+/// with every later call. The library sends only buffers that read nothing
+/// outside themselves this way. Without the GL thread it is OP_BATCH; a host
+/// without it answers EINVAL, which is how the library finds out.
+pub const OP_BATCH_ASYNC: u64 = 0x1_0010;
 /// What the presenting thread has done: frames presented in v0, frames
 /// replaced before they could be in v1. Global, not per client -- one thread
 /// serves every channel -- and answerable without a context.
@@ -80,8 +86,8 @@ pub const OP_PRESENT_STATS: u64 = 0x1_000F;
 /// The protocol: HELLO's second result, which the library checks.
 pub const PROTOCOL: u64 = 2;
 
-const EINVAL: u64 = 22;
-const EFAULT: u64 = 14;
+pub(crate) const EINVAL: u64 = 22;
+pub(crate) const EFAULT: u64 = 14;
 
 /// A pbuffer's drawable id, which must not collide with an X window's (X ids
 /// keep their top three bits clear) -- and must not have bit 31 set: a 32-bit
@@ -92,7 +98,7 @@ const PBUFFER_ID: u32 = 0x4000_0000;
 /// The largest drawable, either way.
 const MAX_DIM: i32 = 8192;
 /// The largest command buffer (the library's is 128 KB).
-const MAX_BATCH: usize = 16 << 20;
+pub(crate) const MAX_BATCH: usize = 16 << 20;
 /// The frame clock the video-sync counter counts, as a real SGI's display
 /// would (GLX_SGI_video_sync).
 const FRAME_NS: u128 = 1_000_000_000 / 60;
@@ -256,6 +262,25 @@ impl GlService {
         self.resumes.push_back(Resume { args: *args, stage });
     }
 
+    /// An OP_BATCH_ASYNC buffer, run on the GL thread after its call was
+    /// answered. Its bytes were read then; nothing else of the program's
+    /// memory can be, since the program has moved on. A command that tries
+    /// (the library should never send one) stops the buffer there.
+    pub fn run_detached(&mut self, client: u32, bytes: Vec<u8>) {
+        if !self.clients.contains_key(&client) {
+            return;
+        }
+        self.select_client(client);
+        // Never a real call's registers: those carry a serial.
+        let args = [OP_BATCH_ASYNC, 0, client as u64, 0, 0, 0, 0, 0];
+        if let Reply::NeedPage(f) = self.run_batch(&mut Detached, &args, client, bytes, 0, 0, Vec::new()) {
+            self.resumes.retain(|r| r.args != args);
+            if self.reported.insert(usize::MAX - 2) {
+                log::warn!("host GL: a batch sent without waiting reached for program memory at {:#x}; it was cut short there", f.page);
+            }
+        }
+    }
+
     /// Phase 3: store the call's results in the guest, then answer.
     fn finish(&mut self, mem: &mut dyn GuestMemory, args: &[u64; 8], writes: Writes, reply: Reply) -> Reply {
         let writes = merge(writes);
@@ -432,7 +457,7 @@ impl Service for GlService {
             };
         }
         match op {
-            OP_BATCH => {
+            OP_BATCH | OP_BATCH_ASYNC => {
                 let [buf, len] = slots!(2);
                 if len == 0 {
                     return Reply::Ok(0, 0);
@@ -756,8 +781,20 @@ impl Drop for GlService {
     }
 }
 
+/// The program's memory as a detached batch sees it: out of reach.
+struct Detached;
+
+impl GuestMemory for Detached {
+    fn read(&mut self, addr: u64, _buf: &mut [u8]) -> Result<(), Fault> {
+        Err(Fault { page: addr & !(iris_hostcall::PAGE - 1), write: false })
+    }
+    fn write(&mut self, addr: u64, _data: &[u8]) -> Result<(), Fault> {
+        Err(Fault { page: addr & !(iris_hostcall::PAGE - 1), write: true })
+    }
+}
+
 /// `n` 8-byte big-endian argument slots at guest address `at`.
-fn read_slots<const N: usize>(mem: &mut dyn GuestMemory, at: u64) -> Result<[u64; N], Fault> {
+pub(crate) fn read_slots<const N: usize>(mem: &mut dyn GuestMemory, at: u64) -> Result<[u64; N], Fault> {
     let bytes = read_guest(mem, at, N * 8)?;
     let mut out = [0u64; N];
     for (i, v) in out.iter_mut().enumerate() {

iris-hostgl/src/tests.rs

@@ -1441,3 +1441,154 @@ fn accumulation_buffer_operations() {
     assert_eq!(u32::from_be_bytes(g.mem.get(bits, 4).try_into().unwrap()), 16, "GL_ACCUM_RED_BITS");
     near(pixel(g.mem.get(px, 32 * 32 * 4), 32, 2, 2), [0, 255, 0], "the program's colour after a pass");
 }
+
+// ---- the GL thread (threaded.rs) ----
+
+/// One call through a service other than a bare GlService, need-page and
+/// not-yet retries done the library's way.
+fn call_on(svc: &mut dyn Service, mem: &mut Proc, args: [u64; 8]) -> Reply {
+    for _ in 0..1_000_000 {
+        match svc.call(mem, &args) {
+            Reply::NeedPage(f) => mem.touch(f),
+            Reply::Err(iris_hostcall::ENOTYET) => std::thread::yield_now(),
+            r => return r,
+        }
+    }
+    panic!("no progress");
+}
+
+/// A process talking to the GL thread: hello, a context on a 64 x 64 window.
+struct Threads {
+    svc: crate::threaded::Threaded,
+    mem: Proc,
+    client: u64,
+    serial: u64,
+    slots: u64,
+}
+
+impl Threads {
+    fn new() -> Threads {
+        let mut svc = crate::threaded::Threaded::spawn(GlService::new(Box::new(Cgl::new())));
+        let mut mem = Proc::new(512);
+        let slots = mem.alloc(64, 0);
+        let Reply::Ok(client, PROTOCOL) = call_on(&mut svc, &mut mem, [OP_HELLO, PROTOCOL, 0, 0, 0, 0, 0, 0]) else {
+            panic!("hello")
+        };
+        let mut t = Threads { svc, mem, client, serial: 0, slots };
+        let Reply::Ok(ctx, _) = t.call(OP_CREATE, &[0]) else { panic!("create") };
+        assert_eq!(t.call(OP_MAKECURRENT, &[ctx, 0x0040_0002, 0x0040_0002, 64, 64]), Reply::Ok(0, 0));
+        t
+    }
+
+    fn call(&mut self, op: u64, slots: &[u64]) -> Reply {
+        let s: Vec<u8> = slots.iter().flat_map(|v| v.to_be_bytes()).collect();
+        self.mem.put(self.slots, &s);
+        self.serial += 1;
+        call_on(&mut self.svc, &mut self.mem, [op, self.slots, self.client, self.serial, 0, 0, 0, 0])
+    }
+
+    /// `cmds` in fresh memory, every page of it paged out, sent as `op`.
+    fn batch(&mut self, op: u64, cmds: &[u8]) -> Reply {
+        let at = self.mem.alloc(cmds.len(), 0x40);
+        self.mem.put(at, cmds);
+        self.mem.page_out(at, cmds.len());
+        self.call(op, &[at, cmds.len() as u64])
+    }
+}
+
+/// Buffers sent without waiting run on the GL thread, in order, before a
+/// call that waits: the readback sees both draws, the later one on top.
+#[test]
+fn batches_sent_without_waiting_run_in_order_before_a_waiting_call() {
+    let mut t = Threads::new();
+    let mut e = Enc::default();
+    ortho(&mut e, 64, 64);
+    e.cmd("glClearColor", &[V::F(1.0), V::F(0.0), V::F(0.0), V::F(1.0)]).cmd("glClear", &[V::I(GL_COLOR_BUFFER_BIT)]);
+    let first = e.take();
+    let mut e = Enc::default();
+    e.cmd("glColor3ub", &[V::I(0), V::I(255), V::I(0)])
+        .cmd("glBegin", &[V::I(GL_QUADS)])
+        .cmd("glVertex2f", &[V::F(0.0), V::F(0.0)])
+        .cmd("glVertex2f", &[V::F(32.0), V::F(0.0)])
+        .cmd("glVertex2f", &[V::F(32.0), V::F(64.0)])
+        .cmd("glVertex2f", &[V::F(0.0), V::F(64.0)])
+        .cmd("glEnd", &[]);
+    let second = e.take();
+    assert_eq!(t.batch(OP_BATCH_ASYNC, &first), Reply::Ok(0, 0));
+    assert_eq!(t.batch(OP_BATCH_ASYNC, &second), Reply::Ok(0, 0));
+
+    let px = t.mem.alloc(64 * 64 * 4, 0);
+    let mut e = Enc::default();
+    e.cmd("glReadPixels", &[V::I(0), V::I(0), V::I(64), V::I(64), V::I(GL_RGBA), V::I(GL_UNSIGNED_BYTE), V::A(px)]);
+    let read = e.take();
+    assert_eq!(t.batch(OP_BATCH, &read), Reply::Ok(0, 0));
+    let out = t.mem.get(px, 64 * 64 * 4).to_vec();
+    assert_eq!(pixel(&out, 64, 8, 32), [0, 255, 0, 255], "the second buffer's quad");
+    assert_eq!(t.mem.get(px, 4), &pixel(&out, 64, 0, 0), "read back once, whole");
+    assert_eq!(pixel(&out, 64, 56, 32), [255, 0, 0, 255], "the first buffer's clear");
+}
+
+/// A buffer sent without waiting cannot read or write the program's memory:
+/// the program has moved on. One that tries is cut short there, writes
+/// nothing, and the next call is answered as usual.
+#[test]
+fn a_batch_sent_without_waiting_that_reaches_for_memory_is_cut_short() {
+    let mut t = Threads::new();
+    let px = t.mem.alloc(64 * 64 * 4, 0);
+    let mut e = Enc::default();
+    ortho(&mut e, 64, 64);
+    e.cmd("glClearColor", &[V::F(0.0), V::F(0.0), V::F(1.0), V::F(1.0)])
+        .cmd("glClear", &[V::I(GL_COLOR_BUFFER_BIT)])
+        .cmd("glReadPixels", &[V::I(0), V::I(0), V::I(64), V::I(64), V::I(GL_RGBA), V::I(GL_UNSIGNED_BYTE), V::A(px)]);
+    let cmds = e.take();
+    assert_eq!(t.batch(OP_BATCH_ASYNC, &cmds), Reply::Ok(0, 0));
+    let mut e = Enc::default();
+    e.cmd("glGetError", &[]);
+    let cmds = e.take();
+    assert_eq!(t.batch(OP_BATCH, &cmds), Reply::Ok(0, 0), "the GL thread still answers");
+    assert!(t.mem.get(px, 64 * 64 * 4).iter().all(|&b| b == 0), "nothing was written");
+}
+
+/// Without the GL thread, OP_BATCH_ASYNC is an ordinary batch.
+#[test]
+fn on_the_cpu_thread_a_batch_sent_without_waiting_is_a_batch() {
+    let mut g = Guest::new(256);
+    g.context(0x0040_0003, 64, 64);
+    let px = g.mem.alloc(64 * 64 * 4, 0);
+    let mut e = Enc::default();
+    e.cmd("glClearColor", &[V::F(0.0), V::F(1.0), V::F(0.0), V::F(1.0)])
+        .cmd("glClear", &[V::I(GL_COLOR_BUFFER_BIT)])
+        .cmd("glReadPixels", &[V::I(0), V::I(0), V::I(64), V::I(64), V::I(GL_RGBA), V::I(GL_UNSIGNED_BYTE), V::A(px)]);
+    let cmds = e.take();
+    let at = g.mem.alloc(cmds.len(), 0);
+    g.mem.put(at, &cmds);
+    assert_eq!(g.call(OP_BATCH_ASYNC, &[at, cmds.len() as u64]), Reply::Ok(0, 0));
+    assert_eq!(pixel(g.mem.get(px, 64 * 64 * 4), 64, 10, 10), [0, 255, 0, 255]);
+}
+
+/// While a client's buffers are queued, its next call is not held behind
+/// them -- the emulated CPU would stand still -- but answered ENOTYET; once
+/// they have run, the same call is answered as usual. Another client is
+/// never told ENOTYET for this one's buffers.
+#[test]
+fn a_call_behind_queued_buffers_is_told_to_come_back() {
+    let mut t = Threads::new();
+    let mut e = Enc::default();
+    ortho(&mut e, 64, 64);
+    e.cmd("glBegin", &[V::I(GL_QUADS)]);
+    while e.buf.len() < 120 << 10 {
+        e.cmd("glVertex2f", &[V::F(1.0), V::F(1.0)]);
+    }
+    e.cmd("glEnd", &[]);
+    let big = e.take();
+    for _ in 0..4 {
+        assert_eq!(t.batch(OP_BATCH_ASYNC, &big), Reply::Ok(0, 0));
+    }
+    t.serial += 1;
+    let args = [OP_FINISH, t.slots, t.client, t.serial, 0, 0, 0, 0];
+    assert_eq!(t.svc.call(&mut t.mem, &args), Reply::Err(iris_hostcall::ENOTYET), "four big buffers cannot have run yet");
+    assert_eq!(call_on(&mut t.svc, &mut t.mem, args), Reply::Ok(0, 0), "answered once they have");
+
+    let mut other = Proc::new(16);
+    let Reply::Ok(_, PROTOCOL) = call_on(&mut t.svc, &mut other, [OP_HELLO, PROTOCOL, 0, 0, 0, 0, 0, 0]) else { panic!("hello") };
+}